diff --git a/.gitattributes b/.gitattributes index 89f19f8e80238865b22a8937cd3e16a2dd8615f1..8f99eceb9d29cf206143639a3d825d3fb4396539 100644 --- a/.gitattributes +++ b/.gitattributes @@ -130,3 +130,35 @@ dllm-t-qwen3-0.6b-mdlm_v3/mathpros1sf/checkpoint-35000/tokenizer.json filter=lfs dllm-t-qwen3-0.6b-mdlm_v3/mathpros1sf/checkpoint-37500/tokenizer.json filter=lfs diff=lfs merge=lfs -text dllm-t-qwen3-0.6b-mdlm_v3/mathpros1sf/checkpoint-5000/tokenizer.json filter=lfs diff=lfs merge=lfs -text dllm-t-qwen3-0.6b-mdlm_v3/mathpros1sf/checkpoint-7500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/latest new file mode 100644 index 0000000000000000000000000000000000000000..25c776ee3abcad1c4d1e16e8275e4e00984a237c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/latest @@ -0,0 +1 @@ +global_step10000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..431af262e1ddeece040fe7aadaeb8f4fd7e8797d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5ea9ac3c40e0fc1c9a95729bd0ff81a5bc926975c03f5ac7fac6dacc09b0e11 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..93eb88d3a2210505ee5bb295486e83c53bb8baff --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d140b749b56ee2b256c044e3b58df082afa914d00c746566fe749b034c1edcd +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..b4ea86bb3fea1f4a223d39ad908b2082c2edf314 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d668ee177b90cb198cf194eeb442cf2cc9091e6d0e2a049fb5637f4e7806efb2 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..d253a1f68842a4b271f4debf455ce410599da8dd --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf9f8e80c635716da2d309956424016cb73510422c1e34e10e7a7fa0971ae695 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..a5dce157d0e50d75f7b97caf212b1e59a67e9b34 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd9ddd9f4ccbbc449f60e729687a52b46caac115a82f40da776bd7fbd44d0e2c +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..f0c972f6e74d4fb0af86d99deb73da669afd5092 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1a703839291c6c1e6e16389a8563ad52d335c29bbc1ac1489b7d9d45e6f9a4d +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..03f5a1752e7d67f9eca9475d2eb5f4203613464c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6a81ca1766d94a2894ec20c51036d47e7ae6461649fda0f9d77eff1449889b3 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..dbfc613b4e2343a7dbb2e58ac889b12efc85eb65 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:763677630a554cc19e023637bbde7db5d8b972c9c9855b11f3ec069b8a2685f2 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..a2d5c72f7f8a79c16e66e700460b8a27871c75fa --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33bdccbf6e97450c0d4a4d52ca193c4afe5c1290808067cbedfe7eec8dfa4fe8 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c3b38e2c4b51d5a99b693a86d505062af0a9098 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:034d71d0f0787f0c3ec923c15d690debaef12acfa289bd37c34adb31a082231b +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2dfc80de8371f71fb1b40331e761e62295ef1fb8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/trainer_state.json @@ -0,0 +1,7034 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 16.1031893989016, + "eval_steps": 2000, + "global_step": 10000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.016123343578374565, + "grad_norm": 48.11856460571289, + "learning_rate": 0.0001, + "loss": 299.1088, + "step": 10 + }, + { + "epoch": 0.03224668715674913, + "grad_norm": 9.07711410522461, + "learning_rate": 0.0001, + "loss": 234.0888, + "step": 20 + }, + { + "epoch": 0.0483700307351237, + "grad_norm": 3.5935330390930176, + "learning_rate": 0.0001, + "loss": 231.4619, + "step": 30 + }, + { + "epoch": 0.06449337431349826, + "grad_norm": 3.5784752368927, + "learning_rate": 0.0001, + "loss": 229.7082, + "step": 40 + }, + { + "epoch": 0.08061671789187283, + "grad_norm": 3.5757956504821777, + "learning_rate": 0.0001, + "loss": 230.1062, + "step": 50 + }, + { + "epoch": 0.0967400614702474, + "grad_norm": 3.8763978481292725, + "learning_rate": 0.0001, + "loss": 229.1914, + "step": 60 + }, + { + "epoch": 0.11286340504862195, + "grad_norm": 3.006988525390625, + "learning_rate": 0.0001, + "loss": 229.6482, + "step": 70 + }, + { + "epoch": 0.12898674862699652, + "grad_norm": 2.3527095317840576, + "learning_rate": 0.0001, + "loss": 228.1141, + "step": 80 + }, + { + "epoch": 0.14511009220537108, + "grad_norm": 5.353086471557617, + "learning_rate": 0.0001, + "loss": 228.1448, + "step": 90 + }, + { + "epoch": 0.16123343578374566, + "grad_norm": 2.251854419708252, + "learning_rate": 0.0001, + "loss": 228.3897, + "step": 100 + }, + { + "epoch": 0.17735677936212021, + "grad_norm": 1.5789915323257446, + "learning_rate": 0.0001, + "loss": 227.7662, + "step": 110 + }, + { + "epoch": 0.1934801229404948, + "grad_norm": 3.7646398544311523, + "learning_rate": 0.0001, + "loss": 223.6411, + "step": 120 + }, + { + "epoch": 0.20960346651886935, + "grad_norm": 3.444000720977783, + "learning_rate": 0.0001, + "loss": 221.2073, + "step": 130 + }, + { + "epoch": 0.2257268100972439, + "grad_norm": 2.1871185302734375, + "learning_rate": 0.0001, + "loss": 215.9944, + "step": 140 + }, + { + "epoch": 0.2418501536756185, + "grad_norm": 2.266244411468506, + "learning_rate": 0.0001, + "loss": 212.3066, + "step": 150 + }, + { + "epoch": 0.25797349725399304, + "grad_norm": 2.043832778930664, + "learning_rate": 0.0001, + "loss": 208.1343, + "step": 160 + }, + { + "epoch": 0.2740968408323676, + "grad_norm": 2.2641494274139404, + "learning_rate": 0.0001, + "loss": 203.1485, + "step": 170 + }, + { + "epoch": 0.29022018441074215, + "grad_norm": 2.9727790355682373, + "learning_rate": 0.0001, + "loss": 198.181, + "step": 180 + }, + { + "epoch": 0.30634352798911674, + "grad_norm": 2.751523494720459, + "learning_rate": 0.0001, + "loss": 194.8422, + "step": 190 + }, + { + "epoch": 0.3224668715674913, + "grad_norm": 1.9561562538146973, + "learning_rate": 0.0001, + "loss": 189.9401, + "step": 200 + }, + { + "epoch": 0.33859021514586585, + "grad_norm": 2.3264107704162598, + "learning_rate": 0.0001, + "loss": 186.1309, + "step": 210 + }, + { + "epoch": 0.35471355872424043, + "grad_norm": 5.212426662445068, + "learning_rate": 0.0001, + "loss": 182.2255, + "step": 220 + }, + { + "epoch": 0.370836902302615, + "grad_norm": 2.488032102584839, + "learning_rate": 0.0001, + "loss": 181.0103, + "step": 230 + }, + { + "epoch": 0.3869602458809896, + "grad_norm": 2.6173202991485596, + "learning_rate": 0.0001, + "loss": 177.6162, + "step": 240 + }, + { + "epoch": 0.4030835894593641, + "grad_norm": 2.6245291233062744, + "learning_rate": 0.0001, + "loss": 173.0431, + "step": 250 + }, + { + "epoch": 0.4192069330377387, + "grad_norm": 5.356006145477295, + "learning_rate": 0.0001, + "loss": 172.2371, + "step": 260 + }, + { + "epoch": 0.4353302766161133, + "grad_norm": 2.0453829765319824, + "learning_rate": 0.0001, + "loss": 169.7781, + "step": 270 + }, + { + "epoch": 0.4514536201944878, + "grad_norm": 1.7002843618392944, + "learning_rate": 0.0001, + "loss": 166.9564, + "step": 280 + }, + { + "epoch": 0.4675769637728624, + "grad_norm": 2.181410312652588, + "learning_rate": 0.0001, + "loss": 166.0094, + "step": 290 + }, + { + "epoch": 0.483700307351237, + "grad_norm": 2.2948265075683594, + "learning_rate": 0.0001, + "loss": 162.4388, + "step": 300 + }, + { + "epoch": 0.4998236509296115, + "grad_norm": 3.0272789001464844, + "learning_rate": 0.0001, + "loss": 160.3446, + "step": 310 + }, + { + "epoch": 0.5159469945079861, + "grad_norm": 1.948184847831726, + "learning_rate": 0.0001, + "loss": 158.488, + "step": 320 + }, + { + "epoch": 0.5320703380863606, + "grad_norm": 1.7981538772583008, + "learning_rate": 0.0001, + "loss": 156.2107, + "step": 330 + }, + { + "epoch": 0.5481936816647353, + "grad_norm": 2.049323797225952, + "learning_rate": 0.0001, + "loss": 155.5434, + "step": 340 + }, + { + "epoch": 0.5643170252431098, + "grad_norm": 1.9415442943572998, + "learning_rate": 0.0001, + "loss": 153.8682, + "step": 350 + }, + { + "epoch": 0.5804403688214843, + "grad_norm": 2.2168045043945312, + "learning_rate": 0.0001, + "loss": 150.4087, + "step": 360 + }, + { + "epoch": 0.596563712399859, + "grad_norm": 1.510750651359558, + "learning_rate": 0.0001, + "loss": 151.7655, + "step": 370 + }, + { + "epoch": 0.6126870559782335, + "grad_norm": 2.7469751834869385, + "learning_rate": 0.0001, + "loss": 147.4665, + "step": 380 + }, + { + "epoch": 0.628810399556608, + "grad_norm": 2.0436975955963135, + "learning_rate": 0.0001, + "loss": 147.4405, + "step": 390 + }, + { + "epoch": 0.6449337431349826, + "grad_norm": 1.8219181299209595, + "learning_rate": 0.0001, + "loss": 146.6332, + "step": 400 + }, + { + "epoch": 0.6610570867133572, + "grad_norm": 1.948431134223938, + "learning_rate": 0.0001, + "loss": 143.3298, + "step": 410 + }, + { + "epoch": 0.6771804302917317, + "grad_norm": 1.8887908458709717, + "learning_rate": 0.0001, + "loss": 140.5058, + "step": 420 + }, + { + "epoch": 0.6933037738701063, + "grad_norm": 1.8739320039749146, + "learning_rate": 0.0001, + "loss": 137.6334, + "step": 430 + }, + { + "epoch": 0.7094271174484809, + "grad_norm": 2.057689666748047, + "learning_rate": 0.0001, + "loss": 135.0343, + "step": 440 + }, + { + "epoch": 0.7255504610268555, + "grad_norm": 1.6926478147506714, + "learning_rate": 0.0001, + "loss": 132.8766, + "step": 450 + }, + { + "epoch": 0.74167380460523, + "grad_norm": 1.5039501190185547, + "learning_rate": 0.0001, + "loss": 131.6774, + "step": 460 + }, + { + "epoch": 0.7577971481836046, + "grad_norm": 1.6903547048568726, + "learning_rate": 0.0001, + "loss": 129.3116, + "step": 470 + }, + { + "epoch": 0.7739204917619792, + "grad_norm": 3.7062244415283203, + "learning_rate": 0.0001, + "loss": 127.9314, + "step": 480 + }, + { + "epoch": 0.7900438353403537, + "grad_norm": 2.0586934089660645, + "learning_rate": 0.0001, + "loss": 126.6398, + "step": 490 + }, + { + "epoch": 0.8061671789187282, + "grad_norm": 1.7598215341567993, + "learning_rate": 0.0001, + "loss": 125.1501, + "step": 500 + }, + { + "epoch": 0.8222905224971029, + "grad_norm": 1.5992189645767212, + "learning_rate": 0.0001, + "loss": 124.7039, + "step": 510 + }, + { + "epoch": 0.8384138660754774, + "grad_norm": 2.8735458850860596, + "learning_rate": 0.0001, + "loss": 122.1619, + "step": 520 + }, + { + "epoch": 0.8545372096538519, + "grad_norm": 1.867098093032837, + "learning_rate": 0.0001, + "loss": 122.4506, + "step": 530 + }, + { + "epoch": 0.8706605532322266, + "grad_norm": 2.2088418006896973, + "learning_rate": 0.0001, + "loss": 122.1047, + "step": 540 + }, + { + "epoch": 0.8867838968106011, + "grad_norm": 1.642657995223999, + "learning_rate": 0.0001, + "loss": 120.1141, + "step": 550 + }, + { + "epoch": 0.9029072403889756, + "grad_norm": 1.5567114353179932, + "learning_rate": 0.0001, + "loss": 118.532, + "step": 560 + }, + { + "epoch": 0.9190305839673503, + "grad_norm": 1.667843222618103, + "learning_rate": 0.0001, + "loss": 118.0734, + "step": 570 + }, + { + "epoch": 0.9351539275457248, + "grad_norm": 1.5304442644119263, + "learning_rate": 0.0001, + "loss": 114.4925, + "step": 580 + }, + { + "epoch": 0.9512772711240993, + "grad_norm": 1.6920255422592163, + "learning_rate": 0.0001, + "loss": 114.0729, + "step": 590 + }, + { + "epoch": 0.967400614702474, + "grad_norm": 1.6313318014144897, + "learning_rate": 0.0001, + "loss": 114.2879, + "step": 600 + }, + { + "epoch": 0.9835239582808485, + "grad_norm": 1.9033657312393188, + "learning_rate": 0.0001, + "loss": 110.5299, + "step": 610 + }, + { + "epoch": 0.999647301859223, + "grad_norm": 1.9010754823684692, + "learning_rate": 0.0001, + "loss": 109.1145, + "step": 620 + }, + { + "epoch": 1.0145110092205372, + "grad_norm": 3.5390989780426025, + "learning_rate": 0.0001, + "loss": 101.0258, + "step": 630 + }, + { + "epoch": 1.0306343527989117, + "grad_norm": 2.2925171852111816, + "learning_rate": 0.0001, + "loss": 107.3693, + "step": 640 + }, + { + "epoch": 1.0467576963772862, + "grad_norm": 2.329108238220215, + "learning_rate": 0.0001, + "loss": 104.3778, + "step": 650 + }, + { + "epoch": 1.0628810399556607, + "grad_norm": 1.7325661182403564, + "learning_rate": 0.0001, + "loss": 102.8409, + "step": 660 + }, + { + "epoch": 1.0790043835340353, + "grad_norm": 1.989586353302002, + "learning_rate": 0.0001, + "loss": 104.2577, + "step": 670 + }, + { + "epoch": 1.09512772711241, + "grad_norm": 1.900484323501587, + "learning_rate": 0.0001, + "loss": 102.3051, + "step": 680 + }, + { + "epoch": 1.1112510706907845, + "grad_norm": 2.015859842300415, + "learning_rate": 0.0001, + "loss": 100.893, + "step": 690 + }, + { + "epoch": 1.127374414269159, + "grad_norm": 1.7695508003234863, + "learning_rate": 0.0001, + "loss": 99.215, + "step": 700 + }, + { + "epoch": 1.1434977578475336, + "grad_norm": 2.025923490524292, + "learning_rate": 0.0001, + "loss": 98.8768, + "step": 710 + }, + { + "epoch": 1.1596211014259081, + "grad_norm": 2.946708917617798, + "learning_rate": 0.0001, + "loss": 96.5588, + "step": 720 + }, + { + "epoch": 1.1757444450042827, + "grad_norm": 2.4511258602142334, + "learning_rate": 0.0001, + "loss": 95.0545, + "step": 730 + }, + { + "epoch": 1.1918677885826574, + "grad_norm": 2.3094682693481445, + "learning_rate": 0.0001, + "loss": 97.004, + "step": 740 + }, + { + "epoch": 1.207991132161032, + "grad_norm": 2.1313347816467285, + "learning_rate": 0.0001, + "loss": 94.8819, + "step": 750 + }, + { + "epoch": 1.2241144757394065, + "grad_norm": 1.614182710647583, + "learning_rate": 0.0001, + "loss": 95.5339, + "step": 760 + }, + { + "epoch": 1.240237819317781, + "grad_norm": 2.029953956604004, + "learning_rate": 0.0001, + "loss": 93.7303, + "step": 770 + }, + { + "epoch": 1.2563611628961555, + "grad_norm": 1.7871663570404053, + "learning_rate": 0.0001, + "loss": 93.8354, + "step": 780 + }, + { + "epoch": 1.2724845064745303, + "grad_norm": 1.72455632686615, + "learning_rate": 0.0001, + "loss": 94.7348, + "step": 790 + }, + { + "epoch": 1.2886078500529048, + "grad_norm": 1.9776639938354492, + "learning_rate": 0.0001, + "loss": 93.5957, + "step": 800 + }, + { + "epoch": 1.3047311936312793, + "grad_norm": 1.7187533378601074, + "learning_rate": 0.0001, + "loss": 92.9583, + "step": 810 + }, + { + "epoch": 1.3208545372096538, + "grad_norm": 1.5949162244796753, + "learning_rate": 0.0001, + "loss": 93.3088, + "step": 820 + }, + { + "epoch": 1.3369778807880284, + "grad_norm": 1.4616206884384155, + "learning_rate": 0.0001, + "loss": 93.2125, + "step": 830 + }, + { + "epoch": 1.353101224366403, + "grad_norm": 2.832163095474243, + "learning_rate": 0.0001, + "loss": 90.3301, + "step": 840 + }, + { + "epoch": 1.3692245679447774, + "grad_norm": 2.174501419067383, + "learning_rate": 0.0001, + "loss": 91.6988, + "step": 850 + }, + { + "epoch": 1.3853479115231522, + "grad_norm": 1.2156339883804321, + "learning_rate": 0.0001, + "loss": 92.7419, + "step": 860 + }, + { + "epoch": 1.4014712551015267, + "grad_norm": 1.608261227607727, + "learning_rate": 0.0001, + "loss": 89.8255, + "step": 870 + }, + { + "epoch": 1.4175945986799012, + "grad_norm": 1.6671923398971558, + "learning_rate": 0.0001, + "loss": 90.9683, + "step": 880 + }, + { + "epoch": 1.4337179422582758, + "grad_norm": 1.2346433401107788, + "learning_rate": 0.0001, + "loss": 90.6092, + "step": 890 + }, + { + "epoch": 1.4498412858366505, + "grad_norm": 1.561085820198059, + "learning_rate": 0.0001, + "loss": 88.0842, + "step": 900 + }, + { + "epoch": 1.465964629415025, + "grad_norm": 1.5475201606750488, + "learning_rate": 0.0001, + "loss": 89.8786, + "step": 910 + }, + { + "epoch": 1.4820879729933996, + "grad_norm": 1.6044800281524658, + "learning_rate": 0.0001, + "loss": 88.0364, + "step": 920 + }, + { + "epoch": 1.498211316571774, + "grad_norm": 1.4081908464431763, + "learning_rate": 0.0001, + "loss": 88.8799, + "step": 930 + }, + { + "epoch": 1.5143346601501486, + "grad_norm": 2.341053009033203, + "learning_rate": 0.0001, + "loss": 88.2023, + "step": 940 + }, + { + "epoch": 1.5304580037285231, + "grad_norm": 1.3976799249649048, + "learning_rate": 0.0001, + "loss": 86.6493, + "step": 950 + }, + { + "epoch": 1.5465813473068977, + "grad_norm": 1.2485483884811401, + "learning_rate": 0.0001, + "loss": 88.9224, + "step": 960 + }, + { + "epoch": 1.5627046908852722, + "grad_norm": 1.6824235916137695, + "learning_rate": 0.0001, + "loss": 89.6484, + "step": 970 + }, + { + "epoch": 1.578828034463647, + "grad_norm": 1.64790940284729, + "learning_rate": 0.0001, + "loss": 87.2479, + "step": 980 + }, + { + "epoch": 1.5949513780420215, + "grad_norm": 1.2093443870544434, + "learning_rate": 0.0001, + "loss": 86.8241, + "step": 990 + }, + { + "epoch": 1.611074721620396, + "grad_norm": 1.5306812524795532, + "learning_rate": 0.0001, + "loss": 88.0006, + "step": 1000 + }, + { + "epoch": 1.6271980651987707, + "grad_norm": 1.2465763092041016, + "learning_rate": 0.0001, + "loss": 86.8143, + "step": 1010 + }, + { + "epoch": 1.6433214087771453, + "grad_norm": 1.5224069356918335, + "learning_rate": 0.0001, + "loss": 87.0102, + "step": 1020 + }, + { + "epoch": 1.6594447523555198, + "grad_norm": 1.312179684638977, + "learning_rate": 0.0001, + "loss": 88.7395, + "step": 1030 + }, + { + "epoch": 1.6755680959338943, + "grad_norm": 1.2343158721923828, + "learning_rate": 0.0001, + "loss": 87.2868, + "step": 1040 + }, + { + "epoch": 1.6916914395122689, + "grad_norm": 1.2778265476226807, + "learning_rate": 0.0001, + "loss": 87.043, + "step": 1050 + }, + { + "epoch": 1.7078147830906434, + "grad_norm": 1.2799627780914307, + "learning_rate": 0.0001, + "loss": 87.7843, + "step": 1060 + }, + { + "epoch": 1.723938126669018, + "grad_norm": 1.3688952922821045, + "learning_rate": 0.0001, + "loss": 85.8985, + "step": 1070 + }, + { + "epoch": 1.7400614702473924, + "grad_norm": 1.4317796230316162, + "learning_rate": 0.0001, + "loss": 85.5408, + "step": 1080 + }, + { + "epoch": 1.756184813825767, + "grad_norm": 1.164801001548767, + "learning_rate": 0.0001, + "loss": 85.8484, + "step": 1090 + }, + { + "epoch": 1.7723081574041417, + "grad_norm": 1.3929173946380615, + "learning_rate": 0.0001, + "loss": 86.9031, + "step": 1100 + }, + { + "epoch": 1.7884315009825162, + "grad_norm": 2.116119861602783, + "learning_rate": 0.0001, + "loss": 86.5343, + "step": 1110 + }, + { + "epoch": 1.8045548445608908, + "grad_norm": 1.4117276668548584, + "learning_rate": 0.0001, + "loss": 85.2678, + "step": 1120 + }, + { + "epoch": 1.8206781881392655, + "grad_norm": 1.3213504552841187, + "learning_rate": 0.0001, + "loss": 85.8935, + "step": 1130 + }, + { + "epoch": 1.83680153171764, + "grad_norm": 1.3589770793914795, + "learning_rate": 0.0001, + "loss": 85.032, + "step": 1140 + }, + { + "epoch": 1.8529248752960146, + "grad_norm": 1.3336007595062256, + "learning_rate": 0.0001, + "loss": 84.0993, + "step": 1150 + }, + { + "epoch": 1.869048218874389, + "grad_norm": 1.1081451177597046, + "learning_rate": 0.0001, + "loss": 84.2225, + "step": 1160 + }, + { + "epoch": 1.8851715624527636, + "grad_norm": 1.482038974761963, + "learning_rate": 0.0001, + "loss": 82.1439, + "step": 1170 + }, + { + "epoch": 1.9012949060311382, + "grad_norm": 1.1571478843688965, + "learning_rate": 0.0001, + "loss": 82.5045, + "step": 1180 + }, + { + "epoch": 1.9174182496095127, + "grad_norm": 1.2665468454360962, + "learning_rate": 0.0001, + "loss": 82.8761, + "step": 1190 + }, + { + "epoch": 1.9335415931878872, + "grad_norm": 1.0771509408950806, + "learning_rate": 0.0001, + "loss": 81.5162, + "step": 1200 + }, + { + "epoch": 1.949664936766262, + "grad_norm": 1.2992299795150757, + "learning_rate": 0.0001, + "loss": 82.4038, + "step": 1210 + }, + { + "epoch": 1.9657882803446365, + "grad_norm": 1.2946668863296509, + "learning_rate": 0.0001, + "loss": 83.4837, + "step": 1220 + }, + { + "epoch": 1.981911623923011, + "grad_norm": 1.3505686521530151, + "learning_rate": 0.0001, + "loss": 81.8993, + "step": 1230 + }, + { + "epoch": 1.9980349675013858, + "grad_norm": 1.1247715950012207, + "learning_rate": 0.0001, + "loss": 82.9006, + "step": 1240 + }, + { + "epoch": 2.0128986748627, + "grad_norm": 1.5652127265930176, + "learning_rate": 0.0001, + "loss": 75.8347, + "step": 1250 + }, + { + "epoch": 2.0290220184410743, + "grad_norm": 7.311964988708496, + "learning_rate": 0.0001, + "loss": 80.5435, + "step": 1260 + }, + { + "epoch": 2.045145362019449, + "grad_norm": 1.2229893207550049, + "learning_rate": 0.0001, + "loss": 83.3277, + "step": 1270 + }, + { + "epoch": 2.0612687055978234, + "grad_norm": 1.1387693881988525, + "learning_rate": 0.0001, + "loss": 81.3252, + "step": 1280 + }, + { + "epoch": 2.077392049176198, + "grad_norm": 1.307201623916626, + "learning_rate": 0.0001, + "loss": 82.3311, + "step": 1290 + }, + { + "epoch": 2.0935153927545724, + "grad_norm": 1.1502896547317505, + "learning_rate": 0.0001, + "loss": 81.1682, + "step": 1300 + }, + { + "epoch": 2.109638736332947, + "grad_norm": 0.9291583299636841, + "learning_rate": 0.0001, + "loss": 81.7017, + "step": 1310 + }, + { + "epoch": 2.1257620799113215, + "grad_norm": 1.0334666967391968, + "learning_rate": 0.0001, + "loss": 81.718, + "step": 1320 + }, + { + "epoch": 2.141885423489696, + "grad_norm": 1.1099056005477905, + "learning_rate": 0.0001, + "loss": 81.2232, + "step": 1330 + }, + { + "epoch": 2.1580087670680705, + "grad_norm": 1.289478063583374, + "learning_rate": 0.0001, + "loss": 81.8736, + "step": 1340 + }, + { + "epoch": 2.174132110646445, + "grad_norm": 1.1634794473648071, + "learning_rate": 0.0001, + "loss": 79.9434, + "step": 1350 + }, + { + "epoch": 2.19025545422482, + "grad_norm": 0.9563440680503845, + "learning_rate": 0.0001, + "loss": 80.628, + "step": 1360 + }, + { + "epoch": 2.2063787978031946, + "grad_norm": 1.1268844604492188, + "learning_rate": 0.0001, + "loss": 82.1389, + "step": 1370 + }, + { + "epoch": 2.222502141381569, + "grad_norm": 1.1050443649291992, + "learning_rate": 0.0001, + "loss": 78.9023, + "step": 1380 + }, + { + "epoch": 2.2386254849599436, + "grad_norm": 1.1936142444610596, + "learning_rate": 0.0001, + "loss": 81.2118, + "step": 1390 + }, + { + "epoch": 2.254748828538318, + "grad_norm": 1.7825465202331543, + "learning_rate": 0.0001, + "loss": 80.8804, + "step": 1400 + }, + { + "epoch": 2.2708721721166927, + "grad_norm": 2.437251329421997, + "learning_rate": 0.0001, + "loss": 79.3807, + "step": 1410 + }, + { + "epoch": 2.286995515695067, + "grad_norm": 1.0604854822158813, + "learning_rate": 0.0001, + "loss": 79.5881, + "step": 1420 + }, + { + "epoch": 2.3031188592734417, + "grad_norm": 1.2285529375076294, + "learning_rate": 0.0001, + "loss": 79.5517, + "step": 1430 + }, + { + "epoch": 2.3192422028518163, + "grad_norm": 1.1916660070419312, + "learning_rate": 0.0001, + "loss": 80.5887, + "step": 1440 + }, + { + "epoch": 2.335365546430191, + "grad_norm": 1.224304437637329, + "learning_rate": 0.0001, + "loss": 79.8109, + "step": 1450 + }, + { + "epoch": 2.3514888900085653, + "grad_norm": 0.9228448867797852, + "learning_rate": 0.0001, + "loss": 79.2787, + "step": 1460 + }, + { + "epoch": 2.3676122335869403, + "grad_norm": 1.2085226774215698, + "learning_rate": 0.0001, + "loss": 78.4983, + "step": 1470 + }, + { + "epoch": 2.383735577165315, + "grad_norm": 2.424083948135376, + "learning_rate": 0.0001, + "loss": 79.8657, + "step": 1480 + }, + { + "epoch": 2.3998589207436893, + "grad_norm": 1.0277849435806274, + "learning_rate": 0.0001, + "loss": 80.6915, + "step": 1490 + }, + { + "epoch": 2.415982264322064, + "grad_norm": 1.0226075649261475, + "learning_rate": 0.0001, + "loss": 78.9751, + "step": 1500 + }, + { + "epoch": 2.4321056079004384, + "grad_norm": 0.8860388398170471, + "learning_rate": 0.0001, + "loss": 78.3915, + "step": 1510 + }, + { + "epoch": 2.448228951478813, + "grad_norm": 1.265637755393982, + "learning_rate": 0.0001, + "loss": 78.8737, + "step": 1520 + }, + { + "epoch": 2.4643522950571874, + "grad_norm": 1.2518186569213867, + "learning_rate": 0.0001, + "loss": 76.5169, + "step": 1530 + }, + { + "epoch": 2.480475638635562, + "grad_norm": 1.074277400970459, + "learning_rate": 0.0001, + "loss": 77.9156, + "step": 1540 + }, + { + "epoch": 2.4965989822139365, + "grad_norm": 1.0666711330413818, + "learning_rate": 0.0001, + "loss": 78.4103, + "step": 1550 + }, + { + "epoch": 2.512722325792311, + "grad_norm": 0.9492248892784119, + "learning_rate": 0.0001, + "loss": 76.7393, + "step": 1560 + }, + { + "epoch": 2.5288456693706856, + "grad_norm": 1.127238154411316, + "learning_rate": 0.0001, + "loss": 79.729, + "step": 1570 + }, + { + "epoch": 2.5449690129490605, + "grad_norm": 1.1520992517471313, + "learning_rate": 0.0001, + "loss": 78.5618, + "step": 1580 + }, + { + "epoch": 2.5610923565274346, + "grad_norm": 0.8632035255432129, + "learning_rate": 0.0001, + "loss": 79.032, + "step": 1590 + }, + { + "epoch": 2.5772157001058096, + "grad_norm": 1.0506329536437988, + "learning_rate": 0.0001, + "loss": 77.6204, + "step": 1600 + }, + { + "epoch": 2.593339043684184, + "grad_norm": 1.019895315170288, + "learning_rate": 0.0001, + "loss": 79.2634, + "step": 1610 + }, + { + "epoch": 2.6094623872625586, + "grad_norm": 1.1685571670532227, + "learning_rate": 0.0001, + "loss": 78.8371, + "step": 1620 + }, + { + "epoch": 2.625585730840933, + "grad_norm": 1.147546648979187, + "learning_rate": 0.0001, + "loss": 78.2113, + "step": 1630 + }, + { + "epoch": 2.6417090744193077, + "grad_norm": 1.7336841821670532, + "learning_rate": 0.0001, + "loss": 78.7866, + "step": 1640 + }, + { + "epoch": 2.657832417997682, + "grad_norm": 1.2150614261627197, + "learning_rate": 0.0001, + "loss": 78.5621, + "step": 1650 + }, + { + "epoch": 2.6739557615760567, + "grad_norm": 1.2257503271102905, + "learning_rate": 0.0001, + "loss": 77.5835, + "step": 1660 + }, + { + "epoch": 2.6900791051544313, + "grad_norm": 1.0375311374664307, + "learning_rate": 0.0001, + "loss": 77.7509, + "step": 1670 + }, + { + "epoch": 2.706202448732806, + "grad_norm": 1.2551004886627197, + "learning_rate": 0.0001, + "loss": 77.8615, + "step": 1680 + }, + { + "epoch": 2.7223257923111808, + "grad_norm": 1.0464223623275757, + "learning_rate": 0.0001, + "loss": 77.2837, + "step": 1690 + }, + { + "epoch": 2.738449135889555, + "grad_norm": 1.0254806280136108, + "learning_rate": 0.0001, + "loss": 76.9616, + "step": 1700 + }, + { + "epoch": 2.75457247946793, + "grad_norm": 1.1665682792663574, + "learning_rate": 0.0001, + "loss": 77.9108, + "step": 1710 + }, + { + "epoch": 2.7706958230463044, + "grad_norm": 1.1349453926086426, + "learning_rate": 0.0001, + "loss": 78.7636, + "step": 1720 + }, + { + "epoch": 2.786819166624679, + "grad_norm": 1.4961057901382446, + "learning_rate": 0.0001, + "loss": 76.3862, + "step": 1730 + }, + { + "epoch": 2.8029425102030534, + "grad_norm": 1.10721755027771, + "learning_rate": 0.0001, + "loss": 77.4105, + "step": 1740 + }, + { + "epoch": 2.819065853781428, + "grad_norm": 0.9878576397895813, + "learning_rate": 0.0001, + "loss": 78.4855, + "step": 1750 + }, + { + "epoch": 2.8351891973598025, + "grad_norm": 1.1722687482833862, + "learning_rate": 0.0001, + "loss": 75.1589, + "step": 1760 + }, + { + "epoch": 2.851312540938177, + "grad_norm": 1.6624362468719482, + "learning_rate": 0.0001, + "loss": 77.1647, + "step": 1770 + }, + { + "epoch": 2.8674358845165515, + "grad_norm": 0.9542295932769775, + "learning_rate": 0.0001, + "loss": 77.1656, + "step": 1780 + }, + { + "epoch": 2.883559228094926, + "grad_norm": 2.328542947769165, + "learning_rate": 0.0001, + "loss": 76.4686, + "step": 1790 + }, + { + "epoch": 2.899682571673301, + "grad_norm": 1.2532029151916504, + "learning_rate": 0.0001, + "loss": 76.1727, + "step": 1800 + }, + { + "epoch": 2.915805915251675, + "grad_norm": 0.9604716897010803, + "learning_rate": 0.0001, + "loss": 74.3132, + "step": 1810 + }, + { + "epoch": 2.93192925883005, + "grad_norm": 0.9182595610618591, + "learning_rate": 0.0001, + "loss": 77.8689, + "step": 1820 + }, + { + "epoch": 2.9480526024084246, + "grad_norm": 0.9623042941093445, + "learning_rate": 0.0001, + "loss": 77.7531, + "step": 1830 + }, + { + "epoch": 2.964175945986799, + "grad_norm": 1.0500119924545288, + "learning_rate": 0.0001, + "loss": 77.6007, + "step": 1840 + }, + { + "epoch": 2.9802992895651736, + "grad_norm": 1.0874103307724, + "learning_rate": 0.0001, + "loss": 77.042, + "step": 1850 + }, + { + "epoch": 2.996422633143548, + "grad_norm": 0.9955475330352783, + "learning_rate": 0.0001, + "loss": 75.468, + "step": 1860 + }, + { + "epoch": 3.011286340504862, + "grad_norm": 1.4281760454177856, + "learning_rate": 0.0001, + "loss": 69.5972, + "step": 1870 + }, + { + "epoch": 3.0274096840832367, + "grad_norm": 1.0292123556137085, + "learning_rate": 0.0001, + "loss": 76.0186, + "step": 1880 + }, + { + "epoch": 3.0435330276616113, + "grad_norm": 0.9535816311836243, + "learning_rate": 0.0001, + "loss": 76.0979, + "step": 1890 + }, + { + "epoch": 3.059656371239986, + "grad_norm": 0.9099674820899963, + "learning_rate": 0.0001, + "loss": 75.9641, + "step": 1900 + }, + { + "epoch": 3.0757797148183603, + "grad_norm": 0.9568380117416382, + "learning_rate": 0.0001, + "loss": 75.0656, + "step": 1910 + }, + { + "epoch": 3.091903058396735, + "grad_norm": 0.9701060652732849, + "learning_rate": 0.0001, + "loss": 75.1422, + "step": 1920 + }, + { + "epoch": 3.1080264019751094, + "grad_norm": 1.3879098892211914, + "learning_rate": 0.0001, + "loss": 76.0731, + "step": 1930 + }, + { + "epoch": 3.1241497455534843, + "grad_norm": 0.9782084226608276, + "learning_rate": 0.0001, + "loss": 75.9666, + "step": 1940 + }, + { + "epoch": 3.140273089131859, + "grad_norm": 1.0856964588165283, + "learning_rate": 0.0001, + "loss": 75.0217, + "step": 1950 + }, + { + "epoch": 3.1563964327102334, + "grad_norm": 0.8506444692611694, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 1960 + }, + { + "epoch": 3.172519776288608, + "grad_norm": 1.044980525970459, + "learning_rate": 0.0001, + "loss": 75.3471, + "step": 1970 + }, + { + "epoch": 3.1886431198669825, + "grad_norm": 0.8432291746139526, + "learning_rate": 0.0001, + "loss": 75.3563, + "step": 1980 + }, + { + "epoch": 3.204766463445357, + "grad_norm": 0.9651834964752197, + "learning_rate": 0.0001, + "loss": 76.3991, + "step": 1990 + }, + { + "epoch": 3.2208898070237315, + "grad_norm": 0.957943320274353, + "learning_rate": 0.0001, + "loss": 77.2613, + "step": 2000 + }, + { + "epoch": 3.237013150602106, + "grad_norm": 0.9495745301246643, + "learning_rate": 0.0001, + "loss": 73.3801, + "step": 2010 + }, + { + "epoch": 3.2531364941804806, + "grad_norm": 1.182983160018921, + "learning_rate": 0.0001, + "loss": 74.5695, + "step": 2020 + }, + { + "epoch": 3.269259837758855, + "grad_norm": 0.828452467918396, + "learning_rate": 0.0001, + "loss": 75.4313, + "step": 2030 + }, + { + "epoch": 3.2853831813372296, + "grad_norm": 0.8340104818344116, + "learning_rate": 0.0001, + "loss": 75.1086, + "step": 2040 + }, + { + "epoch": 3.301506524915604, + "grad_norm": 1.1700762510299683, + "learning_rate": 0.0001, + "loss": 75.7496, + "step": 2050 + }, + { + "epoch": 3.317629868493979, + "grad_norm": 0.8709110617637634, + "learning_rate": 0.0001, + "loss": 73.8403, + "step": 2060 + }, + { + "epoch": 3.3337532120723536, + "grad_norm": 0.8928954601287842, + "learning_rate": 0.0001, + "loss": 75.6134, + "step": 2070 + }, + { + "epoch": 3.349876555650728, + "grad_norm": 0.9787353277206421, + "learning_rate": 0.0001, + "loss": 75.8116, + "step": 2080 + }, + { + "epoch": 3.3659998992291027, + "grad_norm": 0.9515123963356018, + "learning_rate": 0.0001, + "loss": 75.5024, + "step": 2090 + }, + { + "epoch": 3.3821232428074772, + "grad_norm": 0.9529262185096741, + "learning_rate": 0.0001, + "loss": 74.1553, + "step": 2100 + }, + { + "epoch": 3.3982465863858518, + "grad_norm": 0.8934710621833801, + "learning_rate": 0.0001, + "loss": 74.1501, + "step": 2110 + }, + { + "epoch": 3.4143699299642263, + "grad_norm": 1.089447021484375, + "learning_rate": 0.0001, + "loss": 74.9123, + "step": 2120 + }, + { + "epoch": 3.430493273542601, + "grad_norm": 1.2569622993469238, + "learning_rate": 0.0001, + "loss": 74.3962, + "step": 2130 + }, + { + "epoch": 3.4466166171209753, + "grad_norm": 1.020094394683838, + "learning_rate": 0.0001, + "loss": 74.6617, + "step": 2140 + }, + { + "epoch": 3.46273996069935, + "grad_norm": 1.1579667329788208, + "learning_rate": 0.0001, + "loss": 74.2745, + "step": 2150 + }, + { + "epoch": 3.4788633042777244, + "grad_norm": 1.448935627937317, + "learning_rate": 0.0001, + "loss": 71.5285, + "step": 2160 + }, + { + "epoch": 3.4949866478560994, + "grad_norm": 1.0030494928359985, + "learning_rate": 0.0001, + "loss": 74.6666, + "step": 2170 + }, + { + "epoch": 3.511109991434474, + "grad_norm": 0.9010888934135437, + "learning_rate": 0.0001, + "loss": 74.9355, + "step": 2180 + }, + { + "epoch": 3.5272333350128484, + "grad_norm": 1.1595124006271362, + "learning_rate": 0.0001, + "loss": 74.5049, + "step": 2190 + }, + { + "epoch": 3.543356678591223, + "grad_norm": 1.066405177116394, + "learning_rate": 0.0001, + "loss": 74.6336, + "step": 2200 + }, + { + "epoch": 3.5594800221695975, + "grad_norm": 1.2301596403121948, + "learning_rate": 0.0001, + "loss": 74.6664, + "step": 2210 + }, + { + "epoch": 3.575603365747972, + "grad_norm": 1.0073511600494385, + "learning_rate": 0.0001, + "loss": 73.1405, + "step": 2220 + }, + { + "epoch": 3.5917267093263465, + "grad_norm": 0.9913995862007141, + "learning_rate": 0.0001, + "loss": 75.7443, + "step": 2230 + }, + { + "epoch": 3.607850052904721, + "grad_norm": 0.9555648565292358, + "learning_rate": 0.0001, + "loss": 74.8036, + "step": 2240 + }, + { + "epoch": 3.6239733964830956, + "grad_norm": 0.9350649118423462, + "learning_rate": 0.0001, + "loss": 73.5783, + "step": 2250 + }, + { + "epoch": 3.64009674006147, + "grad_norm": 0.935840368270874, + "learning_rate": 0.0001, + "loss": 75.1694, + "step": 2260 + }, + { + "epoch": 3.6562200836398446, + "grad_norm": 0.9303803443908691, + "learning_rate": 0.0001, + "loss": 72.8282, + "step": 2270 + }, + { + "epoch": 3.6723434272182196, + "grad_norm": 0.937164843082428, + "learning_rate": 0.0001, + "loss": 70.3305, + "step": 2280 + }, + { + "epoch": 3.6884667707965937, + "grad_norm": 1.027329444885254, + "learning_rate": 0.0001, + "loss": 73.748, + "step": 2290 + }, + { + "epoch": 3.7045901143749687, + "grad_norm": 1.0915215015411377, + "learning_rate": 0.0001, + "loss": 72.2265, + "step": 2300 + }, + { + "epoch": 3.720713457953343, + "grad_norm": 1.1031261682510376, + "learning_rate": 0.0001, + "loss": 71.2795, + "step": 2310 + }, + { + "epoch": 3.7368368015317177, + "grad_norm": 1.1466140747070312, + "learning_rate": 0.0001, + "loss": 72.0116, + "step": 2320 + }, + { + "epoch": 3.7529601451100922, + "grad_norm": 1.0441662073135376, + "learning_rate": 0.0001, + "loss": 73.3735, + "step": 2330 + }, + { + "epoch": 3.7690834886884668, + "grad_norm": 0.8610432744026184, + "learning_rate": 0.0001, + "loss": 73.3433, + "step": 2340 + }, + { + "epoch": 3.7852068322668413, + "grad_norm": 0.885720431804657, + "learning_rate": 0.0001, + "loss": 73.575, + "step": 2350 + }, + { + "epoch": 3.801330175845216, + "grad_norm": 1.0023950338363647, + "learning_rate": 0.0001, + "loss": 73.4166, + "step": 2360 + }, + { + "epoch": 3.8174535194235903, + "grad_norm": 1.3820304870605469, + "learning_rate": 0.0001, + "loss": 72.9599, + "step": 2370 + }, + { + "epoch": 3.833576863001965, + "grad_norm": 0.9645710587501526, + "learning_rate": 0.0001, + "loss": 72.3922, + "step": 2380 + }, + { + "epoch": 3.84970020658034, + "grad_norm": 1.273369312286377, + "learning_rate": 0.0001, + "loss": 72.0158, + "step": 2390 + }, + { + "epoch": 3.865823550158714, + "grad_norm": 0.859863817691803, + "learning_rate": 0.0001, + "loss": 74.6642, + "step": 2400 + }, + { + "epoch": 3.881946893737089, + "grad_norm": 0.9396682381629944, + "learning_rate": 0.0001, + "loss": 71.9003, + "step": 2410 + }, + { + "epoch": 3.8980702373154634, + "grad_norm": 0.9976151585578918, + "learning_rate": 0.0001, + "loss": 73.0716, + "step": 2420 + }, + { + "epoch": 3.914193580893838, + "grad_norm": 1.0730767250061035, + "learning_rate": 0.0001, + "loss": 72.7847, + "step": 2430 + }, + { + "epoch": 3.9303169244722125, + "grad_norm": 1.002018690109253, + "learning_rate": 0.0001, + "loss": 71.7309, + "step": 2440 + }, + { + "epoch": 3.946440268050587, + "grad_norm": 0.9102753400802612, + "learning_rate": 0.0001, + "loss": 71.616, + "step": 2450 + }, + { + "epoch": 3.9625636116289615, + "grad_norm": 0.9827467799186707, + "learning_rate": 0.0001, + "loss": 72.22, + "step": 2460 + }, + { + "epoch": 3.978686955207336, + "grad_norm": 1.2781893014907837, + "learning_rate": 0.0001, + "loss": 73.2048, + "step": 2470 + }, + { + "epoch": 3.9948102987857106, + "grad_norm": 0.9042410850524902, + "learning_rate": 0.0001, + "loss": 72.9843, + "step": 2480 + }, + { + "epoch": 4.009674006147025, + "grad_norm": 0.8662117123603821, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 2490 + }, + { + "epoch": 4.0257973497254, + "grad_norm": 0.9448756575584412, + "learning_rate": 0.0001, + "loss": 73.36, + "step": 2500 + }, + { + "epoch": 4.041920693303774, + "grad_norm": 0.9054566621780396, + "learning_rate": 0.0001, + "loss": 73.94, + "step": 2510 + }, + { + "epoch": 4.058044036882149, + "grad_norm": 0.957398533821106, + "learning_rate": 0.0001, + "loss": 73.2019, + "step": 2520 + }, + { + "epoch": 4.074167380460523, + "grad_norm": 0.9444408416748047, + "learning_rate": 0.0001, + "loss": 70.6553, + "step": 2530 + }, + { + "epoch": 4.090290724038898, + "grad_norm": 0.8716063499450684, + "learning_rate": 0.0001, + "loss": 70.1048, + "step": 2540 + }, + { + "epoch": 4.106414067617272, + "grad_norm": 0.949249267578125, + "learning_rate": 0.0001, + "loss": 72.4453, + "step": 2550 + }, + { + "epoch": 4.122537411195647, + "grad_norm": 0.9843282699584961, + "learning_rate": 0.0001, + "loss": 72.1725, + "step": 2560 + }, + { + "epoch": 4.138660754774021, + "grad_norm": 0.8977480530738831, + "learning_rate": 0.0001, + "loss": 72.5272, + "step": 2570 + }, + { + "epoch": 4.154784098352396, + "grad_norm": 0.9957827925682068, + "learning_rate": 0.0001, + "loss": 72.289, + "step": 2580 + }, + { + "epoch": 4.170907441930771, + "grad_norm": 1.1358791589736938, + "learning_rate": 0.0001, + "loss": 72.1949, + "step": 2590 + }, + { + "epoch": 4.187030785509145, + "grad_norm": 0.9953081607818604, + "learning_rate": 0.0001, + "loss": 72.6004, + "step": 2600 + }, + { + "epoch": 4.20315412908752, + "grad_norm": 1.019061803817749, + "learning_rate": 0.0001, + "loss": 72.3772, + "step": 2610 + }, + { + "epoch": 4.219277472665894, + "grad_norm": 0.9062562584877014, + "learning_rate": 0.0001, + "loss": 72.0566, + "step": 2620 + }, + { + "epoch": 4.235400816244269, + "grad_norm": 1.0877611637115479, + "learning_rate": 0.0001, + "loss": 72.854, + "step": 2630 + }, + { + "epoch": 4.251524159822643, + "grad_norm": 1.2419648170471191, + "learning_rate": 0.0001, + "loss": 72.6498, + "step": 2640 + }, + { + "epoch": 4.267647503401018, + "grad_norm": 0.9507225751876831, + "learning_rate": 0.0001, + "loss": 68.9583, + "step": 2650 + }, + { + "epoch": 4.283770846979392, + "grad_norm": 1.121191382408142, + "learning_rate": 0.0001, + "loss": 72.8464, + "step": 2660 + }, + { + "epoch": 4.299894190557767, + "grad_norm": 1.0543361902236938, + "learning_rate": 0.0001, + "loss": 71.9882, + "step": 2670 + }, + { + "epoch": 4.316017534136141, + "grad_norm": 0.8622955083847046, + "learning_rate": 0.0001, + "loss": 71.7373, + "step": 2680 + }, + { + "epoch": 4.332140877714516, + "grad_norm": 0.911253809928894, + "learning_rate": 0.0001, + "loss": 71.1968, + "step": 2690 + }, + { + "epoch": 4.34826422129289, + "grad_norm": 4.405472755432129, + "learning_rate": 0.0001, + "loss": 70.4128, + "step": 2700 + }, + { + "epoch": 4.364387564871265, + "grad_norm": 0.9142985939979553, + "learning_rate": 0.0001, + "loss": 69.7548, + "step": 2710 + }, + { + "epoch": 4.38051090844964, + "grad_norm": 1.1022522449493408, + "learning_rate": 0.0001, + "loss": 71.8282, + "step": 2720 + }, + { + "epoch": 4.396634252028014, + "grad_norm": 1.018386721611023, + "learning_rate": 0.0001, + "loss": 71.0853, + "step": 2730 + }, + { + "epoch": 4.412757595606389, + "grad_norm": 1.0483211278915405, + "learning_rate": 0.0001, + "loss": 72.1478, + "step": 2740 + }, + { + "epoch": 4.428880939184763, + "grad_norm": 1.026861548423767, + "learning_rate": 0.0001, + "loss": 71.8947, + "step": 2750 + }, + { + "epoch": 4.445004282763138, + "grad_norm": 1.1858552694320679, + "learning_rate": 0.0001, + "loss": 71.7016, + "step": 2760 + }, + { + "epoch": 4.461127626341512, + "grad_norm": 1.1212221384048462, + "learning_rate": 0.0001, + "loss": 71.4454, + "step": 2770 + }, + { + "epoch": 4.477250969919887, + "grad_norm": 0.8062102794647217, + "learning_rate": 0.0001, + "loss": 73.2672, + "step": 2780 + }, + { + "epoch": 4.493374313498261, + "grad_norm": 1.104483962059021, + "learning_rate": 0.0001, + "loss": 70.548, + "step": 2790 + }, + { + "epoch": 4.509497657076636, + "grad_norm": 0.942057728767395, + "learning_rate": 0.0001, + "loss": 72.3584, + "step": 2800 + }, + { + "epoch": 4.525621000655011, + "grad_norm": 0.9028333425521851, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 2810 + }, + { + "epoch": 4.541744344233385, + "grad_norm": 0.9096795916557312, + "learning_rate": 0.0001, + "loss": 70.6596, + "step": 2820 + }, + { + "epoch": 4.55786768781176, + "grad_norm": 0.9038829207420349, + "learning_rate": 0.0001, + "loss": 72.6148, + "step": 2830 + }, + { + "epoch": 4.573991031390134, + "grad_norm": 1.8511613607406616, + "learning_rate": 0.0001, + "loss": 71.7205, + "step": 2840 + }, + { + "epoch": 4.590114374968509, + "grad_norm": 1.3492343425750732, + "learning_rate": 0.0001, + "loss": 72.1492, + "step": 2850 + }, + { + "epoch": 4.6062377185468835, + "grad_norm": 1.1469528675079346, + "learning_rate": 0.0001, + "loss": 70.5892, + "step": 2860 + }, + { + "epoch": 4.622361062125258, + "grad_norm": 1.1374379396438599, + "learning_rate": 0.0001, + "loss": 70.061, + "step": 2870 + }, + { + "epoch": 4.6384844057036325, + "grad_norm": 0.9597565531730652, + "learning_rate": 0.0001, + "loss": 71.0869, + "step": 2880 + }, + { + "epoch": 4.6546077492820075, + "grad_norm": 0.8356521129608154, + "learning_rate": 0.0001, + "loss": 72.1891, + "step": 2890 + }, + { + "epoch": 4.670731092860382, + "grad_norm": 0.8882647752761841, + "learning_rate": 0.0001, + "loss": 71.3845, + "step": 2900 + }, + { + "epoch": 4.6868544364387565, + "grad_norm": 1.1904003620147705, + "learning_rate": 0.0001, + "loss": 72.3086, + "step": 2910 + }, + { + "epoch": 4.702977780017131, + "grad_norm": 0.8010085821151733, + "learning_rate": 0.0001, + "loss": 71.3051, + "step": 2920 + }, + { + "epoch": 4.719101123595506, + "grad_norm": 0.7572135329246521, + "learning_rate": 0.0001, + "loss": 72.5605, + "step": 2930 + }, + { + "epoch": 4.735224467173881, + "grad_norm": 1.0866942405700684, + "learning_rate": 0.0001, + "loss": 70.0935, + "step": 2940 + }, + { + "epoch": 4.751347810752255, + "grad_norm": 0.9358721971511841, + "learning_rate": 0.0001, + "loss": 71.0391, + "step": 2950 + }, + { + "epoch": 4.76747115433063, + "grad_norm": 1.0276161432266235, + "learning_rate": 0.0001, + "loss": 69.8462, + "step": 2960 + }, + { + "epoch": 4.783594497909004, + "grad_norm": 0.9573063850402832, + "learning_rate": 0.0001, + "loss": 69.7089, + "step": 2970 + }, + { + "epoch": 4.799717841487379, + "grad_norm": 1.0638892650604248, + "learning_rate": 0.0001, + "loss": 71.8376, + "step": 2980 + }, + { + "epoch": 4.815841185065753, + "grad_norm": 0.9894681572914124, + "learning_rate": 0.0001, + "loss": 70.1764, + "step": 2990 + }, + { + "epoch": 4.831964528644128, + "grad_norm": 1.1265769004821777, + "learning_rate": 0.0001, + "loss": 71.7918, + "step": 3000 + }, + { + "epoch": 4.848087872222502, + "grad_norm": 2.2792513370513916, + "learning_rate": 0.0001, + "loss": 70.4341, + "step": 3010 + }, + { + "epoch": 4.864211215800877, + "grad_norm": 0.8469329476356506, + "learning_rate": 0.0001, + "loss": 72.1137, + "step": 3020 + }, + { + "epoch": 4.880334559379252, + "grad_norm": 0.9333013296127319, + "learning_rate": 0.0001, + "loss": 71.0789, + "step": 3030 + }, + { + "epoch": 4.896457902957626, + "grad_norm": 0.7888994216918945, + "learning_rate": 0.0001, + "loss": 70.8197, + "step": 3040 + }, + { + "epoch": 4.912581246536, + "grad_norm": 0.9194874167442322, + "learning_rate": 0.0001, + "loss": 72.071, + "step": 3050 + }, + { + "epoch": 4.928704590114375, + "grad_norm": 1.1252421140670776, + "learning_rate": 0.0001, + "loss": 71.6943, + "step": 3060 + }, + { + "epoch": 4.94482793369275, + "grad_norm": 1.1801071166992188, + "learning_rate": 0.0001, + "loss": 70.8937, + "step": 3070 + }, + { + "epoch": 4.960951277271124, + "grad_norm": 0.9162415266036987, + "learning_rate": 0.0001, + "loss": 70.024, + "step": 3080 + }, + { + "epoch": 4.977074620849499, + "grad_norm": 0.8833363652229309, + "learning_rate": 0.0001, + "loss": 71.3835, + "step": 3090 + }, + { + "epoch": 4.993197964427873, + "grad_norm": 0.9985883831977844, + "learning_rate": 0.0001, + "loss": 71.635, + "step": 3100 + }, + { + "epoch": 5.0080616717891875, + "grad_norm": 1.097298264503479, + "learning_rate": 0.0001, + "loss": 65.3542, + "step": 3110 + }, + { + "epoch": 5.024185015367562, + "grad_norm": 0.9703903198242188, + "learning_rate": 0.0001, + "loss": 69.8587, + "step": 3120 + }, + { + "epoch": 5.0403083589459365, + "grad_norm": 0.8894070386886597, + "learning_rate": 0.0001, + "loss": 71.2868, + "step": 3130 + }, + { + "epoch": 5.056431702524311, + "grad_norm": 0.6984127163887024, + "learning_rate": 0.0001, + "loss": 69.0425, + "step": 3140 + }, + { + "epoch": 5.072555046102686, + "grad_norm": 0.8607762455940247, + "learning_rate": 0.0001, + "loss": 69.5354, + "step": 3150 + }, + { + "epoch": 5.08867838968106, + "grad_norm": 1.1215914487838745, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 3160 + }, + { + "epoch": 5.104801733259435, + "grad_norm": 0.9114409685134888, + "learning_rate": 0.0001, + "loss": 68.4186, + "step": 3170 + }, + { + "epoch": 5.12092507683781, + "grad_norm": 1.010400652885437, + "learning_rate": 0.0001, + "loss": 68.3965, + "step": 3180 + }, + { + "epoch": 5.137048420416184, + "grad_norm": 0.8741958141326904, + "learning_rate": 0.0001, + "loss": 70.6728, + "step": 3190 + }, + { + "epoch": 5.153171763994559, + "grad_norm": 0.8760207891464233, + "learning_rate": 0.0001, + "loss": 72.0704, + "step": 3200 + }, + { + "epoch": 5.169295107572933, + "grad_norm": 1.328247308731079, + "learning_rate": 0.0001, + "loss": 69.9082, + "step": 3210 + }, + { + "epoch": 5.185418451151308, + "grad_norm": 0.8263765573501587, + "learning_rate": 0.0001, + "loss": 70.3539, + "step": 3220 + }, + { + "epoch": 5.201541794729682, + "grad_norm": 0.9358834028244019, + "learning_rate": 0.0001, + "loss": 70.4221, + "step": 3230 + }, + { + "epoch": 5.217665138308057, + "grad_norm": 1.0769531726837158, + "learning_rate": 0.0001, + "loss": 68.8167, + "step": 3240 + }, + { + "epoch": 5.233788481886431, + "grad_norm": 1.1438790559768677, + "learning_rate": 0.0001, + "loss": 68.9288, + "step": 3250 + }, + { + "epoch": 5.249911825464806, + "grad_norm": 0.9534218311309814, + "learning_rate": 0.0001, + "loss": 70.3277, + "step": 3260 + }, + { + "epoch": 5.26603516904318, + "grad_norm": 0.88532954454422, + "learning_rate": 0.0001, + "loss": 67.5781, + "step": 3270 + }, + { + "epoch": 5.282158512621555, + "grad_norm": 0.7986133694648743, + "learning_rate": 0.0001, + "loss": 69.759, + "step": 3280 + }, + { + "epoch": 5.29828185619993, + "grad_norm": 0.9100566506385803, + "learning_rate": 0.0001, + "loss": 68.6368, + "step": 3290 + }, + { + "epoch": 5.314405199778304, + "grad_norm": 0.8861745595932007, + "learning_rate": 0.0001, + "loss": 70.5756, + "step": 3300 + }, + { + "epoch": 5.330528543356679, + "grad_norm": 1.0729680061340332, + "learning_rate": 0.0001, + "loss": 70.3178, + "step": 3310 + }, + { + "epoch": 5.346651886935053, + "grad_norm": 1.0288280248641968, + "learning_rate": 0.0001, + "loss": 69.422, + "step": 3320 + }, + { + "epoch": 5.362775230513428, + "grad_norm": 0.8335140347480774, + "learning_rate": 0.0001, + "loss": 69.8281, + "step": 3330 + }, + { + "epoch": 5.378898574091802, + "grad_norm": 0.9789717197418213, + "learning_rate": 0.0001, + "loss": 71.1532, + "step": 3340 + }, + { + "epoch": 5.395021917670177, + "grad_norm": 0.9884677529335022, + "learning_rate": 0.0001, + "loss": 69.5673, + "step": 3350 + }, + { + "epoch": 5.411145261248551, + "grad_norm": 1.050154447555542, + "learning_rate": 0.0001, + "loss": 70.8015, + "step": 3360 + }, + { + "epoch": 5.427268604826926, + "grad_norm": 0.9645342230796814, + "learning_rate": 0.0001, + "loss": 69.1615, + "step": 3370 + }, + { + "epoch": 5.4433919484053, + "grad_norm": 0.8383159041404724, + "learning_rate": 0.0001, + "loss": 70.6299, + "step": 3380 + }, + { + "epoch": 5.459515291983675, + "grad_norm": 0.9250556826591492, + "learning_rate": 0.0001, + "loss": 69.231, + "step": 3390 + }, + { + "epoch": 5.475638635562049, + "grad_norm": 0.8108687400817871, + "learning_rate": 0.0001, + "loss": 70.5858, + "step": 3400 + }, + { + "epoch": 5.491761979140424, + "grad_norm": 0.8748818039894104, + "learning_rate": 0.0001, + "loss": 69.9001, + "step": 3410 + }, + { + "epoch": 5.507885322718799, + "grad_norm": 0.7411381006240845, + "learning_rate": 0.0001, + "loss": 68.3248, + "step": 3420 + }, + { + "epoch": 5.524008666297173, + "grad_norm": 1.0753326416015625, + "learning_rate": 0.0001, + "loss": 69.5831, + "step": 3430 + }, + { + "epoch": 5.540132009875548, + "grad_norm": 0.875254213809967, + "learning_rate": 0.0001, + "loss": 69.7516, + "step": 3440 + }, + { + "epoch": 5.556255353453922, + "grad_norm": 0.7637733817100525, + "learning_rate": 0.0001, + "loss": 68.0078, + "step": 3450 + }, + { + "epoch": 5.572378697032297, + "grad_norm": 0.968203604221344, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 3460 + }, + { + "epoch": 5.588502040610671, + "grad_norm": 0.8903141617774963, + "learning_rate": 0.0001, + "loss": 68.5835, + "step": 3470 + }, + { + "epoch": 5.604625384189046, + "grad_norm": 0.9536195397377014, + "learning_rate": 0.0001, + "loss": 71.0259, + "step": 3480 + }, + { + "epoch": 5.62074872776742, + "grad_norm": 0.8951714038848877, + "learning_rate": 0.0001, + "loss": 69.4422, + "step": 3490 + }, + { + "epoch": 5.636872071345795, + "grad_norm": 0.9639522433280945, + "learning_rate": 0.0001, + "loss": 66.5173, + "step": 3500 + }, + { + "epoch": 5.65299541492417, + "grad_norm": 1.0022730827331543, + "learning_rate": 0.0001, + "loss": 68.3443, + "step": 3510 + }, + { + "epoch": 5.669118758502544, + "grad_norm": 1.0654487609863281, + "learning_rate": 0.0001, + "loss": 69.0835, + "step": 3520 + }, + { + "epoch": 5.685242102080919, + "grad_norm": 0.9934629797935486, + "learning_rate": 0.0001, + "loss": 69.2278, + "step": 3530 + }, + { + "epoch": 5.7013654456592935, + "grad_norm": 0.8773566484451294, + "learning_rate": 0.0001, + "loss": 68.9339, + "step": 3540 + }, + { + "epoch": 5.7174887892376685, + "grad_norm": 0.7756998538970947, + "learning_rate": 0.0001, + "loss": 69.2683, + "step": 3550 + }, + { + "epoch": 5.7336121328160425, + "grad_norm": 0.8819359540939331, + "learning_rate": 0.0001, + "loss": 68.5071, + "step": 3560 + }, + { + "epoch": 5.7497354763944175, + "grad_norm": 1.1069825887680054, + "learning_rate": 0.0001, + "loss": 68.5236, + "step": 3570 + }, + { + "epoch": 5.765858819972792, + "grad_norm": 0.8006312847137451, + "learning_rate": 0.0001, + "loss": 68.7192, + "step": 3580 + }, + { + "epoch": 5.781982163551167, + "grad_norm": 1.0278719663619995, + "learning_rate": 0.0001, + "loss": 69.2545, + "step": 3590 + }, + { + "epoch": 5.798105507129541, + "grad_norm": 0.9205041527748108, + "learning_rate": 0.0001, + "loss": 69.7698, + "step": 3600 + }, + { + "epoch": 5.814228850707916, + "grad_norm": 1.0769038200378418, + "learning_rate": 0.0001, + "loss": 67.6568, + "step": 3610 + }, + { + "epoch": 5.83035219428629, + "grad_norm": 1.0565663576126099, + "learning_rate": 0.0001, + "loss": 68.9166, + "step": 3620 + }, + { + "epoch": 5.846475537864665, + "grad_norm": 0.9008663296699524, + "learning_rate": 0.0001, + "loss": 69.5837, + "step": 3630 + }, + { + "epoch": 5.86259888144304, + "grad_norm": 0.8369411826133728, + "learning_rate": 0.0001, + "loss": 68.5661, + "step": 3640 + }, + { + "epoch": 5.878722225021414, + "grad_norm": 1.0472379922866821, + "learning_rate": 0.0001, + "loss": 68.3122, + "step": 3650 + }, + { + "epoch": 5.894845568599789, + "grad_norm": 0.965277373790741, + "learning_rate": 0.0001, + "loss": 69.8062, + "step": 3660 + }, + { + "epoch": 5.910968912178163, + "grad_norm": 0.8936495780944824, + "learning_rate": 0.0001, + "loss": 67.1016, + "step": 3670 + }, + { + "epoch": 5.927092255756538, + "grad_norm": 0.8559162020683289, + "learning_rate": 0.0001, + "loss": 69.9986, + "step": 3680 + }, + { + "epoch": 5.943215599334912, + "grad_norm": 1.4815773963928223, + "learning_rate": 0.0001, + "loss": 66.7648, + "step": 3690 + }, + { + "epoch": 5.959338942913287, + "grad_norm": 0.9034469723701477, + "learning_rate": 0.0001, + "loss": 68.0767, + "step": 3700 + }, + { + "epoch": 5.975462286491661, + "grad_norm": 1.838799238204956, + "learning_rate": 0.0001, + "loss": 66.5848, + "step": 3710 + }, + { + "epoch": 5.991585630070036, + "grad_norm": 0.9274787902832031, + "learning_rate": 0.0001, + "loss": 70.0396, + "step": 3720 + }, + { + "epoch": 6.0064493374313495, + "grad_norm": 1.016974687576294, + "learning_rate": 0.0001, + "loss": 62.2268, + "step": 3730 + }, + { + "epoch": 6.022572681009724, + "grad_norm": 1.0522642135620117, + "learning_rate": 0.0001, + "loss": 69.3381, + "step": 3740 + }, + { + "epoch": 6.0386960245880985, + "grad_norm": 1.1157186031341553, + "learning_rate": 0.0001, + "loss": 68.8969, + "step": 3750 + }, + { + "epoch": 6.0548193681664735, + "grad_norm": 0.9913288950920105, + "learning_rate": 0.0001, + "loss": 67.3634, + "step": 3760 + }, + { + "epoch": 6.0709427117448485, + "grad_norm": 0.8823490142822266, + "learning_rate": 0.0001, + "loss": 70.978, + "step": 3770 + }, + { + "epoch": 6.0870660553232225, + "grad_norm": 0.853837788105011, + "learning_rate": 0.0001, + "loss": 68.9924, + "step": 3780 + }, + { + "epoch": 6.1031893989015975, + "grad_norm": 0.7937766909599304, + "learning_rate": 0.0001, + "loss": 68.1994, + "step": 3790 + }, + { + "epoch": 6.119312742479972, + "grad_norm": 1.7638057470321655, + "learning_rate": 0.0001, + "loss": 68.5078, + "step": 3800 + }, + { + "epoch": 6.135436086058347, + "grad_norm": 1.0542887449264526, + "learning_rate": 0.0001, + "loss": 68.9362, + "step": 3810 + }, + { + "epoch": 6.151559429636721, + "grad_norm": 1.06019926071167, + "learning_rate": 0.0001, + "loss": 67.2493, + "step": 3820 + }, + { + "epoch": 6.167682773215096, + "grad_norm": 0.9765111804008484, + "learning_rate": 0.0001, + "loss": 70.4797, + "step": 3830 + }, + { + "epoch": 6.18380611679347, + "grad_norm": 1.006372332572937, + "learning_rate": 0.0001, + "loss": 69.4233, + "step": 3840 + }, + { + "epoch": 6.199929460371845, + "grad_norm": 0.8274241089820862, + "learning_rate": 0.0001, + "loss": 67.9907, + "step": 3850 + }, + { + "epoch": 6.216052803950219, + "grad_norm": 0.9905765056610107, + "learning_rate": 0.0001, + "loss": 69.6988, + "step": 3860 + }, + { + "epoch": 6.232176147528594, + "grad_norm": 1.0065921545028687, + "learning_rate": 0.0001, + "loss": 68.7678, + "step": 3870 + }, + { + "epoch": 6.248299491106969, + "grad_norm": 0.8845255970954895, + "learning_rate": 0.0001, + "loss": 68.3039, + "step": 3880 + }, + { + "epoch": 6.264422834685343, + "grad_norm": 0.9030968546867371, + "learning_rate": 0.0001, + "loss": 68.671, + "step": 3890 + }, + { + "epoch": 6.280546178263718, + "grad_norm": 0.8650258183479309, + "learning_rate": 0.0001, + "loss": 68.5309, + "step": 3900 + }, + { + "epoch": 6.296669521842092, + "grad_norm": 0.8961272835731506, + "learning_rate": 0.0001, + "loss": 67.5611, + "step": 3910 + }, + { + "epoch": 6.312792865420467, + "grad_norm": 1.1702407598495483, + "learning_rate": 0.0001, + "loss": 68.2768, + "step": 3920 + }, + { + "epoch": 6.328916208998841, + "grad_norm": 0.8598962426185608, + "learning_rate": 0.0001, + "loss": 69.1961, + "step": 3930 + }, + { + "epoch": 6.345039552577216, + "grad_norm": 0.8714122176170349, + "learning_rate": 0.0001, + "loss": 66.8628, + "step": 3940 + }, + { + "epoch": 6.36116289615559, + "grad_norm": 0.7757895588874817, + "learning_rate": 0.0001, + "loss": 68.6908, + "step": 3950 + }, + { + "epoch": 6.377286239733965, + "grad_norm": 0.9189490675926208, + "learning_rate": 0.0001, + "loss": 67.8685, + "step": 3960 + }, + { + "epoch": 6.393409583312339, + "grad_norm": 0.8161299228668213, + "learning_rate": 0.0001, + "loss": 66.816, + "step": 3970 + }, + { + "epoch": 6.409532926890714, + "grad_norm": 0.8559791445732117, + "learning_rate": 0.0001, + "loss": 68.4302, + "step": 3980 + }, + { + "epoch": 6.425656270469089, + "grad_norm": 0.8087984919548035, + "learning_rate": 0.0001, + "loss": 66.2646, + "step": 3990 + }, + { + "epoch": 6.441779614047463, + "grad_norm": 0.7733110785484314, + "learning_rate": 0.0001, + "loss": 69.2769, + "step": 4000 + }, + { + "epoch": 6.457902957625838, + "grad_norm": 0.9335984587669373, + "learning_rate": 0.0001, + "loss": 67.1199, + "step": 4010 + }, + { + "epoch": 6.474026301204212, + "grad_norm": 0.9546418786048889, + "learning_rate": 0.0001, + "loss": 68.8706, + "step": 4020 + }, + { + "epoch": 6.490149644782587, + "grad_norm": 0.9670593738555908, + "learning_rate": 0.0001, + "loss": 68.2329, + "step": 4030 + }, + { + "epoch": 6.506272988360961, + "grad_norm": 0.8428588509559631, + "learning_rate": 0.0001, + "loss": 68.0105, + "step": 4040 + }, + { + "epoch": 6.522396331939336, + "grad_norm": 0.9755622744560242, + "learning_rate": 0.0001, + "loss": 68.656, + "step": 4050 + }, + { + "epoch": 6.53851967551771, + "grad_norm": 0.8972346186637878, + "learning_rate": 0.0001, + "loss": 69.3665, + "step": 4060 + }, + { + "epoch": 6.554643019096085, + "grad_norm": 0.9716668128967285, + "learning_rate": 0.0001, + "loss": 68.1053, + "step": 4070 + }, + { + "epoch": 6.570766362674459, + "grad_norm": 0.8147910833358765, + "learning_rate": 0.0001, + "loss": 67.9778, + "step": 4080 + }, + { + "epoch": 6.586889706252834, + "grad_norm": 0.8304234147071838, + "learning_rate": 0.0001, + "loss": 67.7069, + "step": 4090 + }, + { + "epoch": 6.603013049831208, + "grad_norm": 0.9356856346130371, + "learning_rate": 0.0001, + "loss": 68.1942, + "step": 4100 + }, + { + "epoch": 6.619136393409583, + "grad_norm": 1.017134189605713, + "learning_rate": 0.0001, + "loss": 67.4211, + "step": 4110 + }, + { + "epoch": 6.635259736987958, + "grad_norm": 1.142558217048645, + "learning_rate": 0.0001, + "loss": 68.4413, + "step": 4120 + }, + { + "epoch": 6.651383080566332, + "grad_norm": 0.9071153998374939, + "learning_rate": 0.0001, + "loss": 69.3666, + "step": 4130 + }, + { + "epoch": 6.667506424144707, + "grad_norm": 0.9620397686958313, + "learning_rate": 0.0001, + "loss": 68.208, + "step": 4140 + }, + { + "epoch": 6.683629767723081, + "grad_norm": 0.8282128572463989, + "learning_rate": 0.0001, + "loss": 67.1702, + "step": 4150 + }, + { + "epoch": 6.699753111301456, + "grad_norm": 0.8838199973106384, + "learning_rate": 0.0001, + "loss": 69.0074, + "step": 4160 + }, + { + "epoch": 6.71587645487983, + "grad_norm": 0.8864356279373169, + "learning_rate": 0.0001, + "loss": 67.8052, + "step": 4170 + }, + { + "epoch": 6.731999798458205, + "grad_norm": 0.9373385906219482, + "learning_rate": 0.0001, + "loss": 67.446, + "step": 4180 + }, + { + "epoch": 6.7481231420365795, + "grad_norm": 0.8308956027030945, + "learning_rate": 0.0001, + "loss": 68.716, + "step": 4190 + }, + { + "epoch": 6.7642464856149545, + "grad_norm": 0.8031372427940369, + "learning_rate": 0.0001, + "loss": 68.0482, + "step": 4200 + }, + { + "epoch": 6.780369829193329, + "grad_norm": 0.9258586764335632, + "learning_rate": 0.0001, + "loss": 68.5607, + "step": 4210 + }, + { + "epoch": 6.7964931727717035, + "grad_norm": 0.8147327899932861, + "learning_rate": 0.0001, + "loss": 67.7265, + "step": 4220 + }, + { + "epoch": 6.8126165163500785, + "grad_norm": 0.8717535734176636, + "learning_rate": 0.0001, + "loss": 67.8097, + "step": 4230 + }, + { + "epoch": 6.828739859928453, + "grad_norm": 0.9876209497451782, + "learning_rate": 0.0001, + "loss": 67.9068, + "step": 4240 + }, + { + "epoch": 6.8448632035068275, + "grad_norm": 0.718376874923706, + "learning_rate": 0.0001, + "loss": 68.1762, + "step": 4250 + }, + { + "epoch": 6.860986547085202, + "grad_norm": 0.8264320492744446, + "learning_rate": 0.0001, + "loss": 67.8391, + "step": 4260 + }, + { + "epoch": 6.877109890663577, + "grad_norm": 0.7478179335594177, + "learning_rate": 0.0001, + "loss": 67.6635, + "step": 4270 + }, + { + "epoch": 6.893233234241951, + "grad_norm": 0.8394957184791565, + "learning_rate": 0.0001, + "loss": 68.6073, + "step": 4280 + }, + { + "epoch": 6.909356577820326, + "grad_norm": 0.8534071445465088, + "learning_rate": 0.0001, + "loss": 69.6206, + "step": 4290 + }, + { + "epoch": 6.9254799213987, + "grad_norm": 0.9137582778930664, + "learning_rate": 0.0001, + "loss": 67.4362, + "step": 4300 + }, + { + "epoch": 6.941603264977075, + "grad_norm": 0.8869971036911011, + "learning_rate": 0.0001, + "loss": 67.6811, + "step": 4310 + }, + { + "epoch": 6.957726608555449, + "grad_norm": 0.889025866985321, + "learning_rate": 0.0001, + "loss": 65.3354, + "step": 4320 + }, + { + "epoch": 6.973849952133824, + "grad_norm": 0.7968549728393555, + "learning_rate": 0.0001, + "loss": 66.192, + "step": 4330 + }, + { + "epoch": 6.989973295712199, + "grad_norm": 1.0077433586120605, + "learning_rate": 0.0001, + "loss": 67.5216, + "step": 4340 + }, + { + "epoch": 7.004837003073512, + "grad_norm": 0.7900649309158325, + "learning_rate": 0.0001, + "loss": 62.2421, + "step": 4350 + }, + { + "epoch": 7.020960346651887, + "grad_norm": 1.0160410404205322, + "learning_rate": 0.0001, + "loss": 68.0978, + "step": 4360 + }, + { + "epoch": 7.037083690230261, + "grad_norm": 0.841387927532196, + "learning_rate": 0.0001, + "loss": 68.1008, + "step": 4370 + }, + { + "epoch": 7.053207033808636, + "grad_norm": 0.87884122133255, + "learning_rate": 0.0001, + "loss": 66.4667, + "step": 4380 + }, + { + "epoch": 7.06933037738701, + "grad_norm": 0.8580564856529236, + "learning_rate": 0.0001, + "loss": 67.2002, + "step": 4390 + }, + { + "epoch": 7.085453720965385, + "grad_norm": 0.7560063600540161, + "learning_rate": 0.0001, + "loss": 67.6743, + "step": 4400 + }, + { + "epoch": 7.1015770645437595, + "grad_norm": 0.855478048324585, + "learning_rate": 0.0001, + "loss": 67.822, + "step": 4410 + }, + { + "epoch": 7.1177004081221344, + "grad_norm": 0.7994911670684814, + "learning_rate": 0.0001, + "loss": 66.0094, + "step": 4420 + }, + { + "epoch": 7.1338237517005085, + "grad_norm": 0.8728616833686829, + "learning_rate": 0.0001, + "loss": 66.405, + "step": 4430 + }, + { + "epoch": 7.1499470952788835, + "grad_norm": 0.8434081673622131, + "learning_rate": 0.0001, + "loss": 67.0535, + "step": 4440 + }, + { + "epoch": 7.166070438857258, + "grad_norm": 0.8543094396591187, + "learning_rate": 0.0001, + "loss": 66.2417, + "step": 4450 + }, + { + "epoch": 7.182193782435633, + "grad_norm": 0.7209319472312927, + "learning_rate": 0.0001, + "loss": 67.3958, + "step": 4460 + }, + { + "epoch": 7.1983171260140075, + "grad_norm": 0.6735154390335083, + "learning_rate": 0.0001, + "loss": 67.4086, + "step": 4470 + }, + { + "epoch": 7.214440469592382, + "grad_norm": 0.9126104116439819, + "learning_rate": 0.0001, + "loss": 68.4191, + "step": 4480 + }, + { + "epoch": 7.230563813170757, + "grad_norm": 0.9334300756454468, + "learning_rate": 0.0001, + "loss": 68.3247, + "step": 4490 + }, + { + "epoch": 7.246687156749131, + "grad_norm": 1.5956038236618042, + "learning_rate": 0.0001, + "loss": 68.2779, + "step": 4500 + }, + { + "epoch": 7.262810500327506, + "grad_norm": 1.5012775659561157, + "learning_rate": 0.0001, + "loss": 68.3533, + "step": 4510 + }, + { + "epoch": 7.27893384390588, + "grad_norm": 1.1376632452011108, + "learning_rate": 0.0001, + "loss": 65.3627, + "step": 4520 + }, + { + "epoch": 7.295057187484255, + "grad_norm": 0.8597291111946106, + "learning_rate": 0.0001, + "loss": 67.8653, + "step": 4530 + }, + { + "epoch": 7.311180531062629, + "grad_norm": 0.7645048499107361, + "learning_rate": 0.0001, + "loss": 65.4592, + "step": 4540 + }, + { + "epoch": 7.327303874641004, + "grad_norm": 0.7717318534851074, + "learning_rate": 0.0001, + "loss": 66.902, + "step": 4550 + }, + { + "epoch": 7.343427218219378, + "grad_norm": 0.8884950876235962, + "learning_rate": 0.0001, + "loss": 67.1858, + "step": 4560 + }, + { + "epoch": 7.359550561797753, + "grad_norm": 0.7826898694038391, + "learning_rate": 0.0001, + "loss": 68.4673, + "step": 4570 + }, + { + "epoch": 7.375673905376128, + "grad_norm": 0.7484182119369507, + "learning_rate": 0.0001, + "loss": 67.4859, + "step": 4580 + }, + { + "epoch": 7.391797248954502, + "grad_norm": 0.918194591999054, + "learning_rate": 0.0001, + "loss": 67.7393, + "step": 4590 + }, + { + "epoch": 7.407920592532877, + "grad_norm": 1.0696896314620972, + "learning_rate": 0.0001, + "loss": 67.0101, + "step": 4600 + }, + { + "epoch": 7.424043936111251, + "grad_norm": 0.8190692067146301, + "learning_rate": 0.0001, + "loss": 68.5358, + "step": 4610 + }, + { + "epoch": 7.440167279689626, + "grad_norm": 0.9063783884048462, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 4620 + }, + { + "epoch": 7.456290623268, + "grad_norm": 0.7588100433349609, + "learning_rate": 0.0001, + "loss": 68.0066, + "step": 4630 + }, + { + "epoch": 7.472413966846375, + "grad_norm": 0.897281289100647, + "learning_rate": 0.0001, + "loss": 67.2499, + "step": 4640 + }, + { + "epoch": 7.488537310424749, + "grad_norm": 0.9505569338798523, + "learning_rate": 0.0001, + "loss": 67.5452, + "step": 4650 + }, + { + "epoch": 7.504660654003124, + "grad_norm": 0.8409159183502197, + "learning_rate": 0.0001, + "loss": 66.6293, + "step": 4660 + }, + { + "epoch": 7.520783997581498, + "grad_norm": 0.9878574013710022, + "learning_rate": 0.0001, + "loss": 65.3379, + "step": 4670 + }, + { + "epoch": 7.536907341159873, + "grad_norm": 1.0410685539245605, + "learning_rate": 0.0001, + "loss": 65.8028, + "step": 4680 + }, + { + "epoch": 7.553030684738248, + "grad_norm": 0.8417725563049316, + "learning_rate": 0.0001, + "loss": 65.4283, + "step": 4690 + }, + { + "epoch": 7.569154028316622, + "grad_norm": 0.7660863399505615, + "learning_rate": 0.0001, + "loss": 66.6765, + "step": 4700 + }, + { + "epoch": 7.585277371894997, + "grad_norm": 0.8074233531951904, + "learning_rate": 0.0001, + "loss": 67.1082, + "step": 4710 + }, + { + "epoch": 7.601400715473371, + "grad_norm": 0.7653326988220215, + "learning_rate": 0.0001, + "loss": 66.028, + "step": 4720 + }, + { + "epoch": 7.617524059051746, + "grad_norm": 1.1764041185379028, + "learning_rate": 0.0001, + "loss": 65.2587, + "step": 4730 + }, + { + "epoch": 7.63364740263012, + "grad_norm": 1.7604950666427612, + "learning_rate": 0.0001, + "loss": 66.0543, + "step": 4740 + }, + { + "epoch": 7.649770746208495, + "grad_norm": 0.941438615322113, + "learning_rate": 0.0001, + "loss": 65.6954, + "step": 4750 + }, + { + "epoch": 7.665894089786869, + "grad_norm": 0.740069568157196, + "learning_rate": 0.0001, + "loss": 67.9701, + "step": 4760 + }, + { + "epoch": 7.682017433365244, + "grad_norm": 0.9867332577705383, + "learning_rate": 0.0001, + "loss": 66.1445, + "step": 4770 + }, + { + "epoch": 7.698140776943618, + "grad_norm": 0.8156166672706604, + "learning_rate": 0.0001, + "loss": 67.2647, + "step": 4780 + }, + { + "epoch": 7.714264120521993, + "grad_norm": 0.7689579129219055, + "learning_rate": 0.0001, + "loss": 67.0961, + "step": 4790 + }, + { + "epoch": 7.730387464100367, + "grad_norm": 0.8159700036048889, + "learning_rate": 0.0001, + "loss": 66.4974, + "step": 4800 + }, + { + "epoch": 7.746510807678742, + "grad_norm": 0.8554222583770752, + "learning_rate": 0.0001, + "loss": 64.5057, + "step": 4810 + }, + { + "epoch": 7.762634151257117, + "grad_norm": 0.856629490852356, + "learning_rate": 0.0001, + "loss": 67.7819, + "step": 4820 + }, + { + "epoch": 7.778757494835491, + "grad_norm": 1.157179832458496, + "learning_rate": 0.0001, + "loss": 66.0122, + "step": 4830 + }, + { + "epoch": 7.794880838413866, + "grad_norm": 0.9597711563110352, + "learning_rate": 0.0001, + "loss": 66.8248, + "step": 4840 + }, + { + "epoch": 7.8110041819922404, + "grad_norm": 0.9175744652748108, + "learning_rate": 0.0001, + "loss": 65.7998, + "step": 4850 + }, + { + "epoch": 7.827127525570615, + "grad_norm": 0.8467298150062561, + "learning_rate": 0.0001, + "loss": 67.5363, + "step": 4860 + }, + { + "epoch": 7.8432508691489895, + "grad_norm": 0.863922655582428, + "learning_rate": 0.0001, + "loss": 67.7219, + "step": 4870 + }, + { + "epoch": 7.8593742127273645, + "grad_norm": 0.9896290302276611, + "learning_rate": 0.0001, + "loss": 66.7098, + "step": 4880 + }, + { + "epoch": 7.8754975563057386, + "grad_norm": 0.8828520774841309, + "learning_rate": 0.0001, + "loss": 66.0192, + "step": 4890 + }, + { + "epoch": 7.8916208998841135, + "grad_norm": 0.9093656539916992, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 4900 + }, + { + "epoch": 7.9077442434624885, + "grad_norm": 0.8665091395378113, + "learning_rate": 0.0001, + "loss": 66.5851, + "step": 4910 + }, + { + "epoch": 7.923867587040863, + "grad_norm": 0.8027398586273193, + "learning_rate": 0.0001, + "loss": 64.8607, + "step": 4920 + }, + { + "epoch": 7.9399909306192376, + "grad_norm": 0.7808436751365662, + "learning_rate": 0.0001, + "loss": 65.052, + "step": 4930 + }, + { + "epoch": 7.956114274197612, + "grad_norm": 0.7246070504188538, + "learning_rate": 0.0001, + "loss": 64.789, + "step": 4940 + }, + { + "epoch": 7.972237617775987, + "grad_norm": 1.0405027866363525, + "learning_rate": 0.0001, + "loss": 66.7735, + "step": 4950 + }, + { + "epoch": 7.988360961354361, + "grad_norm": 0.812921404838562, + "learning_rate": 0.0001, + "loss": 64.1286, + "step": 4960 + }, + { + "epoch": 8.003224668715674, + "grad_norm": 0.884215772151947, + "learning_rate": 0.0001, + "loss": 61.0812, + "step": 4970 + }, + { + "epoch": 8.01934801229405, + "grad_norm": 0.8519641757011414, + "learning_rate": 0.0001, + "loss": 66.2784, + "step": 4980 + }, + { + "epoch": 8.035471355872424, + "grad_norm": 0.8331199884414673, + "learning_rate": 0.0001, + "loss": 67.1405, + "step": 4990 + }, + { + "epoch": 8.0515946994508, + "grad_norm": 0.8568746447563171, + "learning_rate": 0.0001, + "loss": 66.7835, + "step": 5000 + }, + { + "epoch": 8.067718043029172, + "grad_norm": 1.170482873916626, + "learning_rate": 0.0001, + "loss": 64.2269, + "step": 5010 + }, + { + "epoch": 8.083841386607547, + "grad_norm": 0.9171113967895508, + "learning_rate": 0.0001, + "loss": 66.5567, + "step": 5020 + }, + { + "epoch": 8.099964730185922, + "grad_norm": 0.8988714814186096, + "learning_rate": 0.0001, + "loss": 66.0471, + "step": 5030 + }, + { + "epoch": 8.116088073764297, + "grad_norm": 0.7651671171188354, + "learning_rate": 0.0001, + "loss": 65.7185, + "step": 5040 + }, + { + "epoch": 8.132211417342672, + "grad_norm": 0.6890843510627747, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 5050 + }, + { + "epoch": 8.148334760921045, + "grad_norm": 0.7683818340301514, + "learning_rate": 0.0001, + "loss": 65.895, + "step": 5060 + }, + { + "epoch": 8.16445810449942, + "grad_norm": 0.9398573637008667, + "learning_rate": 0.0001, + "loss": 64.674, + "step": 5070 + }, + { + "epoch": 8.180581448077795, + "grad_norm": 0.7708531022071838, + "learning_rate": 0.0001, + "loss": 65.0637, + "step": 5080 + }, + { + "epoch": 8.19670479165617, + "grad_norm": 0.8840665221214294, + "learning_rate": 0.0001, + "loss": 65.6449, + "step": 5090 + }, + { + "epoch": 8.212828135234544, + "grad_norm": 0.8390524983406067, + "learning_rate": 0.0001, + "loss": 65.4738, + "step": 5100 + }, + { + "epoch": 8.228951478812919, + "grad_norm": 1.7559128999710083, + "learning_rate": 0.0001, + "loss": 68.911, + "step": 5110 + }, + { + "epoch": 8.245074822391294, + "grad_norm": 0.8293817043304443, + "learning_rate": 0.0001, + "loss": 65.2601, + "step": 5120 + }, + { + "epoch": 8.261198165969668, + "grad_norm": 0.8947123289108276, + "learning_rate": 0.0001, + "loss": 65.8611, + "step": 5130 + }, + { + "epoch": 8.277321509548042, + "grad_norm": 0.8526618480682373, + "learning_rate": 0.0001, + "loss": 65.559, + "step": 5140 + }, + { + "epoch": 8.293444853126417, + "grad_norm": 1.7423408031463623, + "learning_rate": 0.0001, + "loss": 64.5007, + "step": 5150 + }, + { + "epoch": 8.309568196704792, + "grad_norm": 1.0512502193450928, + "learning_rate": 0.0001, + "loss": 66.2593, + "step": 5160 + }, + { + "epoch": 8.325691540283167, + "grad_norm": 0.8129869103431702, + "learning_rate": 0.0001, + "loss": 64.7989, + "step": 5170 + }, + { + "epoch": 8.341814883861542, + "grad_norm": 0.852199912071228, + "learning_rate": 0.0001, + "loss": 65.9627, + "step": 5180 + }, + { + "epoch": 8.357938227439915, + "grad_norm": 0.8932474851608276, + "learning_rate": 0.0001, + "loss": 64.1035, + "step": 5190 + }, + { + "epoch": 8.37406157101829, + "grad_norm": 1.2631597518920898, + "learning_rate": 0.0001, + "loss": 65.3406, + "step": 5200 + }, + { + "epoch": 8.390184914596665, + "grad_norm": 0.9062619209289551, + "learning_rate": 0.0001, + "loss": 65.1841, + "step": 5210 + }, + { + "epoch": 8.40630825817504, + "grad_norm": 0.9153940081596375, + "learning_rate": 0.0001, + "loss": 66.3852, + "step": 5220 + }, + { + "epoch": 8.422431601753413, + "grad_norm": 0.8056164979934692, + "learning_rate": 0.0001, + "loss": 66.511, + "step": 5230 + }, + { + "epoch": 8.438554945331788, + "grad_norm": 0.7506036162376404, + "learning_rate": 0.0001, + "loss": 65.275, + "step": 5240 + }, + { + "epoch": 8.454678288910163, + "grad_norm": 0.7994081377983093, + "learning_rate": 0.0001, + "loss": 64.9946, + "step": 5250 + }, + { + "epoch": 8.470801632488538, + "grad_norm": 0.9887873530387878, + "learning_rate": 0.0001, + "loss": 67.2057, + "step": 5260 + }, + { + "epoch": 8.486924976066913, + "grad_norm": 0.8495044112205505, + "learning_rate": 0.0001, + "loss": 66.2481, + "step": 5270 + }, + { + "epoch": 8.503048319645286, + "grad_norm": 0.7353820204734802, + "learning_rate": 0.0001, + "loss": 65.6163, + "step": 5280 + }, + { + "epoch": 8.519171663223661, + "grad_norm": 0.795216977596283, + "learning_rate": 0.0001, + "loss": 64.808, + "step": 5290 + }, + { + "epoch": 8.535295006802036, + "grad_norm": 0.8121851682662964, + "learning_rate": 0.0001, + "loss": 65.9755, + "step": 5300 + }, + { + "epoch": 8.55141835038041, + "grad_norm": 0.8164243698120117, + "learning_rate": 0.0001, + "loss": 66.0583, + "step": 5310 + }, + { + "epoch": 8.567541693958784, + "grad_norm": 0.9893137216567993, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 5320 + }, + { + "epoch": 8.583665037537159, + "grad_norm": 1.3616434335708618, + "learning_rate": 0.0001, + "loss": 64.1444, + "step": 5330 + }, + { + "epoch": 8.599788381115534, + "grad_norm": 0.7002740502357483, + "learning_rate": 0.0001, + "loss": 65.0906, + "step": 5340 + }, + { + "epoch": 8.615911724693909, + "grad_norm": 0.838424563407898, + "learning_rate": 0.0001, + "loss": 65.4578, + "step": 5350 + }, + { + "epoch": 8.632035068272282, + "grad_norm": 0.988902747631073, + "learning_rate": 0.0001, + "loss": 64.3405, + "step": 5360 + }, + { + "epoch": 8.648158411850657, + "grad_norm": 0.7941252589225769, + "learning_rate": 0.0001, + "loss": 67.0146, + "step": 5370 + }, + { + "epoch": 8.664281755429032, + "grad_norm": 0.7727184295654297, + "learning_rate": 0.0001, + "loss": 64.8011, + "step": 5380 + }, + { + "epoch": 8.680405099007407, + "grad_norm": 0.9514367580413818, + "learning_rate": 0.0001, + "loss": 65.8165, + "step": 5390 + }, + { + "epoch": 8.69652844258578, + "grad_norm": 0.870544970035553, + "learning_rate": 0.0001, + "loss": 65.803, + "step": 5400 + }, + { + "epoch": 8.712651786164155, + "grad_norm": 0.7348082065582275, + "learning_rate": 0.0001, + "loss": 65.8409, + "step": 5410 + }, + { + "epoch": 8.72877512974253, + "grad_norm": 0.7543814778327942, + "learning_rate": 0.0001, + "loss": 65.5958, + "step": 5420 + }, + { + "epoch": 8.744898473320905, + "grad_norm": 0.7520368099212646, + "learning_rate": 0.0001, + "loss": 65.3323, + "step": 5430 + }, + { + "epoch": 8.76102181689928, + "grad_norm": 0.7610031366348267, + "learning_rate": 0.0001, + "loss": 65.4386, + "step": 5440 + }, + { + "epoch": 8.777145160477653, + "grad_norm": 0.8854930400848389, + "learning_rate": 0.0001, + "loss": 66.181, + "step": 5450 + }, + { + "epoch": 8.793268504056028, + "grad_norm": 0.7659342885017395, + "learning_rate": 0.0001, + "loss": 65.2586, + "step": 5460 + }, + { + "epoch": 8.809391847634403, + "grad_norm": 0.8592572212219238, + "learning_rate": 0.0001, + "loss": 66.3306, + "step": 5470 + }, + { + "epoch": 8.825515191212778, + "grad_norm": 0.8169172406196594, + "learning_rate": 0.0001, + "loss": 65.8283, + "step": 5480 + }, + { + "epoch": 8.841638534791151, + "grad_norm": 0.7380614876747131, + "learning_rate": 0.0001, + "loss": 65.8711, + "step": 5490 + }, + { + "epoch": 8.857761878369526, + "grad_norm": 0.9993331432342529, + "learning_rate": 0.0001, + "loss": 64.9378, + "step": 5500 + }, + { + "epoch": 8.873885221947901, + "grad_norm": 0.6252862811088562, + "learning_rate": 0.0001, + "loss": 64.1655, + "step": 5510 + }, + { + "epoch": 8.890008565526276, + "grad_norm": 0.8441852331161499, + "learning_rate": 0.0001, + "loss": 64.9795, + "step": 5520 + }, + { + "epoch": 8.906131909104651, + "grad_norm": 0.7075499892234802, + "learning_rate": 0.0001, + "loss": 65.5843, + "step": 5530 + }, + { + "epoch": 8.922255252683025, + "grad_norm": 0.7374144196510315, + "learning_rate": 0.0001, + "loss": 67.5064, + "step": 5540 + }, + { + "epoch": 8.9383785962614, + "grad_norm": 0.8073632121086121, + "learning_rate": 0.0001, + "loss": 67.0182, + "step": 5550 + }, + { + "epoch": 8.954501939839774, + "grad_norm": 0.7576195001602173, + "learning_rate": 0.0001, + "loss": 65.9139, + "step": 5560 + }, + { + "epoch": 8.97062528341815, + "grad_norm": 1.6479626893997192, + "learning_rate": 0.0001, + "loss": 66.3517, + "step": 5570 + }, + { + "epoch": 8.986748626996523, + "grad_norm": 0.846513569355011, + "learning_rate": 0.0001, + "loss": 64.2908, + "step": 5580 + }, + { + "epoch": 9.001612334357837, + "grad_norm": 0.9551037549972534, + "learning_rate": 0.0001, + "loss": 61.7312, + "step": 5590 + }, + { + "epoch": 9.017735677936212, + "grad_norm": 0.8250304460525513, + "learning_rate": 0.0001, + "loss": 66.3351, + "step": 5600 + }, + { + "epoch": 9.033859021514587, + "grad_norm": 0.7932960391044617, + "learning_rate": 0.0001, + "loss": 66.7753, + "step": 5610 + }, + { + "epoch": 9.04998236509296, + "grad_norm": 0.9839881658554077, + "learning_rate": 0.0001, + "loss": 64.5146, + "step": 5620 + }, + { + "epoch": 9.066105708671335, + "grad_norm": 0.7337850332260132, + "learning_rate": 0.0001, + "loss": 65.0544, + "step": 5630 + }, + { + "epoch": 9.08222905224971, + "grad_norm": 0.7496634721755981, + "learning_rate": 0.0001, + "loss": 64.1451, + "step": 5640 + }, + { + "epoch": 9.098352395828085, + "grad_norm": 0.9457297921180725, + "learning_rate": 0.0001, + "loss": 65.3036, + "step": 5650 + }, + { + "epoch": 9.11447573940646, + "grad_norm": 0.8315348625183105, + "learning_rate": 0.0001, + "loss": 66.0585, + "step": 5660 + }, + { + "epoch": 9.130599082984833, + "grad_norm": 0.9666195511817932, + "learning_rate": 0.0001, + "loss": 64.6802, + "step": 5670 + }, + { + "epoch": 9.146722426563208, + "grad_norm": 0.7416576147079468, + "learning_rate": 0.0001, + "loss": 65.1557, + "step": 5680 + }, + { + "epoch": 9.162845770141583, + "grad_norm": 0.7723799347877502, + "learning_rate": 0.0001, + "loss": 66.2871, + "step": 5690 + }, + { + "epoch": 9.178969113719958, + "grad_norm": 0.7461305856704712, + "learning_rate": 0.0001, + "loss": 65.5795, + "step": 5700 + }, + { + "epoch": 9.195092457298331, + "grad_norm": 0.6663411259651184, + "learning_rate": 0.0001, + "loss": 66.4094, + "step": 5710 + }, + { + "epoch": 9.211215800876706, + "grad_norm": 0.9340134859085083, + "learning_rate": 0.0001, + "loss": 63.5753, + "step": 5720 + }, + { + "epoch": 9.227339144455081, + "grad_norm": 0.7643963694572449, + "learning_rate": 0.0001, + "loss": 65.3359, + "step": 5730 + }, + { + "epoch": 9.243462488033456, + "grad_norm": 0.6931570768356323, + "learning_rate": 0.0001, + "loss": 65.0755, + "step": 5740 + }, + { + "epoch": 9.259585831611831, + "grad_norm": 0.9096988439559937, + "learning_rate": 0.0001, + "loss": 64.7983, + "step": 5750 + }, + { + "epoch": 9.275709175190205, + "grad_norm": 0.8360128402709961, + "learning_rate": 0.0001, + "loss": 64.3604, + "step": 5760 + }, + { + "epoch": 9.29183251876858, + "grad_norm": 0.9302591681480408, + "learning_rate": 0.0001, + "loss": 66.2754, + "step": 5770 + }, + { + "epoch": 9.307955862346954, + "grad_norm": 0.7000735998153687, + "learning_rate": 0.0001, + "loss": 65.2241, + "step": 5780 + }, + { + "epoch": 9.32407920592533, + "grad_norm": 0.7321175932884216, + "learning_rate": 0.0001, + "loss": 65.3558, + "step": 5790 + }, + { + "epoch": 9.340202549503703, + "grad_norm": 0.7948715090751648, + "learning_rate": 0.0001, + "loss": 64.2793, + "step": 5800 + }, + { + "epoch": 9.356325893082078, + "grad_norm": 0.7676129341125488, + "learning_rate": 0.0001, + "loss": 66.4778, + "step": 5810 + }, + { + "epoch": 9.372449236660453, + "grad_norm": 0.977236270904541, + "learning_rate": 0.0001, + "loss": 64.5587, + "step": 5820 + }, + { + "epoch": 9.388572580238828, + "grad_norm": 0.86808842420578, + "learning_rate": 0.0001, + "loss": 65.8026, + "step": 5830 + }, + { + "epoch": 9.4046959238172, + "grad_norm": 0.7455324530601501, + "learning_rate": 0.0001, + "loss": 65.066, + "step": 5840 + }, + { + "epoch": 9.420819267395576, + "grad_norm": 0.6784212589263916, + "learning_rate": 0.0001, + "loss": 65.4167, + "step": 5850 + }, + { + "epoch": 9.43694261097395, + "grad_norm": 1.033731460571289, + "learning_rate": 0.0001, + "loss": 66.4502, + "step": 5860 + }, + { + "epoch": 9.453065954552326, + "grad_norm": 0.7946107983589172, + "learning_rate": 0.0001, + "loss": 63.7567, + "step": 5870 + }, + { + "epoch": 9.4691892981307, + "grad_norm": 0.7826364636421204, + "learning_rate": 0.0001, + "loss": 65.4484, + "step": 5880 + }, + { + "epoch": 9.485312641709074, + "grad_norm": 0.7605016827583313, + "learning_rate": 0.0001, + "loss": 66.8459, + "step": 5890 + }, + { + "epoch": 9.501435985287449, + "grad_norm": 0.7383126020431519, + "learning_rate": 0.0001, + "loss": 64.6873, + "step": 5900 + }, + { + "epoch": 9.517559328865824, + "grad_norm": 0.8355686664581299, + "learning_rate": 0.0001, + "loss": 63.5145, + "step": 5910 + }, + { + "epoch": 9.533682672444199, + "grad_norm": 0.8827091455459595, + "learning_rate": 0.0001, + "loss": 67.2207, + "step": 5920 + }, + { + "epoch": 9.549806016022572, + "grad_norm": 0.6988142728805542, + "learning_rate": 0.0001, + "loss": 64.2289, + "step": 5930 + }, + { + "epoch": 9.565929359600947, + "grad_norm": 0.7609000205993652, + "learning_rate": 0.0001, + "loss": 63.9324, + "step": 5940 + }, + { + "epoch": 9.582052703179322, + "grad_norm": 0.8053853511810303, + "learning_rate": 0.0001, + "loss": 62.5564, + "step": 5950 + }, + { + "epoch": 9.598176046757697, + "grad_norm": 0.7684671878814697, + "learning_rate": 0.0001, + "loss": 66.2669, + "step": 5960 + }, + { + "epoch": 9.614299390336072, + "grad_norm": 0.8411065340042114, + "learning_rate": 0.0001, + "loss": 64.5087, + "step": 5970 + }, + { + "epoch": 9.630422733914445, + "grad_norm": 0.963498592376709, + "learning_rate": 0.0001, + "loss": 68.1736, + "step": 5980 + }, + { + "epoch": 9.64654607749282, + "grad_norm": 0.8715981841087341, + "learning_rate": 0.0001, + "loss": 64.1686, + "step": 5990 + }, + { + "epoch": 9.662669421071195, + "grad_norm": 0.8217800855636597, + "learning_rate": 0.0001, + "loss": 64.2211, + "step": 6000 + }, + { + "epoch": 9.67879276464957, + "grad_norm": 0.7824742197990417, + "learning_rate": 0.0001, + "loss": 65.9052, + "step": 6010 + }, + { + "epoch": 9.694916108227943, + "grad_norm": 0.7440009117126465, + "learning_rate": 0.0001, + "loss": 63.3749, + "step": 6020 + }, + { + "epoch": 9.711039451806318, + "grad_norm": 0.8759479522705078, + "learning_rate": 0.0001, + "loss": 63.4488, + "step": 6030 + }, + { + "epoch": 9.727162795384693, + "grad_norm": 3.662144184112549, + "learning_rate": 0.0001, + "loss": 62.8787, + "step": 6040 + }, + { + "epoch": 9.743286138963068, + "grad_norm": 1.007149577140808, + "learning_rate": 0.0001, + "loss": 66.0053, + "step": 6050 + }, + { + "epoch": 9.759409482541441, + "grad_norm": 0.8369242548942566, + "learning_rate": 0.0001, + "loss": 67.233, + "step": 6060 + }, + { + "epoch": 9.775532826119816, + "grad_norm": 0.8160434365272522, + "learning_rate": 0.0001, + "loss": 65.2089, + "step": 6070 + }, + { + "epoch": 9.791656169698191, + "grad_norm": 0.8482319712638855, + "learning_rate": 0.0001, + "loss": 65.0718, + "step": 6080 + }, + { + "epoch": 9.807779513276566, + "grad_norm": 0.8800129890441895, + "learning_rate": 0.0001, + "loss": 65.145, + "step": 6090 + }, + { + "epoch": 9.82390285685494, + "grad_norm": 0.8911405801773071, + "learning_rate": 0.0001, + "loss": 65.7952, + "step": 6100 + }, + { + "epoch": 9.840026200433314, + "grad_norm": 0.8108358979225159, + "learning_rate": 0.0001, + "loss": 65.1928, + "step": 6110 + }, + { + "epoch": 9.85614954401169, + "grad_norm": 0.8919743895530701, + "learning_rate": 0.0001, + "loss": 65.8129, + "step": 6120 + }, + { + "epoch": 9.872272887590064, + "grad_norm": 0.73152756690979, + "learning_rate": 0.0001, + "loss": 65.7141, + "step": 6130 + }, + { + "epoch": 9.88839623116844, + "grad_norm": 0.8583903312683105, + "learning_rate": 0.0001, + "loss": 64.8231, + "step": 6140 + }, + { + "epoch": 9.904519574746812, + "grad_norm": 0.798543393611908, + "learning_rate": 0.0001, + "loss": 64.5073, + "step": 6150 + }, + { + "epoch": 9.920642918325187, + "grad_norm": 0.8657358288764954, + "learning_rate": 0.0001, + "loss": 64.1207, + "step": 6160 + }, + { + "epoch": 9.936766261903562, + "grad_norm": 0.9324677586555481, + "learning_rate": 0.0001, + "loss": 64.7129, + "step": 6170 + }, + { + "epoch": 9.952889605481937, + "grad_norm": 0.7371154427528381, + "learning_rate": 0.0001, + "loss": 64.0251, + "step": 6180 + }, + { + "epoch": 9.96901294906031, + "grad_norm": 0.8273237347602844, + "learning_rate": 0.0001, + "loss": 65.8633, + "step": 6190 + }, + { + "epoch": 9.985136292638686, + "grad_norm": 0.7219365239143372, + "learning_rate": 0.0001, + "loss": 63.4029, + "step": 6200 + }, + { + "epoch": 10.0, + "grad_norm": 0.76242995262146, + "learning_rate": 0.0001, + "loss": 60.9994, + "step": 6210 + }, + { + "epoch": 10.016123343578375, + "grad_norm": 0.8677287697792053, + "learning_rate": 0.0001, + "loss": 65.9687, + "step": 6220 + }, + { + "epoch": 10.03224668715675, + "grad_norm": 0.7073521018028259, + "learning_rate": 0.0001, + "loss": 63.0083, + "step": 6230 + }, + { + "epoch": 10.048370030735123, + "grad_norm": 0.794535756111145, + "learning_rate": 0.0001, + "loss": 63.0616, + "step": 6240 + }, + { + "epoch": 10.064493374313498, + "grad_norm": 0.6974631547927856, + "learning_rate": 0.0001, + "loss": 64.6579, + "step": 6250 + }, + { + "epoch": 10.080616717891873, + "grad_norm": 0.8279150128364563, + "learning_rate": 0.0001, + "loss": 65.1069, + "step": 6260 + }, + { + "epoch": 10.096740061470248, + "grad_norm": 0.8682932257652283, + "learning_rate": 0.0001, + "loss": 64.5019, + "step": 6270 + }, + { + "epoch": 10.112863405048621, + "grad_norm": 1.0158190727233887, + "learning_rate": 0.0001, + "loss": 65.8854, + "step": 6280 + }, + { + "epoch": 10.128986748626996, + "grad_norm": 0.6911011934280396, + "learning_rate": 0.0001, + "loss": 62.4879, + "step": 6290 + }, + { + "epoch": 10.145110092205371, + "grad_norm": 0.7394022941589355, + "learning_rate": 0.0001, + "loss": 64.2604, + "step": 6300 + }, + { + "epoch": 10.161233435783746, + "grad_norm": 0.7941621541976929, + "learning_rate": 0.0001, + "loss": 64.5133, + "step": 6310 + }, + { + "epoch": 10.17735677936212, + "grad_norm": 1.0238994359970093, + "learning_rate": 0.0001, + "loss": 63.8594, + "step": 6320 + }, + { + "epoch": 10.193480122940494, + "grad_norm": 0.7332146763801575, + "learning_rate": 0.0001, + "loss": 66.5434, + "step": 6330 + }, + { + "epoch": 10.20960346651887, + "grad_norm": 0.9375078082084656, + "learning_rate": 0.0001, + "loss": 63.4793, + "step": 6340 + }, + { + "epoch": 10.225726810097244, + "grad_norm": 0.715931236743927, + "learning_rate": 0.0001, + "loss": 65.8812, + "step": 6350 + }, + { + "epoch": 10.24185015367562, + "grad_norm": 1.291536808013916, + "learning_rate": 0.0001, + "loss": 64.9125, + "step": 6360 + }, + { + "epoch": 10.257973497253992, + "grad_norm": 0.8942781686782837, + "learning_rate": 0.0001, + "loss": 67.1706, + "step": 6370 + }, + { + "epoch": 10.274096840832367, + "grad_norm": 0.8272124528884888, + "learning_rate": 0.0001, + "loss": 62.1721, + "step": 6380 + }, + { + "epoch": 10.290220184410742, + "grad_norm": 0.8191152215003967, + "learning_rate": 0.0001, + "loss": 64.8931, + "step": 6390 + }, + { + "epoch": 10.306343527989117, + "grad_norm": 1.7500241994857788, + "learning_rate": 0.0001, + "loss": 62.9693, + "step": 6400 + }, + { + "epoch": 10.32246687156749, + "grad_norm": 1.0159430503845215, + "learning_rate": 0.0001, + "loss": 63.0889, + "step": 6410 + }, + { + "epoch": 10.338590215145866, + "grad_norm": 0.7300348281860352, + "learning_rate": 0.0001, + "loss": 65.1742, + "step": 6420 + }, + { + "epoch": 10.35471355872424, + "grad_norm": 1.3132387399673462, + "learning_rate": 0.0001, + "loss": 64.8867, + "step": 6430 + }, + { + "epoch": 10.370836902302615, + "grad_norm": 0.7502451539039612, + "learning_rate": 0.0001, + "loss": 66.5094, + "step": 6440 + }, + { + "epoch": 10.38696024588099, + "grad_norm": 0.9408686757087708, + "learning_rate": 0.0001, + "loss": 65.1515, + "step": 6450 + }, + { + "epoch": 10.403083589459364, + "grad_norm": 0.769598126411438, + "learning_rate": 0.0001, + "loss": 65.1478, + "step": 6460 + }, + { + "epoch": 10.419206933037739, + "grad_norm": 0.9086162447929382, + "learning_rate": 0.0001, + "loss": 63.5263, + "step": 6470 + }, + { + "epoch": 10.435330276616114, + "grad_norm": 0.7899322509765625, + "learning_rate": 0.0001, + "loss": 63.3939, + "step": 6480 + }, + { + "epoch": 10.451453620194489, + "grad_norm": 0.9135434031486511, + "learning_rate": 0.0001, + "loss": 63.867, + "step": 6490 + }, + { + "epoch": 10.467576963772862, + "grad_norm": 0.9312737584114075, + "learning_rate": 0.0001, + "loss": 63.6939, + "step": 6500 + }, + { + "epoch": 10.483700307351237, + "grad_norm": 0.861764132976532, + "learning_rate": 0.0001, + "loss": 63.4523, + "step": 6510 + }, + { + "epoch": 10.499823650929612, + "grad_norm": 0.7056366801261902, + "learning_rate": 0.0001, + "loss": 63.2889, + "step": 6520 + }, + { + "epoch": 10.515946994507987, + "grad_norm": 0.7007710337638855, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 6530 + }, + { + "epoch": 10.53207033808636, + "grad_norm": 0.7091413736343384, + "learning_rate": 0.0001, + "loss": 63.4185, + "step": 6540 + }, + { + "epoch": 10.548193681664735, + "grad_norm": 0.7147234678268433, + "learning_rate": 0.0001, + "loss": 65.0726, + "step": 6550 + }, + { + "epoch": 10.56431702524311, + "grad_norm": 0.7954673767089844, + "learning_rate": 0.0001, + "loss": 65.0334, + "step": 6560 + }, + { + "epoch": 10.580440368821485, + "grad_norm": 0.725352942943573, + "learning_rate": 0.0001, + "loss": 62.6663, + "step": 6570 + }, + { + "epoch": 10.59656371239986, + "grad_norm": 0.9563508629798889, + "learning_rate": 0.0001, + "loss": 64.0705, + "step": 6580 + }, + { + "epoch": 10.612687055978233, + "grad_norm": 0.7778734564781189, + "learning_rate": 0.0001, + "loss": 63.7619, + "step": 6590 + }, + { + "epoch": 10.628810399556608, + "grad_norm": 0.8414369821548462, + "learning_rate": 0.0001, + "loss": 63.9238, + "step": 6600 + }, + { + "epoch": 10.644933743134983, + "grad_norm": 0.8070562481880188, + "learning_rate": 0.0001, + "loss": 63.9591, + "step": 6610 + }, + { + "epoch": 10.661057086713358, + "grad_norm": 0.6493734121322632, + "learning_rate": 0.0001, + "loss": 63.7237, + "step": 6620 + }, + { + "epoch": 10.677180430291731, + "grad_norm": 0.8263372778892517, + "learning_rate": 0.0001, + "loss": 64.2262, + "step": 6630 + }, + { + "epoch": 10.693303773870106, + "grad_norm": 1.0090227127075195, + "learning_rate": 0.0001, + "loss": 63.5469, + "step": 6640 + }, + { + "epoch": 10.709427117448481, + "grad_norm": 0.998319685459137, + "learning_rate": 0.0001, + "loss": 62.9113, + "step": 6650 + }, + { + "epoch": 10.725550461026856, + "grad_norm": 1.006489634513855, + "learning_rate": 0.0001, + "loss": 66.1458, + "step": 6660 + }, + { + "epoch": 10.741673804605231, + "grad_norm": 0.9575757384300232, + "learning_rate": 0.0001, + "loss": 63.0913, + "step": 6670 + }, + { + "epoch": 10.757797148183604, + "grad_norm": 0.6832238435745239, + "learning_rate": 0.0001, + "loss": 66.7501, + "step": 6680 + }, + { + "epoch": 10.773920491761979, + "grad_norm": 0.8075717687606812, + "learning_rate": 0.0001, + "loss": 64.2839, + "step": 6690 + }, + { + "epoch": 10.790043835340354, + "grad_norm": 0.9402624368667603, + "learning_rate": 0.0001, + "loss": 64.9296, + "step": 6700 + }, + { + "epoch": 10.806167178918729, + "grad_norm": 0.7681158185005188, + "learning_rate": 0.0001, + "loss": 63.5686, + "step": 6710 + }, + { + "epoch": 10.822290522497102, + "grad_norm": 0.8451285362243652, + "learning_rate": 0.0001, + "loss": 64.5404, + "step": 6720 + }, + { + "epoch": 10.838413866075477, + "grad_norm": 0.7278769016265869, + "learning_rate": 0.0001, + "loss": 64.1341, + "step": 6730 + }, + { + "epoch": 10.854537209653852, + "grad_norm": 0.8524075746536255, + "learning_rate": 0.0001, + "loss": 64.0483, + "step": 6740 + }, + { + "epoch": 10.870660553232227, + "grad_norm": 0.703598141670227, + "learning_rate": 0.0001, + "loss": 64.0072, + "step": 6750 + }, + { + "epoch": 10.8867838968106, + "grad_norm": 0.7740384936332703, + "learning_rate": 0.0001, + "loss": 63.4171, + "step": 6760 + }, + { + "epoch": 10.902907240388975, + "grad_norm": 0.8302356004714966, + "learning_rate": 0.0001, + "loss": 65.5159, + "step": 6770 + }, + { + "epoch": 10.91903058396735, + "grad_norm": 1.0289733409881592, + "learning_rate": 0.0001, + "loss": 64.9785, + "step": 6780 + }, + { + "epoch": 10.935153927545725, + "grad_norm": 0.6721987128257751, + "learning_rate": 0.0001, + "loss": 63.965, + "step": 6790 + }, + { + "epoch": 10.951277271124098, + "grad_norm": 0.7614948749542236, + "learning_rate": 0.0001, + "loss": 64.1375, + "step": 6800 + }, + { + "epoch": 10.967400614702473, + "grad_norm": 0.7342216372489929, + "learning_rate": 0.0001, + "loss": 64.4342, + "step": 6810 + }, + { + "epoch": 10.983523958280848, + "grad_norm": 0.8808079957962036, + "learning_rate": 0.0001, + "loss": 65.2864, + "step": 6820 + }, + { + "epoch": 10.999647301859223, + "grad_norm": 0.818477988243103, + "learning_rate": 0.0001, + "loss": 63.9931, + "step": 6830 + }, + { + "epoch": 11.014511009220538, + "grad_norm": 0.7867474555969238, + "learning_rate": 0.0001, + "loss": 59.2426, + "step": 6840 + }, + { + "epoch": 11.030634352798911, + "grad_norm": 0.6163678765296936, + "learning_rate": 0.0001, + "loss": 65.0149, + "step": 6850 + }, + { + "epoch": 11.046757696377286, + "grad_norm": 0.8334609866142273, + "learning_rate": 0.0001, + "loss": 63.5017, + "step": 6860 + }, + { + "epoch": 11.062881039955661, + "grad_norm": 0.7335264086723328, + "learning_rate": 0.0001, + "loss": 64.6307, + "step": 6870 + }, + { + "epoch": 11.079004383534036, + "grad_norm": 0.7723846435546875, + "learning_rate": 0.0001, + "loss": 64.316, + "step": 6880 + }, + { + "epoch": 11.09512772711241, + "grad_norm": 0.7972145080566406, + "learning_rate": 0.0001, + "loss": 63.1469, + "step": 6890 + }, + { + "epoch": 11.111251070690784, + "grad_norm": 0.7696133852005005, + "learning_rate": 0.0001, + "loss": 64.0338, + "step": 6900 + }, + { + "epoch": 11.127374414269159, + "grad_norm": 0.8972133994102478, + "learning_rate": 0.0001, + "loss": 65.5727, + "step": 6910 + }, + { + "epoch": 11.143497757847534, + "grad_norm": 0.6776295900344849, + "learning_rate": 0.0001, + "loss": 66.176, + "step": 6920 + }, + { + "epoch": 11.159621101425909, + "grad_norm": 0.7345874905586243, + "learning_rate": 0.0001, + "loss": 64.0949, + "step": 6930 + }, + { + "epoch": 11.175744445004282, + "grad_norm": 0.7988374829292297, + "learning_rate": 0.0001, + "loss": 64.7721, + "step": 6940 + }, + { + "epoch": 11.191867788582657, + "grad_norm": 0.7552387714385986, + "learning_rate": 0.0001, + "loss": 63.1053, + "step": 6950 + }, + { + "epoch": 11.207991132161032, + "grad_norm": 0.6388214230537415, + "learning_rate": 0.0001, + "loss": 63.7871, + "step": 6960 + }, + { + "epoch": 11.224114475739407, + "grad_norm": 0.6837960481643677, + "learning_rate": 0.0001, + "loss": 63.906, + "step": 6970 + }, + { + "epoch": 11.24023781931778, + "grad_norm": 0.7050679326057434, + "learning_rate": 0.0001, + "loss": 64.2803, + "step": 6980 + }, + { + "epoch": 11.256361162896155, + "grad_norm": 0.8619478940963745, + "learning_rate": 0.0001, + "loss": 62.8786, + "step": 6990 + }, + { + "epoch": 11.27248450647453, + "grad_norm": 0.9759813547134399, + "learning_rate": 0.0001, + "loss": 65.2299, + "step": 7000 + }, + { + "epoch": 11.288607850052905, + "grad_norm": 0.7584514021873474, + "learning_rate": 0.0001, + "loss": 61.4178, + "step": 7010 + }, + { + "epoch": 11.304731193631278, + "grad_norm": 0.8467874526977539, + "learning_rate": 0.0001, + "loss": 65.6229, + "step": 7020 + }, + { + "epoch": 11.320854537209653, + "grad_norm": 0.7121654152870178, + "learning_rate": 0.0001, + "loss": 63.9061, + "step": 7030 + }, + { + "epoch": 11.336977880788028, + "grad_norm": 0.8885557055473328, + "learning_rate": 0.0001, + "loss": 65.1436, + "step": 7040 + }, + { + "epoch": 11.353101224366403, + "grad_norm": 0.6320177912712097, + "learning_rate": 0.0001, + "loss": 62.8535, + "step": 7050 + }, + { + "epoch": 11.369224567944778, + "grad_norm": 0.9287629723548889, + "learning_rate": 0.0001, + "loss": 62.7407, + "step": 7060 + }, + { + "epoch": 11.385347911523152, + "grad_norm": 0.7705608606338501, + "learning_rate": 0.0001, + "loss": 63.469, + "step": 7070 + }, + { + "epoch": 11.401471255101526, + "grad_norm": 0.6687246561050415, + "learning_rate": 0.0001, + "loss": 62.722, + "step": 7080 + }, + { + "epoch": 11.417594598679901, + "grad_norm": 0.7774648666381836, + "learning_rate": 0.0001, + "loss": 63.1145, + "step": 7090 + }, + { + "epoch": 11.433717942258276, + "grad_norm": 0.7808363437652588, + "learning_rate": 0.0001, + "loss": 63.9342, + "step": 7100 + }, + { + "epoch": 11.44984128583665, + "grad_norm": 0.8033562302589417, + "learning_rate": 0.0001, + "loss": 65.7876, + "step": 7110 + }, + { + "epoch": 11.465964629415025, + "grad_norm": 0.8143486976623535, + "learning_rate": 0.0001, + "loss": 64.8782, + "step": 7120 + }, + { + "epoch": 11.4820879729934, + "grad_norm": 0.7855561971664429, + "learning_rate": 0.0001, + "loss": 63.4944, + "step": 7130 + }, + { + "epoch": 11.498211316571775, + "grad_norm": 0.7525351047515869, + "learning_rate": 0.0001, + "loss": 62.4088, + "step": 7140 + }, + { + "epoch": 11.514334660150148, + "grad_norm": 0.7193494439125061, + "learning_rate": 0.0001, + "loss": 64.2386, + "step": 7150 + }, + { + "epoch": 11.530458003728523, + "grad_norm": 0.892281174659729, + "learning_rate": 0.0001, + "loss": 63.8075, + "step": 7160 + }, + { + "epoch": 11.546581347306898, + "grad_norm": 0.720287561416626, + "learning_rate": 0.0001, + "loss": 63.3259, + "step": 7170 + }, + { + "epoch": 11.562704690885273, + "grad_norm": 1.4404689073562622, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 7180 + }, + { + "epoch": 11.578828034463648, + "grad_norm": 0.9082726240158081, + "learning_rate": 0.0001, + "loss": 64.6877, + "step": 7190 + }, + { + "epoch": 11.59495137804202, + "grad_norm": 0.7137670516967773, + "learning_rate": 0.0001, + "loss": 63.4832, + "step": 7200 + }, + { + "epoch": 11.611074721620396, + "grad_norm": 0.6848503947257996, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 7210 + }, + { + "epoch": 11.62719806519877, + "grad_norm": 0.7483687996864319, + "learning_rate": 0.0001, + "loss": 63.294, + "step": 7220 + }, + { + "epoch": 11.643321408777146, + "grad_norm": 0.935321569442749, + "learning_rate": 0.0001, + "loss": 62.8669, + "step": 7230 + }, + { + "epoch": 11.659444752355519, + "grad_norm": 0.8359326124191284, + "learning_rate": 0.0001, + "loss": 64.4482, + "step": 7240 + }, + { + "epoch": 11.675568095933894, + "grad_norm": 0.7838515043258667, + "learning_rate": 0.0001, + "loss": 64.1975, + "step": 7250 + }, + { + "epoch": 11.691691439512269, + "grad_norm": 0.9128443598747253, + "learning_rate": 0.0001, + "loss": 63.1822, + "step": 7260 + }, + { + "epoch": 11.707814783090644, + "grad_norm": 1.0545419454574585, + "learning_rate": 0.0001, + "loss": 63.3112, + "step": 7270 + }, + { + "epoch": 11.723938126669019, + "grad_norm": 0.8572486639022827, + "learning_rate": 0.0001, + "loss": 64.9605, + "step": 7280 + }, + { + "epoch": 11.740061470247392, + "grad_norm": 1.0894116163253784, + "learning_rate": 0.0001, + "loss": 64.3671, + "step": 7290 + }, + { + "epoch": 11.756184813825767, + "grad_norm": 0.7500355839729309, + "learning_rate": 0.0001, + "loss": 62.6381, + "step": 7300 + }, + { + "epoch": 11.772308157404142, + "grad_norm": 0.7315185070037842, + "learning_rate": 0.0001, + "loss": 66.1438, + "step": 7310 + }, + { + "epoch": 11.788431500982517, + "grad_norm": 0.8123629093170166, + "learning_rate": 0.0001, + "loss": 62.6012, + "step": 7320 + }, + { + "epoch": 11.80455484456089, + "grad_norm": 0.8072763085365295, + "learning_rate": 0.0001, + "loss": 63.4424, + "step": 7330 + }, + { + "epoch": 11.820678188139265, + "grad_norm": 1.1077874898910522, + "learning_rate": 0.0001, + "loss": 63.2812, + "step": 7340 + }, + { + "epoch": 11.83680153171764, + "grad_norm": 0.7116726636886597, + "learning_rate": 0.0001, + "loss": 63.6077, + "step": 7350 + }, + { + "epoch": 11.852924875296015, + "grad_norm": 0.8817790746688843, + "learning_rate": 0.0001, + "loss": 63.1938, + "step": 7360 + }, + { + "epoch": 11.86904821887439, + "grad_norm": 1.021730661392212, + "learning_rate": 0.0001, + "loss": 65.2215, + "step": 7370 + }, + { + "epoch": 11.885171562452763, + "grad_norm": 0.9871250987052917, + "learning_rate": 0.0001, + "loss": 64.6468, + "step": 7380 + }, + { + "epoch": 11.901294906031138, + "grad_norm": 1.3591430187225342, + "learning_rate": 0.0001, + "loss": 61.5546, + "step": 7390 + }, + { + "epoch": 11.917418249609513, + "grad_norm": 0.8114227056503296, + "learning_rate": 0.0001, + "loss": 61.8215, + "step": 7400 + }, + { + "epoch": 11.933541593187888, + "grad_norm": 0.7151288986206055, + "learning_rate": 0.0001, + "loss": 61.4856, + "step": 7410 + }, + { + "epoch": 11.949664936766261, + "grad_norm": 0.7494287490844727, + "learning_rate": 0.0001, + "loss": 63.6451, + "step": 7420 + }, + { + "epoch": 11.965788280344636, + "grad_norm": 0.7377745509147644, + "learning_rate": 0.0001, + "loss": 63.002, + "step": 7430 + }, + { + "epoch": 11.981911623923011, + "grad_norm": 0.6534659266471863, + "learning_rate": 0.0001, + "loss": 63.6404, + "step": 7440 + }, + { + "epoch": 11.998034967501386, + "grad_norm": 0.7548736333847046, + "learning_rate": 0.0001, + "loss": 64.1986, + "step": 7450 + }, + { + "epoch": 12.012898674862699, + "grad_norm": 0.7227553129196167, + "learning_rate": 0.0001, + "loss": 56.8327, + "step": 7460 + }, + { + "epoch": 12.029022018441074, + "grad_norm": 0.6348695158958435, + "learning_rate": 0.0001, + "loss": 62.4472, + "step": 7470 + }, + { + "epoch": 12.045145362019449, + "grad_norm": 0.8249921202659607, + "learning_rate": 0.0001, + "loss": 65.1877, + "step": 7480 + }, + { + "epoch": 12.061268705597824, + "grad_norm": 0.7152777314186096, + "learning_rate": 0.0001, + "loss": 64.2609, + "step": 7490 + }, + { + "epoch": 12.077392049176197, + "grad_norm": 0.7399184703826904, + "learning_rate": 0.0001, + "loss": 63.3997, + "step": 7500 + }, + { + "epoch": 12.093515392754572, + "grad_norm": 0.6972354650497437, + "learning_rate": 0.0001, + "loss": 61.3573, + "step": 7510 + }, + { + "epoch": 12.109638736332947, + "grad_norm": 0.7666720151901245, + "learning_rate": 0.0001, + "loss": 63.8438, + "step": 7520 + }, + { + "epoch": 12.125762079911322, + "grad_norm": 0.7309302687644958, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 7530 + }, + { + "epoch": 12.141885423489697, + "grad_norm": 1.3805177211761475, + "learning_rate": 0.0001, + "loss": 64.68, + "step": 7540 + }, + { + "epoch": 12.15800876706807, + "grad_norm": 1.0315922498703003, + "learning_rate": 0.0001, + "loss": 63.6372, + "step": 7550 + }, + { + "epoch": 12.174132110646445, + "grad_norm": 1.2012155055999756, + "learning_rate": 0.0001, + "loss": 62.5765, + "step": 7560 + }, + { + "epoch": 12.19025545422482, + "grad_norm": 0.7769846320152283, + "learning_rate": 0.0001, + "loss": 63.9737, + "step": 7570 + }, + { + "epoch": 12.206378797803195, + "grad_norm": 0.6587379574775696, + "learning_rate": 0.0001, + "loss": 63.382, + "step": 7580 + }, + { + "epoch": 12.222502141381568, + "grad_norm": 0.8309435248374939, + "learning_rate": 0.0001, + "loss": 64.4027, + "step": 7590 + }, + { + "epoch": 12.238625484959943, + "grad_norm": 0.8221393823623657, + "learning_rate": 0.0001, + "loss": 62.8561, + "step": 7600 + }, + { + "epoch": 12.254748828538318, + "grad_norm": 0.8629311323165894, + "learning_rate": 0.0001, + "loss": 64.3673, + "step": 7610 + }, + { + "epoch": 12.270872172116693, + "grad_norm": 0.698113739490509, + "learning_rate": 0.0001, + "loss": 63.6153, + "step": 7620 + }, + { + "epoch": 12.286995515695068, + "grad_norm": 0.72906094789505, + "learning_rate": 0.0001, + "loss": 64.8439, + "step": 7630 + }, + { + "epoch": 12.303118859273441, + "grad_norm": 0.7402384877204895, + "learning_rate": 0.0001, + "loss": 64.2103, + "step": 7640 + }, + { + "epoch": 12.319242202851816, + "grad_norm": 0.9972009062767029, + "learning_rate": 0.0001, + "loss": 63.803, + "step": 7650 + }, + { + "epoch": 12.335365546430191, + "grad_norm": 0.7852274775505066, + "learning_rate": 0.0001, + "loss": 64.8144, + "step": 7660 + }, + { + "epoch": 12.351488890008566, + "grad_norm": 0.7433444857597351, + "learning_rate": 0.0001, + "loss": 63.0123, + "step": 7670 + }, + { + "epoch": 12.36761223358694, + "grad_norm": 0.8548759818077087, + "learning_rate": 0.0001, + "loss": 63.2215, + "step": 7680 + }, + { + "epoch": 12.383735577165314, + "grad_norm": 0.6759565472602844, + "learning_rate": 0.0001, + "loss": 62.1246, + "step": 7690 + }, + { + "epoch": 12.39985892074369, + "grad_norm": 0.9079536199569702, + "learning_rate": 0.0001, + "loss": 64.1692, + "step": 7700 + }, + { + "epoch": 12.415982264322064, + "grad_norm": 0.6890071034431458, + "learning_rate": 0.0001, + "loss": 62.5602, + "step": 7710 + }, + { + "epoch": 12.432105607900438, + "grad_norm": 0.8096348643302917, + "learning_rate": 0.0001, + "loss": 61.0103, + "step": 7720 + }, + { + "epoch": 12.448228951478812, + "grad_norm": 0.7359873056411743, + "learning_rate": 0.0001, + "loss": 62.1914, + "step": 7730 + }, + { + "epoch": 12.464352295057187, + "grad_norm": 1.0701091289520264, + "learning_rate": 0.0001, + "loss": 61.7295, + "step": 7740 + }, + { + "epoch": 12.480475638635562, + "grad_norm": 0.791928768157959, + "learning_rate": 0.0001, + "loss": 62.6812, + "step": 7750 + }, + { + "epoch": 12.496598982213937, + "grad_norm": 0.7719446420669556, + "learning_rate": 0.0001, + "loss": 61.9114, + "step": 7760 + }, + { + "epoch": 12.51272232579231, + "grad_norm": 0.8720338344573975, + "learning_rate": 0.0001, + "loss": 64.8107, + "step": 7770 + }, + { + "epoch": 12.528845669370686, + "grad_norm": 0.9999920129776001, + "learning_rate": 0.0001, + "loss": 61.198, + "step": 7780 + }, + { + "epoch": 12.54496901294906, + "grad_norm": 0.710754930973053, + "learning_rate": 0.0001, + "loss": 64.1462, + "step": 7790 + }, + { + "epoch": 12.561092356527435, + "grad_norm": 0.7401363849639893, + "learning_rate": 0.0001, + "loss": 61.8988, + "step": 7800 + }, + { + "epoch": 12.577215700105809, + "grad_norm": 0.9183729290962219, + "learning_rate": 0.0001, + "loss": 62.6392, + "step": 7810 + }, + { + "epoch": 12.593339043684184, + "grad_norm": 0.7341399788856506, + "learning_rate": 0.0001, + "loss": 62.7068, + "step": 7820 + }, + { + "epoch": 12.609462387262559, + "grad_norm": 0.8066722750663757, + "learning_rate": 0.0001, + "loss": 64.0625, + "step": 7830 + }, + { + "epoch": 12.625585730840934, + "grad_norm": 0.6088835597038269, + "learning_rate": 0.0001, + "loss": 64.3208, + "step": 7840 + }, + { + "epoch": 12.641709074419307, + "grad_norm": 0.6969302892684937, + "learning_rate": 0.0001, + "loss": 62.5817, + "step": 7850 + }, + { + "epoch": 12.657832417997682, + "grad_norm": 0.82635098695755, + "learning_rate": 0.0001, + "loss": 62.0061, + "step": 7860 + }, + { + "epoch": 12.673955761576057, + "grad_norm": 0.7016922235488892, + "learning_rate": 0.0001, + "loss": 62.5166, + "step": 7870 + }, + { + "epoch": 12.690079105154432, + "grad_norm": 0.8035777807235718, + "learning_rate": 0.0001, + "loss": 63.6543, + "step": 7880 + }, + { + "epoch": 12.706202448732807, + "grad_norm": 0.8573809862136841, + "learning_rate": 0.0001, + "loss": 62.5258, + "step": 7890 + }, + { + "epoch": 12.72232579231118, + "grad_norm": 0.9497472643852234, + "learning_rate": 0.0001, + "loss": 63.1133, + "step": 7900 + }, + { + "epoch": 12.738449135889555, + "grad_norm": 0.7494457960128784, + "learning_rate": 0.0001, + "loss": 63.8157, + "step": 7910 + }, + { + "epoch": 12.75457247946793, + "grad_norm": 0.7657570242881775, + "learning_rate": 0.0001, + "loss": 62.6255, + "step": 7920 + }, + { + "epoch": 12.770695823046305, + "grad_norm": 0.7848380208015442, + "learning_rate": 0.0001, + "loss": 62.9095, + "step": 7930 + }, + { + "epoch": 12.786819166624678, + "grad_norm": 0.8386877775192261, + "learning_rate": 0.0001, + "loss": 63.2254, + "step": 7940 + }, + { + "epoch": 12.802942510203053, + "grad_norm": 0.6774776577949524, + "learning_rate": 0.0001, + "loss": 64.3929, + "step": 7950 + }, + { + "epoch": 12.819065853781428, + "grad_norm": 0.798547625541687, + "learning_rate": 0.0001, + "loss": 62.9884, + "step": 7960 + }, + { + "epoch": 12.835189197359803, + "grad_norm": 0.9340888261795044, + "learning_rate": 0.0001, + "loss": 62.6197, + "step": 7970 + }, + { + "epoch": 12.851312540938178, + "grad_norm": 0.8211432695388794, + "learning_rate": 0.0001, + "loss": 62.7756, + "step": 7980 + }, + { + "epoch": 12.867435884516551, + "grad_norm": 0.7612001299858093, + "learning_rate": 0.0001, + "loss": 62.0792, + "step": 7990 + }, + { + "epoch": 12.883559228094926, + "grad_norm": 0.7743386626243591, + "learning_rate": 0.0001, + "loss": 63.9203, + "step": 8000 + }, + { + "epoch": 12.899682571673301, + "grad_norm": 0.7419672608375549, + "learning_rate": 0.0001, + "loss": 62.1705, + "step": 8010 + }, + { + "epoch": 12.915805915251676, + "grad_norm": 0.6556256413459778, + "learning_rate": 0.0001, + "loss": 64.5863, + "step": 8020 + }, + { + "epoch": 12.93192925883005, + "grad_norm": 1.136175274848938, + "learning_rate": 0.0001, + "loss": 63.2619, + "step": 8030 + }, + { + "epoch": 12.948052602408424, + "grad_norm": 0.8631499409675598, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8040 + }, + { + "epoch": 12.9641759459868, + "grad_norm": 0.8386172652244568, + "learning_rate": 0.0001, + "loss": 63.0598, + "step": 8050 + }, + { + "epoch": 12.980299289565174, + "grad_norm": 0.8078306913375854, + "learning_rate": 0.0001, + "loss": 62.6877, + "step": 8060 + }, + { + "epoch": 12.996422633143549, + "grad_norm": 0.8486189842224121, + "learning_rate": 0.0001, + "loss": 62.8643, + "step": 8070 + }, + { + "epoch": 13.011286340504862, + "grad_norm": 0.8717893958091736, + "learning_rate": 0.0001, + "loss": 57.6947, + "step": 8080 + }, + { + "epoch": 13.027409684083237, + "grad_norm": 1.2023789882659912, + "learning_rate": 0.0001, + "loss": 60.4728, + "step": 8090 + }, + { + "epoch": 13.043533027661612, + "grad_norm": 0.6879557967185974, + "learning_rate": 0.0001, + "loss": 61.7643, + "step": 8100 + }, + { + "epoch": 13.059656371239987, + "grad_norm": 1.0918916463851929, + "learning_rate": 0.0001, + "loss": 65.3682, + "step": 8110 + }, + { + "epoch": 13.07577971481836, + "grad_norm": 0.85753333568573, + "learning_rate": 0.0001, + "loss": 63.0815, + "step": 8120 + }, + { + "epoch": 13.091903058396735, + "grad_norm": 0.6568583846092224, + "learning_rate": 0.0001, + "loss": 64.7167, + "step": 8130 + }, + { + "epoch": 13.10802640197511, + "grad_norm": 0.6178144812583923, + "learning_rate": 0.0001, + "loss": 62.4294, + "step": 8140 + }, + { + "epoch": 13.124149745553485, + "grad_norm": 0.7930784821510315, + "learning_rate": 0.0001, + "loss": 64.542, + "step": 8150 + }, + { + "epoch": 13.140273089131858, + "grad_norm": 0.7675037980079651, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8160 + }, + { + "epoch": 13.156396432710233, + "grad_norm": 0.786997377872467, + "learning_rate": 0.0001, + "loss": 64.1495, + "step": 8170 + }, + { + "epoch": 13.172519776288608, + "grad_norm": 0.7752560377120972, + "learning_rate": 0.0001, + "loss": 61.6483, + "step": 8180 + }, + { + "epoch": 13.188643119866983, + "grad_norm": 0.8455203771591187, + "learning_rate": 0.0001, + "loss": 63.3551, + "step": 8190 + }, + { + "epoch": 13.204766463445356, + "grad_norm": 0.7223097681999207, + "learning_rate": 0.0001, + "loss": 63.1473, + "step": 8200 + }, + { + "epoch": 13.220889807023731, + "grad_norm": 0.7688442468643188, + "learning_rate": 0.0001, + "loss": 62.6723, + "step": 8210 + }, + { + "epoch": 13.237013150602106, + "grad_norm": 0.7730395793914795, + "learning_rate": 0.0001, + "loss": 64.3424, + "step": 8220 + }, + { + "epoch": 13.253136494180481, + "grad_norm": 0.8479727506637573, + "learning_rate": 0.0001, + "loss": 61.4336, + "step": 8230 + }, + { + "epoch": 13.269259837758856, + "grad_norm": 0.9070560932159424, + "learning_rate": 0.0001, + "loss": 63.1221, + "step": 8240 + }, + { + "epoch": 13.28538318133723, + "grad_norm": 0.8924600481987, + "learning_rate": 0.0001, + "loss": 63.9172, + "step": 8250 + }, + { + "epoch": 13.301506524915604, + "grad_norm": 1.0688443183898926, + "learning_rate": 0.0001, + "loss": 63.7944, + "step": 8260 + }, + { + "epoch": 13.31762986849398, + "grad_norm": 0.6866253018379211, + "learning_rate": 0.0001, + "loss": 65.5847, + "step": 8270 + }, + { + "epoch": 13.333753212072354, + "grad_norm": 0.711171567440033, + "learning_rate": 0.0001, + "loss": 62.698, + "step": 8280 + }, + { + "epoch": 13.349876555650727, + "grad_norm": 0.8128448724746704, + "learning_rate": 0.0001, + "loss": 61.5992, + "step": 8290 + }, + { + "epoch": 13.365999899229102, + "grad_norm": 0.7483755350112915, + "learning_rate": 0.0001, + "loss": 63.782, + "step": 8300 + }, + { + "epoch": 13.382123242807477, + "grad_norm": 0.7872599363327026, + "learning_rate": 0.0001, + "loss": 65.4922, + "step": 8310 + }, + { + "epoch": 13.398246586385852, + "grad_norm": 0.6861543655395508, + "learning_rate": 0.0001, + "loss": 62.6946, + "step": 8320 + }, + { + "epoch": 13.414369929964227, + "grad_norm": 0.7192599773406982, + "learning_rate": 0.0001, + "loss": 62.894, + "step": 8330 + }, + { + "epoch": 13.4304932735426, + "grad_norm": 0.7665286064147949, + "learning_rate": 0.0001, + "loss": 62.6253, + "step": 8340 + }, + { + "epoch": 13.446616617120975, + "grad_norm": 0.7909309267997742, + "learning_rate": 0.0001, + "loss": 62.5355, + "step": 8350 + }, + { + "epoch": 13.46273996069935, + "grad_norm": 0.7339541912078857, + "learning_rate": 0.0001, + "loss": 62.8514, + "step": 8360 + }, + { + "epoch": 13.478863304277725, + "grad_norm": 0.9893004298210144, + "learning_rate": 0.0001, + "loss": 63.7751, + "step": 8370 + }, + { + "epoch": 13.494986647856098, + "grad_norm": 0.9109547734260559, + "learning_rate": 0.0001, + "loss": 62.6697, + "step": 8380 + }, + { + "epoch": 13.511109991434473, + "grad_norm": 0.6807034015655518, + "learning_rate": 0.0001, + "loss": 62.1418, + "step": 8390 + }, + { + "epoch": 13.527233335012848, + "grad_norm": 0.915533185005188, + "learning_rate": 0.0001, + "loss": 61.7645, + "step": 8400 + }, + { + "epoch": 13.543356678591223, + "grad_norm": 0.8662852048873901, + "learning_rate": 0.0001, + "loss": 62.5546, + "step": 8410 + }, + { + "epoch": 13.559480022169598, + "grad_norm": 0.8678365349769592, + "learning_rate": 0.0001, + "loss": 63.8795, + "step": 8420 + }, + { + "epoch": 13.575603365747972, + "grad_norm": 0.6808590888977051, + "learning_rate": 0.0001, + "loss": 64.158, + "step": 8430 + }, + { + "epoch": 13.591726709326347, + "grad_norm": 0.7896480560302734, + "learning_rate": 0.0001, + "loss": 62.1052, + "step": 8440 + }, + { + "epoch": 13.607850052904721, + "grad_norm": 0.8973979949951172, + "learning_rate": 0.0001, + "loss": 64.478, + "step": 8450 + }, + { + "epoch": 13.623973396483096, + "grad_norm": 0.8228474259376526, + "learning_rate": 0.0001, + "loss": 61.2416, + "step": 8460 + }, + { + "epoch": 13.64009674006147, + "grad_norm": 0.9414492845535278, + "learning_rate": 0.0001, + "loss": 62.816, + "step": 8470 + }, + { + "epoch": 13.656220083639845, + "grad_norm": 0.9369436502456665, + "learning_rate": 0.0001, + "loss": 63.4951, + "step": 8480 + }, + { + "epoch": 13.67234342721822, + "grad_norm": 0.7152880430221558, + "learning_rate": 0.0001, + "loss": 62.4737, + "step": 8490 + }, + { + "epoch": 13.688466770796595, + "grad_norm": 0.7364594340324402, + "learning_rate": 0.0001, + "loss": 63.8688, + "step": 8500 + }, + { + "epoch": 13.704590114374968, + "grad_norm": 0.8499143123626709, + "learning_rate": 0.0001, + "loss": 64.5654, + "step": 8510 + }, + { + "epoch": 13.720713457953343, + "grad_norm": 0.7699156999588013, + "learning_rate": 0.0001, + "loss": 63.3726, + "step": 8520 + }, + { + "epoch": 13.736836801531718, + "grad_norm": 0.9146804809570312, + "learning_rate": 0.0001, + "loss": 63.6565, + "step": 8530 + }, + { + "epoch": 13.752960145110093, + "grad_norm": 0.773318350315094, + "learning_rate": 0.0001, + "loss": 65.0719, + "step": 8540 + }, + { + "epoch": 13.769083488688466, + "grad_norm": 0.6158450245857239, + "learning_rate": 0.0001, + "loss": 60.7291, + "step": 8550 + }, + { + "epoch": 13.78520683226684, + "grad_norm": 0.8053523302078247, + "learning_rate": 0.0001, + "loss": 62.5622, + "step": 8560 + }, + { + "epoch": 13.801330175845216, + "grad_norm": 0.8983457684516907, + "learning_rate": 0.0001, + "loss": 64.2921, + "step": 8570 + }, + { + "epoch": 13.81745351942359, + "grad_norm": 0.8214695453643799, + "learning_rate": 0.0001, + "loss": 62.2649, + "step": 8580 + }, + { + "epoch": 13.833576863001966, + "grad_norm": 0.8079172372817993, + "learning_rate": 0.0001, + "loss": 64.0828, + "step": 8590 + }, + { + "epoch": 13.849700206580339, + "grad_norm": 1.126568078994751, + "learning_rate": 0.0001, + "loss": 62.3927, + "step": 8600 + }, + { + "epoch": 13.865823550158714, + "grad_norm": 0.7597793936729431, + "learning_rate": 0.0001, + "loss": 62.5922, + "step": 8610 + }, + { + "epoch": 13.881946893737089, + "grad_norm": 0.671531617641449, + "learning_rate": 0.0001, + "loss": 61.9885, + "step": 8620 + }, + { + "epoch": 13.898070237315464, + "grad_norm": 0.823423445224762, + "learning_rate": 0.0001, + "loss": 62.0632, + "step": 8630 + }, + { + "epoch": 13.914193580893837, + "grad_norm": 0.7694658637046814, + "learning_rate": 0.0001, + "loss": 64.5485, + "step": 8640 + }, + { + "epoch": 13.930316924472212, + "grad_norm": 0.7394612431526184, + "learning_rate": 0.0001, + "loss": 61.9626, + "step": 8650 + }, + { + "epoch": 13.946440268050587, + "grad_norm": 0.724054217338562, + "learning_rate": 0.0001, + "loss": 62.3761, + "step": 8660 + }, + { + "epoch": 13.962563611628962, + "grad_norm": 0.7160128951072693, + "learning_rate": 0.0001, + "loss": 60.5142, + "step": 8670 + }, + { + "epoch": 13.978686955207337, + "grad_norm": 1.0045746564865112, + "learning_rate": 0.0001, + "loss": 62.7059, + "step": 8680 + }, + { + "epoch": 13.99481029878571, + "grad_norm": 0.6980776190757751, + "learning_rate": 0.0001, + "loss": 62.9427, + "step": 8690 + }, + { + "epoch": 14.009674006147025, + "grad_norm": 0.810961127281189, + "learning_rate": 0.0001, + "loss": 58.7552, + "step": 8700 + }, + { + "epoch": 14.0257973497254, + "grad_norm": 0.9210551381111145, + "learning_rate": 0.0001, + "loss": 63.7412, + "step": 8710 + }, + { + "epoch": 14.041920693303775, + "grad_norm": 0.7805785536766052, + "learning_rate": 0.0001, + "loss": 61.477, + "step": 8720 + }, + { + "epoch": 14.058044036882148, + "grad_norm": 0.8528208136558533, + "learning_rate": 0.0001, + "loss": 62.6746, + "step": 8730 + }, + { + "epoch": 14.074167380460523, + "grad_norm": 0.7467619776725769, + "learning_rate": 0.0001, + "loss": 63.8744, + "step": 8740 + }, + { + "epoch": 14.090290724038898, + "grad_norm": 0.6434402465820312, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 8750 + }, + { + "epoch": 14.106414067617273, + "grad_norm": 0.7217757701873779, + "learning_rate": 0.0001, + "loss": 64.0428, + "step": 8760 + }, + { + "epoch": 14.122537411195646, + "grad_norm": 0.8665125966072083, + "learning_rate": 0.0001, + "loss": 62.2772, + "step": 8770 + }, + { + "epoch": 14.13866075477402, + "grad_norm": 0.7868321537971497, + "learning_rate": 0.0001, + "loss": 62.0509, + "step": 8780 + }, + { + "epoch": 14.154784098352396, + "grad_norm": 0.8716760873794556, + "learning_rate": 0.0001, + "loss": 61.6021, + "step": 8790 + }, + { + "epoch": 14.17090744193077, + "grad_norm": 1.0922521352767944, + "learning_rate": 0.0001, + "loss": 64.4197, + "step": 8800 + }, + { + "epoch": 14.187030785509146, + "grad_norm": 0.6545577049255371, + "learning_rate": 0.0001, + "loss": 62.9259, + "step": 8810 + }, + { + "epoch": 14.203154129087519, + "grad_norm": 0.7634063959121704, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 8820 + }, + { + "epoch": 14.219277472665894, + "grad_norm": 0.9075300097465515, + "learning_rate": 0.0001, + "loss": 62.9128, + "step": 8830 + }, + { + "epoch": 14.235400816244269, + "grad_norm": 3.475320816040039, + "learning_rate": 0.0001, + "loss": 62.4841, + "step": 8840 + }, + { + "epoch": 14.251524159822644, + "grad_norm": 0.6839455962181091, + "learning_rate": 0.0001, + "loss": 61.5898, + "step": 8850 + }, + { + "epoch": 14.267647503401017, + "grad_norm": 0.8024231791496277, + "learning_rate": 0.0001, + "loss": 61.1875, + "step": 8860 + }, + { + "epoch": 14.283770846979392, + "grad_norm": 0.8948174118995667, + "learning_rate": 0.0001, + "loss": 62.3386, + "step": 8870 + }, + { + "epoch": 14.299894190557767, + "grad_norm": 0.7571985721588135, + "learning_rate": 0.0001, + "loss": 60.579, + "step": 8880 + }, + { + "epoch": 14.316017534136142, + "grad_norm": 1.432371973991394, + "learning_rate": 0.0001, + "loss": 62.4405, + "step": 8890 + }, + { + "epoch": 14.332140877714515, + "grad_norm": 0.6696875691413879, + "learning_rate": 0.0001, + "loss": 62.9984, + "step": 8900 + }, + { + "epoch": 14.34826422129289, + "grad_norm": 0.749396026134491, + "learning_rate": 0.0001, + "loss": 61.492, + "step": 8910 + }, + { + "epoch": 14.364387564871265, + "grad_norm": 0.664193332195282, + "learning_rate": 0.0001, + "loss": 62.9893, + "step": 8920 + }, + { + "epoch": 14.38051090844964, + "grad_norm": 0.7032697796821594, + "learning_rate": 0.0001, + "loss": 61.1682, + "step": 8930 + }, + { + "epoch": 14.396634252028015, + "grad_norm": 0.813119649887085, + "learning_rate": 0.0001, + "loss": 62.906, + "step": 8940 + }, + { + "epoch": 14.412757595606388, + "grad_norm": 0.7699596881866455, + "learning_rate": 0.0001, + "loss": 64.8087, + "step": 8950 + }, + { + "epoch": 14.428880939184763, + "grad_norm": 0.6424354910850525, + "learning_rate": 0.0001, + "loss": 62.2418, + "step": 8960 + }, + { + "epoch": 14.445004282763138, + "grad_norm": 0.8741731643676758, + "learning_rate": 0.0001, + "loss": 62.656, + "step": 8970 + }, + { + "epoch": 14.461127626341513, + "grad_norm": 0.710617184638977, + "learning_rate": 0.0001, + "loss": 60.8979, + "step": 8980 + }, + { + "epoch": 14.477250969919886, + "grad_norm": 0.6493995785713196, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 8990 + }, + { + "epoch": 14.493374313498261, + "grad_norm": 0.6655835509300232, + "learning_rate": 0.0001, + "loss": 64.151, + "step": 9000 + }, + { + "epoch": 14.509497657076636, + "grad_norm": 0.8140881657600403, + "learning_rate": 0.0001, + "loss": 62.4447, + "step": 9010 + }, + { + "epoch": 14.525621000655011, + "grad_norm": 0.7288498878479004, + "learning_rate": 0.0001, + "loss": 62.5189, + "step": 9020 + }, + { + "epoch": 14.541744344233386, + "grad_norm": 0.6972017884254456, + "learning_rate": 0.0001, + "loss": 61.5441, + "step": 9030 + }, + { + "epoch": 14.55786768781176, + "grad_norm": 0.7444366216659546, + "learning_rate": 0.0001, + "loss": 62.3233, + "step": 9040 + }, + { + "epoch": 14.573991031390134, + "grad_norm": 0.6602845191955566, + "learning_rate": 0.0001, + "loss": 61.4176, + "step": 9050 + }, + { + "epoch": 14.59011437496851, + "grad_norm": 0.7149770855903625, + "learning_rate": 0.0001, + "loss": 62.159, + "step": 9060 + }, + { + "epoch": 14.606237718546884, + "grad_norm": 0.6975772976875305, + "learning_rate": 0.0001, + "loss": 63.1857, + "step": 9070 + }, + { + "epoch": 14.622361062125258, + "grad_norm": 0.7835262417793274, + "learning_rate": 0.0001, + "loss": 61.6795, + "step": 9080 + }, + { + "epoch": 14.638484405703633, + "grad_norm": 5.790656566619873, + "learning_rate": 0.0001, + "loss": 62.7087, + "step": 9090 + }, + { + "epoch": 14.654607749282007, + "grad_norm": 1.0082288980484009, + "learning_rate": 0.0001, + "loss": 61.6808, + "step": 9100 + }, + { + "epoch": 14.670731092860382, + "grad_norm": 0.7758098840713501, + "learning_rate": 0.0001, + "loss": 61.0734, + "step": 9110 + }, + { + "epoch": 14.686854436438756, + "grad_norm": 0.7841722965240479, + "learning_rate": 0.0001, + "loss": 62.8882, + "step": 9120 + }, + { + "epoch": 14.70297778001713, + "grad_norm": 0.7695890069007874, + "learning_rate": 0.0001, + "loss": 60.7811, + "step": 9130 + }, + { + "epoch": 14.719101123595506, + "grad_norm": 0.7444761395454407, + "learning_rate": 0.0001, + "loss": 61.692, + "step": 9140 + }, + { + "epoch": 14.73522446717388, + "grad_norm": 0.7036821246147156, + "learning_rate": 0.0001, + "loss": 63.1698, + "step": 9150 + }, + { + "epoch": 14.751347810752256, + "grad_norm": 0.7891719937324524, + "learning_rate": 0.0001, + "loss": 63.8575, + "step": 9160 + }, + { + "epoch": 14.767471154330629, + "grad_norm": 0.6371109485626221, + "learning_rate": 0.0001, + "loss": 61.3358, + "step": 9170 + }, + { + "epoch": 14.783594497909004, + "grad_norm": 0.6551769375801086, + "learning_rate": 0.0001, + "loss": 62.2705, + "step": 9180 + }, + { + "epoch": 14.799717841487379, + "grad_norm": 0.6922310590744019, + "learning_rate": 0.0001, + "loss": 61.6442, + "step": 9190 + }, + { + "epoch": 14.815841185065754, + "grad_norm": 0.9615464210510254, + "learning_rate": 0.0001, + "loss": 60.932, + "step": 9200 + }, + { + "epoch": 14.831964528644127, + "grad_norm": 0.6367932558059692, + "learning_rate": 0.0001, + "loss": 61.3475, + "step": 9210 + }, + { + "epoch": 14.848087872222502, + "grad_norm": 0.5634505748748779, + "learning_rate": 0.0001, + "loss": 61.7562, + "step": 9220 + }, + { + "epoch": 14.864211215800877, + "grad_norm": 0.7021980285644531, + "learning_rate": 0.0001, + "loss": 62.0672, + "step": 9230 + }, + { + "epoch": 14.880334559379252, + "grad_norm": 0.8832064867019653, + "learning_rate": 0.0001, + "loss": 60.9882, + "step": 9240 + }, + { + "epoch": 14.896457902957625, + "grad_norm": 0.9043561220169067, + "learning_rate": 0.0001, + "loss": 63.0968, + "step": 9250 + }, + { + "epoch": 14.912581246536, + "grad_norm": 0.8958032131195068, + "learning_rate": 0.0001, + "loss": 61.5431, + "step": 9260 + }, + { + "epoch": 14.928704590114375, + "grad_norm": 0.7181615829467773, + "learning_rate": 0.0001, + "loss": 62.3626, + "step": 9270 + }, + { + "epoch": 14.94482793369275, + "grad_norm": 0.7541075348854065, + "learning_rate": 0.0001, + "loss": 61.7186, + "step": 9280 + }, + { + "epoch": 14.960951277271125, + "grad_norm": 1.5627169609069824, + "learning_rate": 0.0001, + "loss": 61.654, + "step": 9290 + }, + { + "epoch": 14.977074620849498, + "grad_norm": 0.7126879096031189, + "learning_rate": 0.0001, + "loss": 61.6806, + "step": 9300 + }, + { + "epoch": 14.993197964427873, + "grad_norm": 0.7397864460945129, + "learning_rate": 0.0001, + "loss": 61.2669, + "step": 9310 + }, + { + "epoch": 15.008061671789187, + "grad_norm": 0.7282766699790955, + "learning_rate": 0.0001, + "loss": 56.2289, + "step": 9320 + }, + { + "epoch": 15.024185015367562, + "grad_norm": 0.8346580862998962, + "learning_rate": 0.0001, + "loss": 61.4335, + "step": 9330 + }, + { + "epoch": 15.040308358945936, + "grad_norm": 0.9117271304130554, + "learning_rate": 0.0001, + "loss": 60.4636, + "step": 9340 + }, + { + "epoch": 15.05643170252431, + "grad_norm": 0.8483272194862366, + "learning_rate": 0.0001, + "loss": 61.3003, + "step": 9350 + }, + { + "epoch": 15.072555046102686, + "grad_norm": 1.1898932456970215, + "learning_rate": 0.0001, + "loss": 62.7169, + "step": 9360 + }, + { + "epoch": 15.08867838968106, + "grad_norm": 0.7258167862892151, + "learning_rate": 0.0001, + "loss": 63.5285, + "step": 9370 + }, + { + "epoch": 15.104801733259436, + "grad_norm": 0.7810865044593811, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 9380 + }, + { + "epoch": 15.120925076837809, + "grad_norm": 1.0359688997268677, + "learning_rate": 0.0001, + "loss": 62.5435, + "step": 9390 + }, + { + "epoch": 15.137048420416184, + "grad_norm": 0.7371546030044556, + "learning_rate": 0.0001, + "loss": 60.5583, + "step": 9400 + }, + { + "epoch": 15.153171763994559, + "grad_norm": 0.7578926086425781, + "learning_rate": 0.0001, + "loss": 61.2853, + "step": 9410 + }, + { + "epoch": 15.169295107572934, + "grad_norm": 0.6161713004112244, + "learning_rate": 0.0001, + "loss": 62.9725, + "step": 9420 + }, + { + "epoch": 15.185418451151307, + "grad_norm": 0.8325598835945129, + "learning_rate": 0.0001, + "loss": 62.857, + "step": 9430 + }, + { + "epoch": 15.201541794729682, + "grad_norm": 1.1438454389572144, + "learning_rate": 0.0001, + "loss": 61.8167, + "step": 9440 + }, + { + "epoch": 15.217665138308057, + "grad_norm": 1.1820837259292603, + "learning_rate": 0.0001, + "loss": 60.9899, + "step": 9450 + }, + { + "epoch": 15.233788481886432, + "grad_norm": 0.6057641506195068, + "learning_rate": 0.0001, + "loss": 62.5144, + "step": 9460 + }, + { + "epoch": 15.249911825464805, + "grad_norm": 0.6661948561668396, + "learning_rate": 0.0001, + "loss": 62.0835, + "step": 9470 + }, + { + "epoch": 15.26603516904318, + "grad_norm": 0.7623168230056763, + "learning_rate": 0.0001, + "loss": 61.9491, + "step": 9480 + }, + { + "epoch": 15.282158512621555, + "grad_norm": 0.6784029006958008, + "learning_rate": 0.0001, + "loss": 63.6006, + "step": 9490 + }, + { + "epoch": 15.29828185619993, + "grad_norm": 0.9433781504631042, + "learning_rate": 0.0001, + "loss": 61.6677, + "step": 9500 + }, + { + "epoch": 15.314405199778305, + "grad_norm": 0.6544079184532166, + "learning_rate": 0.0001, + "loss": 63.1439, + "step": 9510 + }, + { + "epoch": 15.330528543356678, + "grad_norm": 0.7979472279548645, + "learning_rate": 0.0001, + "loss": 62.719, + "step": 9520 + }, + { + "epoch": 15.346651886935053, + "grad_norm": 0.6641252040863037, + "learning_rate": 0.0001, + "loss": 63.2543, + "step": 9530 + }, + { + "epoch": 15.362775230513428, + "grad_norm": 0.7780525088310242, + "learning_rate": 0.0001, + "loss": 62.5659, + "step": 9540 + }, + { + "epoch": 15.378898574091803, + "grad_norm": 0.842349648475647, + "learning_rate": 0.0001, + "loss": 60.041, + "step": 9550 + }, + { + "epoch": 15.395021917670176, + "grad_norm": 0.8991017937660217, + "learning_rate": 0.0001, + "loss": 62.9075, + "step": 9560 + }, + { + "epoch": 15.411145261248551, + "grad_norm": 0.6734052300453186, + "learning_rate": 0.0001, + "loss": 60.8647, + "step": 9570 + }, + { + "epoch": 15.427268604826926, + "grad_norm": 0.7585113644599915, + "learning_rate": 0.0001, + "loss": 62.105, + "step": 9580 + }, + { + "epoch": 15.443391948405301, + "grad_norm": 0.7915840148925781, + "learning_rate": 0.0001, + "loss": 62.2741, + "step": 9590 + }, + { + "epoch": 15.459515291983674, + "grad_norm": 0.8676218390464783, + "learning_rate": 0.0001, + "loss": 62.4548, + "step": 9600 + }, + { + "epoch": 15.47563863556205, + "grad_norm": 0.7698650360107422, + "learning_rate": 0.0001, + "loss": 61.3412, + "step": 9610 + }, + { + "epoch": 15.491761979140424, + "grad_norm": 0.6600908637046814, + "learning_rate": 0.0001, + "loss": 61.1561, + "step": 9620 + }, + { + "epoch": 15.5078853227188, + "grad_norm": 0.5995001792907715, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 9630 + }, + { + "epoch": 15.524008666297174, + "grad_norm": 0.7258557677268982, + "learning_rate": 0.0001, + "loss": 62.4659, + "step": 9640 + }, + { + "epoch": 15.540132009875547, + "grad_norm": 0.6562618613243103, + "learning_rate": 0.0001, + "loss": 63.288, + "step": 9650 + }, + { + "epoch": 15.556255353453922, + "grad_norm": 0.9224995374679565, + "learning_rate": 0.0001, + "loss": 61.4861, + "step": 9660 + }, + { + "epoch": 15.572378697032297, + "grad_norm": 0.7053484916687012, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 9670 + }, + { + "epoch": 15.588502040610672, + "grad_norm": 1.095229148864746, + "learning_rate": 0.0001, + "loss": 62.1542, + "step": 9680 + }, + { + "epoch": 15.604625384189045, + "grad_norm": 0.7563759088516235, + "learning_rate": 0.0001, + "loss": 61.1644, + "step": 9690 + }, + { + "epoch": 15.62074872776742, + "grad_norm": 0.6508835554122925, + "learning_rate": 0.0001, + "loss": 60.1374, + "step": 9700 + }, + { + "epoch": 15.636872071345795, + "grad_norm": 0.664799153804779, + "learning_rate": 0.0001, + "loss": 60.2331, + "step": 9710 + }, + { + "epoch": 15.65299541492417, + "grad_norm": 0.6525771617889404, + "learning_rate": 0.0001, + "loss": 60.6142, + "step": 9720 + }, + { + "epoch": 15.669118758502545, + "grad_norm": 0.7395381331443787, + "learning_rate": 0.0001, + "loss": 63.5623, + "step": 9730 + }, + { + "epoch": 15.685242102080919, + "grad_norm": 0.7063724398612976, + "learning_rate": 0.0001, + "loss": 63.7133, + "step": 9740 + }, + { + "epoch": 15.701365445659293, + "grad_norm": 0.7918921113014221, + "learning_rate": 0.0001, + "loss": 62.5957, + "step": 9750 + }, + { + "epoch": 15.717488789237668, + "grad_norm": 0.8135486245155334, + "learning_rate": 0.0001, + "loss": 62.9146, + "step": 9760 + }, + { + "epoch": 15.733612132816043, + "grad_norm": 0.8422091007232666, + "learning_rate": 0.0001, + "loss": 61.1466, + "step": 9770 + }, + { + "epoch": 15.749735476394417, + "grad_norm": 0.6329447627067566, + "learning_rate": 0.0001, + "loss": 61.7247, + "step": 9780 + }, + { + "epoch": 15.765858819972792, + "grad_norm": 1.0627039670944214, + "learning_rate": 0.0001, + "loss": 60.6789, + "step": 9790 + }, + { + "epoch": 15.781982163551167, + "grad_norm": 0.7316949367523193, + "learning_rate": 0.0001, + "loss": 61.9896, + "step": 9800 + }, + { + "epoch": 15.798105507129542, + "grad_norm": 0.6968954205513, + "learning_rate": 0.0001, + "loss": 60.8571, + "step": 9810 + }, + { + "epoch": 15.814228850707915, + "grad_norm": 0.7226986289024353, + "learning_rate": 0.0001, + "loss": 62.2664, + "step": 9820 + }, + { + "epoch": 15.83035219428629, + "grad_norm": 0.6542951464653015, + "learning_rate": 0.0001, + "loss": 61.3316, + "step": 9830 + }, + { + "epoch": 15.846475537864665, + "grad_norm": 0.7662289142608643, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 9840 + }, + { + "epoch": 15.86259888144304, + "grad_norm": 0.7060043215751648, + "learning_rate": 0.0001, + "loss": 62.2647, + "step": 9850 + }, + { + "epoch": 15.878722225021415, + "grad_norm": 0.6948843002319336, + "learning_rate": 0.0001, + "loss": 62.8678, + "step": 9860 + }, + { + "epoch": 15.894845568599788, + "grad_norm": 0.7787327766418457, + "learning_rate": 0.0001, + "loss": 62.9154, + "step": 9870 + }, + { + "epoch": 15.910968912178163, + "grad_norm": 0.7077026963233948, + "learning_rate": 0.0001, + "loss": 62.9537, + "step": 9880 + }, + { + "epoch": 15.927092255756538, + "grad_norm": 0.9957109093666077, + "learning_rate": 0.0001, + "loss": 60.9461, + "step": 9890 + }, + { + "epoch": 15.943215599334913, + "grad_norm": 0.9767161011695862, + "learning_rate": 0.0001, + "loss": 62.5944, + "step": 9900 + }, + { + "epoch": 15.959338942913286, + "grad_norm": 0.6016744375228882, + "learning_rate": 0.0001, + "loss": 62.5889, + "step": 9910 + }, + { + "epoch": 15.975462286491661, + "grad_norm": 0.5464991331100464, + "learning_rate": 0.0001, + "loss": 61.3138, + "step": 9920 + }, + { + "epoch": 15.991585630070036, + "grad_norm": 0.8242359757423401, + "learning_rate": 0.0001, + "loss": 61.8091, + "step": 9930 + }, + { + "epoch": 16.00644933743135, + "grad_norm": 0.7031534314155579, + "learning_rate": 0.0001, + "loss": 57.4847, + "step": 9940 + }, + { + "epoch": 16.022572681009724, + "grad_norm": 0.8429487943649292, + "learning_rate": 0.0001, + "loss": 61.7286, + "step": 9950 + }, + { + "epoch": 16.0386960245881, + "grad_norm": 0.7200850248336792, + "learning_rate": 0.0001, + "loss": 62.6968, + "step": 9960 + }, + { + "epoch": 16.054819368166473, + "grad_norm": 0.7412559390068054, + "learning_rate": 0.0001, + "loss": 62.413, + "step": 9970 + }, + { + "epoch": 16.07094271174485, + "grad_norm": 0.8179170489311218, + "learning_rate": 0.0001, + "loss": 62.7044, + "step": 9980 + }, + { + "epoch": 16.087066055323223, + "grad_norm": 0.6939452886581421, + "learning_rate": 0.0001, + "loss": 61.9062, + "step": 9990 + }, + { + "epoch": 16.1031893989016, + "grad_norm": 0.7145045399665833, + "learning_rate": 0.0001, + "loss": 61.6201, + "step": 10000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 33, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.222970563029956e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ade09d5233289ba4bc5076af39b8652078582dee --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016a4e9b5d71a1947317d3801b00fe5d0604edbef1f981d7b6c92f219131717c +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-10000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/latest new file mode 100644 index 0000000000000000000000000000000000000000..b8f4d8b87537b41ea73a42841f5fb1704b16b234 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/latest @@ -0,0 +1 @@ +global_step12000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8f88289ae066bf6135872cc6765e2bb1d1e872d5 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58984fb78aae8957d66dc9cc0f3fe55967e6b37e7cdcf887ff3fc485ec829dfe +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..31dfe6a18e96434e6bc9db858b6af80650e8264d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3a5674a9d884cc3cd38a896e445fca8aa6359e044332f5da62d2bb72d549df9 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..06de5422e4a5bf3133374de7570a9c9d1722fa56 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45a959c1656c3bcf0c8cc2f787b606ca5ee19b637c426a8c887718fd4eb500c8 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..b17e33d1fd25d5510eeecd03e415a451b74c70b2 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6694a2cdc71630a31cc3b4dafe6a33f4c1dd07f40cde6efb7f5407bf52321654 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..ece38851fcc96c9e30827781205d7434408642e9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa261e576b901396adf396510ba75cb4ca880ab507f4ae3481527d11c16364d2 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..b0cbc9a31c77fe0232318aefaf2205b0b7c6bbbe --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a629c605a5b301118f2883e6422e2bfe3fef7bcd3662991aac4e0a91c6b371f +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..7a6f3e4b9399ef644cfcda3d0a11405c3e69cd65 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:485e678d54ca1fbf1e7a4789a11b809202bcd87d7b0b833a5a1e214faae7143e +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..a361bd635c5879133b93fee229156b82f6f1141e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1456097cc9ccb3d9506f6116d231b24c4393bb117e40d8078fd64594f0b7013 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..508eb84ec8226bc53f7dac5a51c36410149264ca --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebf7c252cddc91615944790ffab5233ea4ca6ad69c34b12a7c6dac4406712d58 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf54a839aaa564cb41c7b758f7e7846a4a7a40d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b65dd4f7c4351bc70e2ca42eddd7e6f1c0fdc194263d9f4061210a8ec7559c0e +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4d4c8df5a8617fc05e1dfabe85dc08d0025e65a2 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/trainer_state.json @@ -0,0 +1,8434 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 19.32407920592533, + "eval_steps": 2000, + "global_step": 12000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.016123343578374565, + "grad_norm": 48.11856460571289, + "learning_rate": 0.0001, + "loss": 299.1088, + "step": 10 + }, + { + "epoch": 0.03224668715674913, + "grad_norm": 9.07711410522461, + "learning_rate": 0.0001, + "loss": 234.0888, + "step": 20 + }, + { + "epoch": 0.0483700307351237, + "grad_norm": 3.5935330390930176, + "learning_rate": 0.0001, + "loss": 231.4619, + "step": 30 + }, + { + "epoch": 0.06449337431349826, + "grad_norm": 3.5784752368927, + "learning_rate": 0.0001, + "loss": 229.7082, + "step": 40 + }, + { + "epoch": 0.08061671789187283, + "grad_norm": 3.5757956504821777, + "learning_rate": 0.0001, + "loss": 230.1062, + "step": 50 + }, + { + "epoch": 0.0967400614702474, + "grad_norm": 3.8763978481292725, + "learning_rate": 0.0001, + "loss": 229.1914, + "step": 60 + }, + { + "epoch": 0.11286340504862195, + "grad_norm": 3.006988525390625, + "learning_rate": 0.0001, + "loss": 229.6482, + "step": 70 + }, + { + "epoch": 0.12898674862699652, + "grad_norm": 2.3527095317840576, + "learning_rate": 0.0001, + "loss": 228.1141, + "step": 80 + }, + { + "epoch": 0.14511009220537108, + "grad_norm": 5.353086471557617, + "learning_rate": 0.0001, + "loss": 228.1448, + "step": 90 + }, + { + "epoch": 0.16123343578374566, + "grad_norm": 2.251854419708252, + "learning_rate": 0.0001, + "loss": 228.3897, + "step": 100 + }, + { + "epoch": 0.17735677936212021, + "grad_norm": 1.5789915323257446, + "learning_rate": 0.0001, + "loss": 227.7662, + "step": 110 + }, + { + "epoch": 0.1934801229404948, + "grad_norm": 3.7646398544311523, + "learning_rate": 0.0001, + "loss": 223.6411, + "step": 120 + }, + { + "epoch": 0.20960346651886935, + "grad_norm": 3.444000720977783, + "learning_rate": 0.0001, + "loss": 221.2073, + "step": 130 + }, + { + "epoch": 0.2257268100972439, + "grad_norm": 2.1871185302734375, + "learning_rate": 0.0001, + "loss": 215.9944, + "step": 140 + }, + { + "epoch": 0.2418501536756185, + "grad_norm": 2.266244411468506, + "learning_rate": 0.0001, + "loss": 212.3066, + "step": 150 + }, + { + "epoch": 0.25797349725399304, + "grad_norm": 2.043832778930664, + "learning_rate": 0.0001, + "loss": 208.1343, + "step": 160 + }, + { + "epoch": 0.2740968408323676, + "grad_norm": 2.2641494274139404, + "learning_rate": 0.0001, + "loss": 203.1485, + "step": 170 + }, + { + "epoch": 0.29022018441074215, + "grad_norm": 2.9727790355682373, + "learning_rate": 0.0001, + "loss": 198.181, + "step": 180 + }, + { + "epoch": 0.30634352798911674, + "grad_norm": 2.751523494720459, + "learning_rate": 0.0001, + "loss": 194.8422, + "step": 190 + }, + { + "epoch": 0.3224668715674913, + "grad_norm": 1.9561562538146973, + "learning_rate": 0.0001, + "loss": 189.9401, + "step": 200 + }, + { + "epoch": 0.33859021514586585, + "grad_norm": 2.3264107704162598, + "learning_rate": 0.0001, + "loss": 186.1309, + "step": 210 + }, + { + "epoch": 0.35471355872424043, + "grad_norm": 5.212426662445068, + "learning_rate": 0.0001, + "loss": 182.2255, + "step": 220 + }, + { + "epoch": 0.370836902302615, + "grad_norm": 2.488032102584839, + "learning_rate": 0.0001, + "loss": 181.0103, + "step": 230 + }, + { + "epoch": 0.3869602458809896, + "grad_norm": 2.6173202991485596, + "learning_rate": 0.0001, + "loss": 177.6162, + "step": 240 + }, + { + "epoch": 0.4030835894593641, + "grad_norm": 2.6245291233062744, + "learning_rate": 0.0001, + "loss": 173.0431, + "step": 250 + }, + { + "epoch": 0.4192069330377387, + "grad_norm": 5.356006145477295, + "learning_rate": 0.0001, + "loss": 172.2371, + "step": 260 + }, + { + "epoch": 0.4353302766161133, + "grad_norm": 2.0453829765319824, + "learning_rate": 0.0001, + "loss": 169.7781, + "step": 270 + }, + { + "epoch": 0.4514536201944878, + "grad_norm": 1.7002843618392944, + "learning_rate": 0.0001, + "loss": 166.9564, + "step": 280 + }, + { + "epoch": 0.4675769637728624, + "grad_norm": 2.181410312652588, + "learning_rate": 0.0001, + "loss": 166.0094, + "step": 290 + }, + { + "epoch": 0.483700307351237, + "grad_norm": 2.2948265075683594, + "learning_rate": 0.0001, + "loss": 162.4388, + "step": 300 + }, + { + "epoch": 0.4998236509296115, + "grad_norm": 3.0272789001464844, + "learning_rate": 0.0001, + "loss": 160.3446, + "step": 310 + }, + { + "epoch": 0.5159469945079861, + "grad_norm": 1.948184847831726, + "learning_rate": 0.0001, + "loss": 158.488, + "step": 320 + }, + { + "epoch": 0.5320703380863606, + "grad_norm": 1.7981538772583008, + "learning_rate": 0.0001, + "loss": 156.2107, + "step": 330 + }, + { + "epoch": 0.5481936816647353, + "grad_norm": 2.049323797225952, + "learning_rate": 0.0001, + "loss": 155.5434, + "step": 340 + }, + { + "epoch": 0.5643170252431098, + "grad_norm": 1.9415442943572998, + "learning_rate": 0.0001, + "loss": 153.8682, + "step": 350 + }, + { + "epoch": 0.5804403688214843, + "grad_norm": 2.2168045043945312, + "learning_rate": 0.0001, + "loss": 150.4087, + "step": 360 + }, + { + "epoch": 0.596563712399859, + "grad_norm": 1.510750651359558, + "learning_rate": 0.0001, + "loss": 151.7655, + "step": 370 + }, + { + "epoch": 0.6126870559782335, + "grad_norm": 2.7469751834869385, + "learning_rate": 0.0001, + "loss": 147.4665, + "step": 380 + }, + { + "epoch": 0.628810399556608, + "grad_norm": 2.0436975955963135, + "learning_rate": 0.0001, + "loss": 147.4405, + "step": 390 + }, + { + "epoch": 0.6449337431349826, + "grad_norm": 1.8219181299209595, + "learning_rate": 0.0001, + "loss": 146.6332, + "step": 400 + }, + { + "epoch": 0.6610570867133572, + "grad_norm": 1.948431134223938, + "learning_rate": 0.0001, + "loss": 143.3298, + "step": 410 + }, + { + "epoch": 0.6771804302917317, + "grad_norm": 1.8887908458709717, + "learning_rate": 0.0001, + "loss": 140.5058, + "step": 420 + }, + { + "epoch": 0.6933037738701063, + "grad_norm": 1.8739320039749146, + "learning_rate": 0.0001, + "loss": 137.6334, + "step": 430 + }, + { + "epoch": 0.7094271174484809, + "grad_norm": 2.057689666748047, + "learning_rate": 0.0001, + "loss": 135.0343, + "step": 440 + }, + { + "epoch": 0.7255504610268555, + "grad_norm": 1.6926478147506714, + "learning_rate": 0.0001, + "loss": 132.8766, + "step": 450 + }, + { + "epoch": 0.74167380460523, + "grad_norm": 1.5039501190185547, + "learning_rate": 0.0001, + "loss": 131.6774, + "step": 460 + }, + { + "epoch": 0.7577971481836046, + "grad_norm": 1.6903547048568726, + "learning_rate": 0.0001, + "loss": 129.3116, + "step": 470 + }, + { + "epoch": 0.7739204917619792, + "grad_norm": 3.7062244415283203, + "learning_rate": 0.0001, + "loss": 127.9314, + "step": 480 + }, + { + "epoch": 0.7900438353403537, + "grad_norm": 2.0586934089660645, + "learning_rate": 0.0001, + "loss": 126.6398, + "step": 490 + }, + { + "epoch": 0.8061671789187282, + "grad_norm": 1.7598215341567993, + "learning_rate": 0.0001, + "loss": 125.1501, + "step": 500 + }, + { + "epoch": 0.8222905224971029, + "grad_norm": 1.5992189645767212, + "learning_rate": 0.0001, + "loss": 124.7039, + "step": 510 + }, + { + "epoch": 0.8384138660754774, + "grad_norm": 2.8735458850860596, + "learning_rate": 0.0001, + "loss": 122.1619, + "step": 520 + }, + { + "epoch": 0.8545372096538519, + "grad_norm": 1.867098093032837, + "learning_rate": 0.0001, + "loss": 122.4506, + "step": 530 + }, + { + "epoch": 0.8706605532322266, + "grad_norm": 2.2088418006896973, + "learning_rate": 0.0001, + "loss": 122.1047, + "step": 540 + }, + { + "epoch": 0.8867838968106011, + "grad_norm": 1.642657995223999, + "learning_rate": 0.0001, + "loss": 120.1141, + "step": 550 + }, + { + "epoch": 0.9029072403889756, + "grad_norm": 1.5567114353179932, + "learning_rate": 0.0001, + "loss": 118.532, + "step": 560 + }, + { + "epoch": 0.9190305839673503, + "grad_norm": 1.667843222618103, + "learning_rate": 0.0001, + "loss": 118.0734, + "step": 570 + }, + { + "epoch": 0.9351539275457248, + "grad_norm": 1.5304442644119263, + "learning_rate": 0.0001, + "loss": 114.4925, + "step": 580 + }, + { + "epoch": 0.9512772711240993, + "grad_norm": 1.6920255422592163, + "learning_rate": 0.0001, + "loss": 114.0729, + "step": 590 + }, + { + "epoch": 0.967400614702474, + "grad_norm": 1.6313318014144897, + "learning_rate": 0.0001, + "loss": 114.2879, + "step": 600 + }, + { + "epoch": 0.9835239582808485, + "grad_norm": 1.9033657312393188, + "learning_rate": 0.0001, + "loss": 110.5299, + "step": 610 + }, + { + "epoch": 0.999647301859223, + "grad_norm": 1.9010754823684692, + "learning_rate": 0.0001, + "loss": 109.1145, + "step": 620 + }, + { + "epoch": 1.0145110092205372, + "grad_norm": 3.5390989780426025, + "learning_rate": 0.0001, + "loss": 101.0258, + "step": 630 + }, + { + "epoch": 1.0306343527989117, + "grad_norm": 2.2925171852111816, + "learning_rate": 0.0001, + "loss": 107.3693, + "step": 640 + }, + { + "epoch": 1.0467576963772862, + "grad_norm": 2.329108238220215, + "learning_rate": 0.0001, + "loss": 104.3778, + "step": 650 + }, + { + "epoch": 1.0628810399556607, + "grad_norm": 1.7325661182403564, + "learning_rate": 0.0001, + "loss": 102.8409, + "step": 660 + }, + { + "epoch": 1.0790043835340353, + "grad_norm": 1.989586353302002, + "learning_rate": 0.0001, + "loss": 104.2577, + "step": 670 + }, + { + "epoch": 1.09512772711241, + "grad_norm": 1.900484323501587, + "learning_rate": 0.0001, + "loss": 102.3051, + "step": 680 + }, + { + "epoch": 1.1112510706907845, + "grad_norm": 2.015859842300415, + "learning_rate": 0.0001, + "loss": 100.893, + "step": 690 + }, + { + "epoch": 1.127374414269159, + "grad_norm": 1.7695508003234863, + "learning_rate": 0.0001, + "loss": 99.215, + "step": 700 + }, + { + "epoch": 1.1434977578475336, + "grad_norm": 2.025923490524292, + "learning_rate": 0.0001, + "loss": 98.8768, + "step": 710 + }, + { + "epoch": 1.1596211014259081, + "grad_norm": 2.946708917617798, + "learning_rate": 0.0001, + "loss": 96.5588, + "step": 720 + }, + { + "epoch": 1.1757444450042827, + "grad_norm": 2.4511258602142334, + "learning_rate": 0.0001, + "loss": 95.0545, + "step": 730 + }, + { + "epoch": 1.1918677885826574, + "grad_norm": 2.3094682693481445, + "learning_rate": 0.0001, + "loss": 97.004, + "step": 740 + }, + { + "epoch": 1.207991132161032, + "grad_norm": 2.1313347816467285, + "learning_rate": 0.0001, + "loss": 94.8819, + "step": 750 + }, + { + "epoch": 1.2241144757394065, + "grad_norm": 1.614182710647583, + "learning_rate": 0.0001, + "loss": 95.5339, + "step": 760 + }, + { + "epoch": 1.240237819317781, + "grad_norm": 2.029953956604004, + "learning_rate": 0.0001, + "loss": 93.7303, + "step": 770 + }, + { + "epoch": 1.2563611628961555, + "grad_norm": 1.7871663570404053, + "learning_rate": 0.0001, + "loss": 93.8354, + "step": 780 + }, + { + "epoch": 1.2724845064745303, + "grad_norm": 1.72455632686615, + "learning_rate": 0.0001, + "loss": 94.7348, + "step": 790 + }, + { + "epoch": 1.2886078500529048, + "grad_norm": 1.9776639938354492, + "learning_rate": 0.0001, + "loss": 93.5957, + "step": 800 + }, + { + "epoch": 1.3047311936312793, + "grad_norm": 1.7187533378601074, + "learning_rate": 0.0001, + "loss": 92.9583, + "step": 810 + }, + { + "epoch": 1.3208545372096538, + "grad_norm": 1.5949162244796753, + "learning_rate": 0.0001, + "loss": 93.3088, + "step": 820 + }, + { + "epoch": 1.3369778807880284, + "grad_norm": 1.4616206884384155, + "learning_rate": 0.0001, + "loss": 93.2125, + "step": 830 + }, + { + "epoch": 1.353101224366403, + "grad_norm": 2.832163095474243, + "learning_rate": 0.0001, + "loss": 90.3301, + "step": 840 + }, + { + "epoch": 1.3692245679447774, + "grad_norm": 2.174501419067383, + "learning_rate": 0.0001, + "loss": 91.6988, + "step": 850 + }, + { + "epoch": 1.3853479115231522, + "grad_norm": 1.2156339883804321, + "learning_rate": 0.0001, + "loss": 92.7419, + "step": 860 + }, + { + "epoch": 1.4014712551015267, + "grad_norm": 1.608261227607727, + "learning_rate": 0.0001, + "loss": 89.8255, + "step": 870 + }, + { + "epoch": 1.4175945986799012, + "grad_norm": 1.6671923398971558, + "learning_rate": 0.0001, + "loss": 90.9683, + "step": 880 + }, + { + "epoch": 1.4337179422582758, + "grad_norm": 1.2346433401107788, + "learning_rate": 0.0001, + "loss": 90.6092, + "step": 890 + }, + { + "epoch": 1.4498412858366505, + "grad_norm": 1.561085820198059, + "learning_rate": 0.0001, + "loss": 88.0842, + "step": 900 + }, + { + "epoch": 1.465964629415025, + "grad_norm": 1.5475201606750488, + "learning_rate": 0.0001, + "loss": 89.8786, + "step": 910 + }, + { + "epoch": 1.4820879729933996, + "grad_norm": 1.6044800281524658, + "learning_rate": 0.0001, + "loss": 88.0364, + "step": 920 + }, + { + "epoch": 1.498211316571774, + "grad_norm": 1.4081908464431763, + "learning_rate": 0.0001, + "loss": 88.8799, + "step": 930 + }, + { + "epoch": 1.5143346601501486, + "grad_norm": 2.341053009033203, + "learning_rate": 0.0001, + "loss": 88.2023, + "step": 940 + }, + { + "epoch": 1.5304580037285231, + "grad_norm": 1.3976799249649048, + "learning_rate": 0.0001, + "loss": 86.6493, + "step": 950 + }, + { + "epoch": 1.5465813473068977, + "grad_norm": 1.2485483884811401, + "learning_rate": 0.0001, + "loss": 88.9224, + "step": 960 + }, + { + "epoch": 1.5627046908852722, + "grad_norm": 1.6824235916137695, + "learning_rate": 0.0001, + "loss": 89.6484, + "step": 970 + }, + { + "epoch": 1.578828034463647, + "grad_norm": 1.64790940284729, + "learning_rate": 0.0001, + "loss": 87.2479, + "step": 980 + }, + { + "epoch": 1.5949513780420215, + "grad_norm": 1.2093443870544434, + "learning_rate": 0.0001, + "loss": 86.8241, + "step": 990 + }, + { + "epoch": 1.611074721620396, + "grad_norm": 1.5306812524795532, + "learning_rate": 0.0001, + "loss": 88.0006, + "step": 1000 + }, + { + "epoch": 1.6271980651987707, + "grad_norm": 1.2465763092041016, + "learning_rate": 0.0001, + "loss": 86.8143, + "step": 1010 + }, + { + "epoch": 1.6433214087771453, + "grad_norm": 1.5224069356918335, + "learning_rate": 0.0001, + "loss": 87.0102, + "step": 1020 + }, + { + "epoch": 1.6594447523555198, + "grad_norm": 1.312179684638977, + "learning_rate": 0.0001, + "loss": 88.7395, + "step": 1030 + }, + { + "epoch": 1.6755680959338943, + "grad_norm": 1.2343158721923828, + "learning_rate": 0.0001, + "loss": 87.2868, + "step": 1040 + }, + { + "epoch": 1.6916914395122689, + "grad_norm": 1.2778265476226807, + "learning_rate": 0.0001, + "loss": 87.043, + "step": 1050 + }, + { + "epoch": 1.7078147830906434, + "grad_norm": 1.2799627780914307, + "learning_rate": 0.0001, + "loss": 87.7843, + "step": 1060 + }, + { + "epoch": 1.723938126669018, + "grad_norm": 1.3688952922821045, + "learning_rate": 0.0001, + "loss": 85.8985, + "step": 1070 + }, + { + "epoch": 1.7400614702473924, + "grad_norm": 1.4317796230316162, + "learning_rate": 0.0001, + "loss": 85.5408, + "step": 1080 + }, + { + "epoch": 1.756184813825767, + "grad_norm": 1.164801001548767, + "learning_rate": 0.0001, + "loss": 85.8484, + "step": 1090 + }, + { + "epoch": 1.7723081574041417, + "grad_norm": 1.3929173946380615, + "learning_rate": 0.0001, + "loss": 86.9031, + "step": 1100 + }, + { + "epoch": 1.7884315009825162, + "grad_norm": 2.116119861602783, + "learning_rate": 0.0001, + "loss": 86.5343, + "step": 1110 + }, + { + "epoch": 1.8045548445608908, + "grad_norm": 1.4117276668548584, + "learning_rate": 0.0001, + "loss": 85.2678, + "step": 1120 + }, + { + "epoch": 1.8206781881392655, + "grad_norm": 1.3213504552841187, + "learning_rate": 0.0001, + "loss": 85.8935, + "step": 1130 + }, + { + "epoch": 1.83680153171764, + "grad_norm": 1.3589770793914795, + "learning_rate": 0.0001, + "loss": 85.032, + "step": 1140 + }, + { + "epoch": 1.8529248752960146, + "grad_norm": 1.3336007595062256, + "learning_rate": 0.0001, + "loss": 84.0993, + "step": 1150 + }, + { + "epoch": 1.869048218874389, + "grad_norm": 1.1081451177597046, + "learning_rate": 0.0001, + "loss": 84.2225, + "step": 1160 + }, + { + "epoch": 1.8851715624527636, + "grad_norm": 1.482038974761963, + "learning_rate": 0.0001, + "loss": 82.1439, + "step": 1170 + }, + { + "epoch": 1.9012949060311382, + "grad_norm": 1.1571478843688965, + "learning_rate": 0.0001, + "loss": 82.5045, + "step": 1180 + }, + { + "epoch": 1.9174182496095127, + "grad_norm": 1.2665468454360962, + "learning_rate": 0.0001, + "loss": 82.8761, + "step": 1190 + }, + { + "epoch": 1.9335415931878872, + "grad_norm": 1.0771509408950806, + "learning_rate": 0.0001, + "loss": 81.5162, + "step": 1200 + }, + { + "epoch": 1.949664936766262, + "grad_norm": 1.2992299795150757, + "learning_rate": 0.0001, + "loss": 82.4038, + "step": 1210 + }, + { + "epoch": 1.9657882803446365, + "grad_norm": 1.2946668863296509, + "learning_rate": 0.0001, + "loss": 83.4837, + "step": 1220 + }, + { + "epoch": 1.981911623923011, + "grad_norm": 1.3505686521530151, + "learning_rate": 0.0001, + "loss": 81.8993, + "step": 1230 + }, + { + "epoch": 1.9980349675013858, + "grad_norm": 1.1247715950012207, + "learning_rate": 0.0001, + "loss": 82.9006, + "step": 1240 + }, + { + "epoch": 2.0128986748627, + "grad_norm": 1.5652127265930176, + "learning_rate": 0.0001, + "loss": 75.8347, + "step": 1250 + }, + { + "epoch": 2.0290220184410743, + "grad_norm": 7.311964988708496, + "learning_rate": 0.0001, + "loss": 80.5435, + "step": 1260 + }, + { + "epoch": 2.045145362019449, + "grad_norm": 1.2229893207550049, + "learning_rate": 0.0001, + "loss": 83.3277, + "step": 1270 + }, + { + "epoch": 2.0612687055978234, + "grad_norm": 1.1387693881988525, + "learning_rate": 0.0001, + "loss": 81.3252, + "step": 1280 + }, + { + "epoch": 2.077392049176198, + "grad_norm": 1.307201623916626, + "learning_rate": 0.0001, + "loss": 82.3311, + "step": 1290 + }, + { + "epoch": 2.0935153927545724, + "grad_norm": 1.1502896547317505, + "learning_rate": 0.0001, + "loss": 81.1682, + "step": 1300 + }, + { + "epoch": 2.109638736332947, + "grad_norm": 0.9291583299636841, + "learning_rate": 0.0001, + "loss": 81.7017, + "step": 1310 + }, + { + "epoch": 2.1257620799113215, + "grad_norm": 1.0334666967391968, + "learning_rate": 0.0001, + "loss": 81.718, + "step": 1320 + }, + { + "epoch": 2.141885423489696, + "grad_norm": 1.1099056005477905, + "learning_rate": 0.0001, + "loss": 81.2232, + "step": 1330 + }, + { + "epoch": 2.1580087670680705, + "grad_norm": 1.289478063583374, + "learning_rate": 0.0001, + "loss": 81.8736, + "step": 1340 + }, + { + "epoch": 2.174132110646445, + "grad_norm": 1.1634794473648071, + "learning_rate": 0.0001, + "loss": 79.9434, + "step": 1350 + }, + { + "epoch": 2.19025545422482, + "grad_norm": 0.9563440680503845, + "learning_rate": 0.0001, + "loss": 80.628, + "step": 1360 + }, + { + "epoch": 2.2063787978031946, + "grad_norm": 1.1268844604492188, + "learning_rate": 0.0001, + "loss": 82.1389, + "step": 1370 + }, + { + "epoch": 2.222502141381569, + "grad_norm": 1.1050443649291992, + "learning_rate": 0.0001, + "loss": 78.9023, + "step": 1380 + }, + { + "epoch": 2.2386254849599436, + "grad_norm": 1.1936142444610596, + "learning_rate": 0.0001, + "loss": 81.2118, + "step": 1390 + }, + { + "epoch": 2.254748828538318, + "grad_norm": 1.7825465202331543, + "learning_rate": 0.0001, + "loss": 80.8804, + "step": 1400 + }, + { + "epoch": 2.2708721721166927, + "grad_norm": 2.437251329421997, + "learning_rate": 0.0001, + "loss": 79.3807, + "step": 1410 + }, + { + "epoch": 2.286995515695067, + "grad_norm": 1.0604854822158813, + "learning_rate": 0.0001, + "loss": 79.5881, + "step": 1420 + }, + { + "epoch": 2.3031188592734417, + "grad_norm": 1.2285529375076294, + "learning_rate": 0.0001, + "loss": 79.5517, + "step": 1430 + }, + { + "epoch": 2.3192422028518163, + "grad_norm": 1.1916660070419312, + "learning_rate": 0.0001, + "loss": 80.5887, + "step": 1440 + }, + { + "epoch": 2.335365546430191, + "grad_norm": 1.224304437637329, + "learning_rate": 0.0001, + "loss": 79.8109, + "step": 1450 + }, + { + "epoch": 2.3514888900085653, + "grad_norm": 0.9228448867797852, + "learning_rate": 0.0001, + "loss": 79.2787, + "step": 1460 + }, + { + "epoch": 2.3676122335869403, + "grad_norm": 1.2085226774215698, + "learning_rate": 0.0001, + "loss": 78.4983, + "step": 1470 + }, + { + "epoch": 2.383735577165315, + "grad_norm": 2.424083948135376, + "learning_rate": 0.0001, + "loss": 79.8657, + "step": 1480 + }, + { + "epoch": 2.3998589207436893, + "grad_norm": 1.0277849435806274, + "learning_rate": 0.0001, + "loss": 80.6915, + "step": 1490 + }, + { + "epoch": 2.415982264322064, + "grad_norm": 1.0226075649261475, + "learning_rate": 0.0001, + "loss": 78.9751, + "step": 1500 + }, + { + "epoch": 2.4321056079004384, + "grad_norm": 0.8860388398170471, + "learning_rate": 0.0001, + "loss": 78.3915, + "step": 1510 + }, + { + "epoch": 2.448228951478813, + "grad_norm": 1.265637755393982, + "learning_rate": 0.0001, + "loss": 78.8737, + "step": 1520 + }, + { + "epoch": 2.4643522950571874, + "grad_norm": 1.2518186569213867, + "learning_rate": 0.0001, + "loss": 76.5169, + "step": 1530 + }, + { + "epoch": 2.480475638635562, + "grad_norm": 1.074277400970459, + "learning_rate": 0.0001, + "loss": 77.9156, + "step": 1540 + }, + { + "epoch": 2.4965989822139365, + "grad_norm": 1.0666711330413818, + "learning_rate": 0.0001, + "loss": 78.4103, + "step": 1550 + }, + { + "epoch": 2.512722325792311, + "grad_norm": 0.9492248892784119, + "learning_rate": 0.0001, + "loss": 76.7393, + "step": 1560 + }, + { + "epoch": 2.5288456693706856, + "grad_norm": 1.127238154411316, + "learning_rate": 0.0001, + "loss": 79.729, + "step": 1570 + }, + { + "epoch": 2.5449690129490605, + "grad_norm": 1.1520992517471313, + "learning_rate": 0.0001, + "loss": 78.5618, + "step": 1580 + }, + { + "epoch": 2.5610923565274346, + "grad_norm": 0.8632035255432129, + "learning_rate": 0.0001, + "loss": 79.032, + "step": 1590 + }, + { + "epoch": 2.5772157001058096, + "grad_norm": 1.0506329536437988, + "learning_rate": 0.0001, + "loss": 77.6204, + "step": 1600 + }, + { + "epoch": 2.593339043684184, + "grad_norm": 1.019895315170288, + "learning_rate": 0.0001, + "loss": 79.2634, + "step": 1610 + }, + { + "epoch": 2.6094623872625586, + "grad_norm": 1.1685571670532227, + "learning_rate": 0.0001, + "loss": 78.8371, + "step": 1620 + }, + { + "epoch": 2.625585730840933, + "grad_norm": 1.147546648979187, + "learning_rate": 0.0001, + "loss": 78.2113, + "step": 1630 + }, + { + "epoch": 2.6417090744193077, + "grad_norm": 1.7336841821670532, + "learning_rate": 0.0001, + "loss": 78.7866, + "step": 1640 + }, + { + "epoch": 2.657832417997682, + "grad_norm": 1.2150614261627197, + "learning_rate": 0.0001, + "loss": 78.5621, + "step": 1650 + }, + { + "epoch": 2.6739557615760567, + "grad_norm": 1.2257503271102905, + "learning_rate": 0.0001, + "loss": 77.5835, + "step": 1660 + }, + { + "epoch": 2.6900791051544313, + "grad_norm": 1.0375311374664307, + "learning_rate": 0.0001, + "loss": 77.7509, + "step": 1670 + }, + { + "epoch": 2.706202448732806, + "grad_norm": 1.2551004886627197, + "learning_rate": 0.0001, + "loss": 77.8615, + "step": 1680 + }, + { + "epoch": 2.7223257923111808, + "grad_norm": 1.0464223623275757, + "learning_rate": 0.0001, + "loss": 77.2837, + "step": 1690 + }, + { + "epoch": 2.738449135889555, + "grad_norm": 1.0254806280136108, + "learning_rate": 0.0001, + "loss": 76.9616, + "step": 1700 + }, + { + "epoch": 2.75457247946793, + "grad_norm": 1.1665682792663574, + "learning_rate": 0.0001, + "loss": 77.9108, + "step": 1710 + }, + { + "epoch": 2.7706958230463044, + "grad_norm": 1.1349453926086426, + "learning_rate": 0.0001, + "loss": 78.7636, + "step": 1720 + }, + { + "epoch": 2.786819166624679, + "grad_norm": 1.4961057901382446, + "learning_rate": 0.0001, + "loss": 76.3862, + "step": 1730 + }, + { + "epoch": 2.8029425102030534, + "grad_norm": 1.10721755027771, + "learning_rate": 0.0001, + "loss": 77.4105, + "step": 1740 + }, + { + "epoch": 2.819065853781428, + "grad_norm": 0.9878576397895813, + "learning_rate": 0.0001, + "loss": 78.4855, + "step": 1750 + }, + { + "epoch": 2.8351891973598025, + "grad_norm": 1.1722687482833862, + "learning_rate": 0.0001, + "loss": 75.1589, + "step": 1760 + }, + { + "epoch": 2.851312540938177, + "grad_norm": 1.6624362468719482, + "learning_rate": 0.0001, + "loss": 77.1647, + "step": 1770 + }, + { + "epoch": 2.8674358845165515, + "grad_norm": 0.9542295932769775, + "learning_rate": 0.0001, + "loss": 77.1656, + "step": 1780 + }, + { + "epoch": 2.883559228094926, + "grad_norm": 2.328542947769165, + "learning_rate": 0.0001, + "loss": 76.4686, + "step": 1790 + }, + { + "epoch": 2.899682571673301, + "grad_norm": 1.2532029151916504, + "learning_rate": 0.0001, + "loss": 76.1727, + "step": 1800 + }, + { + "epoch": 2.915805915251675, + "grad_norm": 0.9604716897010803, + "learning_rate": 0.0001, + "loss": 74.3132, + "step": 1810 + }, + { + "epoch": 2.93192925883005, + "grad_norm": 0.9182595610618591, + "learning_rate": 0.0001, + "loss": 77.8689, + "step": 1820 + }, + { + "epoch": 2.9480526024084246, + "grad_norm": 0.9623042941093445, + "learning_rate": 0.0001, + "loss": 77.7531, + "step": 1830 + }, + { + "epoch": 2.964175945986799, + "grad_norm": 1.0500119924545288, + "learning_rate": 0.0001, + "loss": 77.6007, + "step": 1840 + }, + { + "epoch": 2.9802992895651736, + "grad_norm": 1.0874103307724, + "learning_rate": 0.0001, + "loss": 77.042, + "step": 1850 + }, + { + "epoch": 2.996422633143548, + "grad_norm": 0.9955475330352783, + "learning_rate": 0.0001, + "loss": 75.468, + "step": 1860 + }, + { + "epoch": 3.011286340504862, + "grad_norm": 1.4281760454177856, + "learning_rate": 0.0001, + "loss": 69.5972, + "step": 1870 + }, + { + "epoch": 3.0274096840832367, + "grad_norm": 1.0292123556137085, + "learning_rate": 0.0001, + "loss": 76.0186, + "step": 1880 + }, + { + "epoch": 3.0435330276616113, + "grad_norm": 0.9535816311836243, + "learning_rate": 0.0001, + "loss": 76.0979, + "step": 1890 + }, + { + "epoch": 3.059656371239986, + "grad_norm": 0.9099674820899963, + "learning_rate": 0.0001, + "loss": 75.9641, + "step": 1900 + }, + { + "epoch": 3.0757797148183603, + "grad_norm": 0.9568380117416382, + "learning_rate": 0.0001, + "loss": 75.0656, + "step": 1910 + }, + { + "epoch": 3.091903058396735, + "grad_norm": 0.9701060652732849, + "learning_rate": 0.0001, + "loss": 75.1422, + "step": 1920 + }, + { + "epoch": 3.1080264019751094, + "grad_norm": 1.3879098892211914, + "learning_rate": 0.0001, + "loss": 76.0731, + "step": 1930 + }, + { + "epoch": 3.1241497455534843, + "grad_norm": 0.9782084226608276, + "learning_rate": 0.0001, + "loss": 75.9666, + "step": 1940 + }, + { + "epoch": 3.140273089131859, + "grad_norm": 1.0856964588165283, + "learning_rate": 0.0001, + "loss": 75.0217, + "step": 1950 + }, + { + "epoch": 3.1563964327102334, + "grad_norm": 0.8506444692611694, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 1960 + }, + { + "epoch": 3.172519776288608, + "grad_norm": 1.044980525970459, + "learning_rate": 0.0001, + "loss": 75.3471, + "step": 1970 + }, + { + "epoch": 3.1886431198669825, + "grad_norm": 0.8432291746139526, + "learning_rate": 0.0001, + "loss": 75.3563, + "step": 1980 + }, + { + "epoch": 3.204766463445357, + "grad_norm": 0.9651834964752197, + "learning_rate": 0.0001, + "loss": 76.3991, + "step": 1990 + }, + { + "epoch": 3.2208898070237315, + "grad_norm": 0.957943320274353, + "learning_rate": 0.0001, + "loss": 77.2613, + "step": 2000 + }, + { + "epoch": 3.237013150602106, + "grad_norm": 0.9495745301246643, + "learning_rate": 0.0001, + "loss": 73.3801, + "step": 2010 + }, + { + "epoch": 3.2531364941804806, + "grad_norm": 1.182983160018921, + "learning_rate": 0.0001, + "loss": 74.5695, + "step": 2020 + }, + { + "epoch": 3.269259837758855, + "grad_norm": 0.828452467918396, + "learning_rate": 0.0001, + "loss": 75.4313, + "step": 2030 + }, + { + "epoch": 3.2853831813372296, + "grad_norm": 0.8340104818344116, + "learning_rate": 0.0001, + "loss": 75.1086, + "step": 2040 + }, + { + "epoch": 3.301506524915604, + "grad_norm": 1.1700762510299683, + "learning_rate": 0.0001, + "loss": 75.7496, + "step": 2050 + }, + { + "epoch": 3.317629868493979, + "grad_norm": 0.8709110617637634, + "learning_rate": 0.0001, + "loss": 73.8403, + "step": 2060 + }, + { + "epoch": 3.3337532120723536, + "grad_norm": 0.8928954601287842, + "learning_rate": 0.0001, + "loss": 75.6134, + "step": 2070 + }, + { + "epoch": 3.349876555650728, + "grad_norm": 0.9787353277206421, + "learning_rate": 0.0001, + "loss": 75.8116, + "step": 2080 + }, + { + "epoch": 3.3659998992291027, + "grad_norm": 0.9515123963356018, + "learning_rate": 0.0001, + "loss": 75.5024, + "step": 2090 + }, + { + "epoch": 3.3821232428074772, + "grad_norm": 0.9529262185096741, + "learning_rate": 0.0001, + "loss": 74.1553, + "step": 2100 + }, + { + "epoch": 3.3982465863858518, + "grad_norm": 0.8934710621833801, + "learning_rate": 0.0001, + "loss": 74.1501, + "step": 2110 + }, + { + "epoch": 3.4143699299642263, + "grad_norm": 1.089447021484375, + "learning_rate": 0.0001, + "loss": 74.9123, + "step": 2120 + }, + { + "epoch": 3.430493273542601, + "grad_norm": 1.2569622993469238, + "learning_rate": 0.0001, + "loss": 74.3962, + "step": 2130 + }, + { + "epoch": 3.4466166171209753, + "grad_norm": 1.020094394683838, + "learning_rate": 0.0001, + "loss": 74.6617, + "step": 2140 + }, + { + "epoch": 3.46273996069935, + "grad_norm": 1.1579667329788208, + "learning_rate": 0.0001, + "loss": 74.2745, + "step": 2150 + }, + { + "epoch": 3.4788633042777244, + "grad_norm": 1.448935627937317, + "learning_rate": 0.0001, + "loss": 71.5285, + "step": 2160 + }, + { + "epoch": 3.4949866478560994, + "grad_norm": 1.0030494928359985, + "learning_rate": 0.0001, + "loss": 74.6666, + "step": 2170 + }, + { + "epoch": 3.511109991434474, + "grad_norm": 0.9010888934135437, + "learning_rate": 0.0001, + "loss": 74.9355, + "step": 2180 + }, + { + "epoch": 3.5272333350128484, + "grad_norm": 1.1595124006271362, + "learning_rate": 0.0001, + "loss": 74.5049, + "step": 2190 + }, + { + "epoch": 3.543356678591223, + "grad_norm": 1.066405177116394, + "learning_rate": 0.0001, + "loss": 74.6336, + "step": 2200 + }, + { + "epoch": 3.5594800221695975, + "grad_norm": 1.2301596403121948, + "learning_rate": 0.0001, + "loss": 74.6664, + "step": 2210 + }, + { + "epoch": 3.575603365747972, + "grad_norm": 1.0073511600494385, + "learning_rate": 0.0001, + "loss": 73.1405, + "step": 2220 + }, + { + "epoch": 3.5917267093263465, + "grad_norm": 0.9913995862007141, + "learning_rate": 0.0001, + "loss": 75.7443, + "step": 2230 + }, + { + "epoch": 3.607850052904721, + "grad_norm": 0.9555648565292358, + "learning_rate": 0.0001, + "loss": 74.8036, + "step": 2240 + }, + { + "epoch": 3.6239733964830956, + "grad_norm": 0.9350649118423462, + "learning_rate": 0.0001, + "loss": 73.5783, + "step": 2250 + }, + { + "epoch": 3.64009674006147, + "grad_norm": 0.935840368270874, + "learning_rate": 0.0001, + "loss": 75.1694, + "step": 2260 + }, + { + "epoch": 3.6562200836398446, + "grad_norm": 0.9303803443908691, + "learning_rate": 0.0001, + "loss": 72.8282, + "step": 2270 + }, + { + "epoch": 3.6723434272182196, + "grad_norm": 0.937164843082428, + "learning_rate": 0.0001, + "loss": 70.3305, + "step": 2280 + }, + { + "epoch": 3.6884667707965937, + "grad_norm": 1.027329444885254, + "learning_rate": 0.0001, + "loss": 73.748, + "step": 2290 + }, + { + "epoch": 3.7045901143749687, + "grad_norm": 1.0915215015411377, + "learning_rate": 0.0001, + "loss": 72.2265, + "step": 2300 + }, + { + "epoch": 3.720713457953343, + "grad_norm": 1.1031261682510376, + "learning_rate": 0.0001, + "loss": 71.2795, + "step": 2310 + }, + { + "epoch": 3.7368368015317177, + "grad_norm": 1.1466140747070312, + "learning_rate": 0.0001, + "loss": 72.0116, + "step": 2320 + }, + { + "epoch": 3.7529601451100922, + "grad_norm": 1.0441662073135376, + "learning_rate": 0.0001, + "loss": 73.3735, + "step": 2330 + }, + { + "epoch": 3.7690834886884668, + "grad_norm": 0.8610432744026184, + "learning_rate": 0.0001, + "loss": 73.3433, + "step": 2340 + }, + { + "epoch": 3.7852068322668413, + "grad_norm": 0.885720431804657, + "learning_rate": 0.0001, + "loss": 73.575, + "step": 2350 + }, + { + "epoch": 3.801330175845216, + "grad_norm": 1.0023950338363647, + "learning_rate": 0.0001, + "loss": 73.4166, + "step": 2360 + }, + { + "epoch": 3.8174535194235903, + "grad_norm": 1.3820304870605469, + "learning_rate": 0.0001, + "loss": 72.9599, + "step": 2370 + }, + { + "epoch": 3.833576863001965, + "grad_norm": 0.9645710587501526, + "learning_rate": 0.0001, + "loss": 72.3922, + "step": 2380 + }, + { + "epoch": 3.84970020658034, + "grad_norm": 1.273369312286377, + "learning_rate": 0.0001, + "loss": 72.0158, + "step": 2390 + }, + { + "epoch": 3.865823550158714, + "grad_norm": 0.859863817691803, + "learning_rate": 0.0001, + "loss": 74.6642, + "step": 2400 + }, + { + "epoch": 3.881946893737089, + "grad_norm": 0.9396682381629944, + "learning_rate": 0.0001, + "loss": 71.9003, + "step": 2410 + }, + { + "epoch": 3.8980702373154634, + "grad_norm": 0.9976151585578918, + "learning_rate": 0.0001, + "loss": 73.0716, + "step": 2420 + }, + { + "epoch": 3.914193580893838, + "grad_norm": 1.0730767250061035, + "learning_rate": 0.0001, + "loss": 72.7847, + "step": 2430 + }, + { + "epoch": 3.9303169244722125, + "grad_norm": 1.002018690109253, + "learning_rate": 0.0001, + "loss": 71.7309, + "step": 2440 + }, + { + "epoch": 3.946440268050587, + "grad_norm": 0.9102753400802612, + "learning_rate": 0.0001, + "loss": 71.616, + "step": 2450 + }, + { + "epoch": 3.9625636116289615, + "grad_norm": 0.9827467799186707, + "learning_rate": 0.0001, + "loss": 72.22, + "step": 2460 + }, + { + "epoch": 3.978686955207336, + "grad_norm": 1.2781893014907837, + "learning_rate": 0.0001, + "loss": 73.2048, + "step": 2470 + }, + { + "epoch": 3.9948102987857106, + "grad_norm": 0.9042410850524902, + "learning_rate": 0.0001, + "loss": 72.9843, + "step": 2480 + }, + { + "epoch": 4.009674006147025, + "grad_norm": 0.8662117123603821, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 2490 + }, + { + "epoch": 4.0257973497254, + "grad_norm": 0.9448756575584412, + "learning_rate": 0.0001, + "loss": 73.36, + "step": 2500 + }, + { + "epoch": 4.041920693303774, + "grad_norm": 0.9054566621780396, + "learning_rate": 0.0001, + "loss": 73.94, + "step": 2510 + }, + { + "epoch": 4.058044036882149, + "grad_norm": 0.957398533821106, + "learning_rate": 0.0001, + "loss": 73.2019, + "step": 2520 + }, + { + "epoch": 4.074167380460523, + "grad_norm": 0.9444408416748047, + "learning_rate": 0.0001, + "loss": 70.6553, + "step": 2530 + }, + { + "epoch": 4.090290724038898, + "grad_norm": 0.8716063499450684, + "learning_rate": 0.0001, + "loss": 70.1048, + "step": 2540 + }, + { + "epoch": 4.106414067617272, + "grad_norm": 0.949249267578125, + "learning_rate": 0.0001, + "loss": 72.4453, + "step": 2550 + }, + { + "epoch": 4.122537411195647, + "grad_norm": 0.9843282699584961, + "learning_rate": 0.0001, + "loss": 72.1725, + "step": 2560 + }, + { + "epoch": 4.138660754774021, + "grad_norm": 0.8977480530738831, + "learning_rate": 0.0001, + "loss": 72.5272, + "step": 2570 + }, + { + "epoch": 4.154784098352396, + "grad_norm": 0.9957827925682068, + "learning_rate": 0.0001, + "loss": 72.289, + "step": 2580 + }, + { + "epoch": 4.170907441930771, + "grad_norm": 1.1358791589736938, + "learning_rate": 0.0001, + "loss": 72.1949, + "step": 2590 + }, + { + "epoch": 4.187030785509145, + "grad_norm": 0.9953081607818604, + "learning_rate": 0.0001, + "loss": 72.6004, + "step": 2600 + }, + { + "epoch": 4.20315412908752, + "grad_norm": 1.019061803817749, + "learning_rate": 0.0001, + "loss": 72.3772, + "step": 2610 + }, + { + "epoch": 4.219277472665894, + "grad_norm": 0.9062562584877014, + "learning_rate": 0.0001, + "loss": 72.0566, + "step": 2620 + }, + { + "epoch": 4.235400816244269, + "grad_norm": 1.0877611637115479, + "learning_rate": 0.0001, + "loss": 72.854, + "step": 2630 + }, + { + "epoch": 4.251524159822643, + "grad_norm": 1.2419648170471191, + "learning_rate": 0.0001, + "loss": 72.6498, + "step": 2640 + }, + { + "epoch": 4.267647503401018, + "grad_norm": 0.9507225751876831, + "learning_rate": 0.0001, + "loss": 68.9583, + "step": 2650 + }, + { + "epoch": 4.283770846979392, + "grad_norm": 1.121191382408142, + "learning_rate": 0.0001, + "loss": 72.8464, + "step": 2660 + }, + { + "epoch": 4.299894190557767, + "grad_norm": 1.0543361902236938, + "learning_rate": 0.0001, + "loss": 71.9882, + "step": 2670 + }, + { + "epoch": 4.316017534136141, + "grad_norm": 0.8622955083847046, + "learning_rate": 0.0001, + "loss": 71.7373, + "step": 2680 + }, + { + "epoch": 4.332140877714516, + "grad_norm": 0.911253809928894, + "learning_rate": 0.0001, + "loss": 71.1968, + "step": 2690 + }, + { + "epoch": 4.34826422129289, + "grad_norm": 4.405472755432129, + "learning_rate": 0.0001, + "loss": 70.4128, + "step": 2700 + }, + { + "epoch": 4.364387564871265, + "grad_norm": 0.9142985939979553, + "learning_rate": 0.0001, + "loss": 69.7548, + "step": 2710 + }, + { + "epoch": 4.38051090844964, + "grad_norm": 1.1022522449493408, + "learning_rate": 0.0001, + "loss": 71.8282, + "step": 2720 + }, + { + "epoch": 4.396634252028014, + "grad_norm": 1.018386721611023, + "learning_rate": 0.0001, + "loss": 71.0853, + "step": 2730 + }, + { + "epoch": 4.412757595606389, + "grad_norm": 1.0483211278915405, + "learning_rate": 0.0001, + "loss": 72.1478, + "step": 2740 + }, + { + "epoch": 4.428880939184763, + "grad_norm": 1.026861548423767, + "learning_rate": 0.0001, + "loss": 71.8947, + "step": 2750 + }, + { + "epoch": 4.445004282763138, + "grad_norm": 1.1858552694320679, + "learning_rate": 0.0001, + "loss": 71.7016, + "step": 2760 + }, + { + "epoch": 4.461127626341512, + "grad_norm": 1.1212221384048462, + "learning_rate": 0.0001, + "loss": 71.4454, + "step": 2770 + }, + { + "epoch": 4.477250969919887, + "grad_norm": 0.8062102794647217, + "learning_rate": 0.0001, + "loss": 73.2672, + "step": 2780 + }, + { + "epoch": 4.493374313498261, + "grad_norm": 1.104483962059021, + "learning_rate": 0.0001, + "loss": 70.548, + "step": 2790 + }, + { + "epoch": 4.509497657076636, + "grad_norm": 0.942057728767395, + "learning_rate": 0.0001, + "loss": 72.3584, + "step": 2800 + }, + { + "epoch": 4.525621000655011, + "grad_norm": 0.9028333425521851, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 2810 + }, + { + "epoch": 4.541744344233385, + "grad_norm": 0.9096795916557312, + "learning_rate": 0.0001, + "loss": 70.6596, + "step": 2820 + }, + { + "epoch": 4.55786768781176, + "grad_norm": 0.9038829207420349, + "learning_rate": 0.0001, + "loss": 72.6148, + "step": 2830 + }, + { + "epoch": 4.573991031390134, + "grad_norm": 1.8511613607406616, + "learning_rate": 0.0001, + "loss": 71.7205, + "step": 2840 + }, + { + "epoch": 4.590114374968509, + "grad_norm": 1.3492343425750732, + "learning_rate": 0.0001, + "loss": 72.1492, + "step": 2850 + }, + { + "epoch": 4.6062377185468835, + "grad_norm": 1.1469528675079346, + "learning_rate": 0.0001, + "loss": 70.5892, + "step": 2860 + }, + { + "epoch": 4.622361062125258, + "grad_norm": 1.1374379396438599, + "learning_rate": 0.0001, + "loss": 70.061, + "step": 2870 + }, + { + "epoch": 4.6384844057036325, + "grad_norm": 0.9597565531730652, + "learning_rate": 0.0001, + "loss": 71.0869, + "step": 2880 + }, + { + "epoch": 4.6546077492820075, + "grad_norm": 0.8356521129608154, + "learning_rate": 0.0001, + "loss": 72.1891, + "step": 2890 + }, + { + "epoch": 4.670731092860382, + "grad_norm": 0.8882647752761841, + "learning_rate": 0.0001, + "loss": 71.3845, + "step": 2900 + }, + { + "epoch": 4.6868544364387565, + "grad_norm": 1.1904003620147705, + "learning_rate": 0.0001, + "loss": 72.3086, + "step": 2910 + }, + { + "epoch": 4.702977780017131, + "grad_norm": 0.8010085821151733, + "learning_rate": 0.0001, + "loss": 71.3051, + "step": 2920 + }, + { + "epoch": 4.719101123595506, + "grad_norm": 0.7572135329246521, + "learning_rate": 0.0001, + "loss": 72.5605, + "step": 2930 + }, + { + "epoch": 4.735224467173881, + "grad_norm": 1.0866942405700684, + "learning_rate": 0.0001, + "loss": 70.0935, + "step": 2940 + }, + { + "epoch": 4.751347810752255, + "grad_norm": 0.9358721971511841, + "learning_rate": 0.0001, + "loss": 71.0391, + "step": 2950 + }, + { + "epoch": 4.76747115433063, + "grad_norm": 1.0276161432266235, + "learning_rate": 0.0001, + "loss": 69.8462, + "step": 2960 + }, + { + "epoch": 4.783594497909004, + "grad_norm": 0.9573063850402832, + "learning_rate": 0.0001, + "loss": 69.7089, + "step": 2970 + }, + { + "epoch": 4.799717841487379, + "grad_norm": 1.0638892650604248, + "learning_rate": 0.0001, + "loss": 71.8376, + "step": 2980 + }, + { + "epoch": 4.815841185065753, + "grad_norm": 0.9894681572914124, + "learning_rate": 0.0001, + "loss": 70.1764, + "step": 2990 + }, + { + "epoch": 4.831964528644128, + "grad_norm": 1.1265769004821777, + "learning_rate": 0.0001, + "loss": 71.7918, + "step": 3000 + }, + { + "epoch": 4.848087872222502, + "grad_norm": 2.2792513370513916, + "learning_rate": 0.0001, + "loss": 70.4341, + "step": 3010 + }, + { + "epoch": 4.864211215800877, + "grad_norm": 0.8469329476356506, + "learning_rate": 0.0001, + "loss": 72.1137, + "step": 3020 + }, + { + "epoch": 4.880334559379252, + "grad_norm": 0.9333013296127319, + "learning_rate": 0.0001, + "loss": 71.0789, + "step": 3030 + }, + { + "epoch": 4.896457902957626, + "grad_norm": 0.7888994216918945, + "learning_rate": 0.0001, + "loss": 70.8197, + "step": 3040 + }, + { + "epoch": 4.912581246536, + "grad_norm": 0.9194874167442322, + "learning_rate": 0.0001, + "loss": 72.071, + "step": 3050 + }, + { + "epoch": 4.928704590114375, + "grad_norm": 1.1252421140670776, + "learning_rate": 0.0001, + "loss": 71.6943, + "step": 3060 + }, + { + "epoch": 4.94482793369275, + "grad_norm": 1.1801071166992188, + "learning_rate": 0.0001, + "loss": 70.8937, + "step": 3070 + }, + { + "epoch": 4.960951277271124, + "grad_norm": 0.9162415266036987, + "learning_rate": 0.0001, + "loss": 70.024, + "step": 3080 + }, + { + "epoch": 4.977074620849499, + "grad_norm": 0.8833363652229309, + "learning_rate": 0.0001, + "loss": 71.3835, + "step": 3090 + }, + { + "epoch": 4.993197964427873, + "grad_norm": 0.9985883831977844, + "learning_rate": 0.0001, + "loss": 71.635, + "step": 3100 + }, + { + "epoch": 5.0080616717891875, + "grad_norm": 1.097298264503479, + "learning_rate": 0.0001, + "loss": 65.3542, + "step": 3110 + }, + { + "epoch": 5.024185015367562, + "grad_norm": 0.9703903198242188, + "learning_rate": 0.0001, + "loss": 69.8587, + "step": 3120 + }, + { + "epoch": 5.0403083589459365, + "grad_norm": 0.8894070386886597, + "learning_rate": 0.0001, + "loss": 71.2868, + "step": 3130 + }, + { + "epoch": 5.056431702524311, + "grad_norm": 0.6984127163887024, + "learning_rate": 0.0001, + "loss": 69.0425, + "step": 3140 + }, + { + "epoch": 5.072555046102686, + "grad_norm": 0.8607762455940247, + "learning_rate": 0.0001, + "loss": 69.5354, + "step": 3150 + }, + { + "epoch": 5.08867838968106, + "grad_norm": 1.1215914487838745, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 3160 + }, + { + "epoch": 5.104801733259435, + "grad_norm": 0.9114409685134888, + "learning_rate": 0.0001, + "loss": 68.4186, + "step": 3170 + }, + { + "epoch": 5.12092507683781, + "grad_norm": 1.010400652885437, + "learning_rate": 0.0001, + "loss": 68.3965, + "step": 3180 + }, + { + "epoch": 5.137048420416184, + "grad_norm": 0.8741958141326904, + "learning_rate": 0.0001, + "loss": 70.6728, + "step": 3190 + }, + { + "epoch": 5.153171763994559, + "grad_norm": 0.8760207891464233, + "learning_rate": 0.0001, + "loss": 72.0704, + "step": 3200 + }, + { + "epoch": 5.169295107572933, + "grad_norm": 1.328247308731079, + "learning_rate": 0.0001, + "loss": 69.9082, + "step": 3210 + }, + { + "epoch": 5.185418451151308, + "grad_norm": 0.8263765573501587, + "learning_rate": 0.0001, + "loss": 70.3539, + "step": 3220 + }, + { + "epoch": 5.201541794729682, + "grad_norm": 0.9358834028244019, + "learning_rate": 0.0001, + "loss": 70.4221, + "step": 3230 + }, + { + "epoch": 5.217665138308057, + "grad_norm": 1.0769531726837158, + "learning_rate": 0.0001, + "loss": 68.8167, + "step": 3240 + }, + { + "epoch": 5.233788481886431, + "grad_norm": 1.1438790559768677, + "learning_rate": 0.0001, + "loss": 68.9288, + "step": 3250 + }, + { + "epoch": 5.249911825464806, + "grad_norm": 0.9534218311309814, + "learning_rate": 0.0001, + "loss": 70.3277, + "step": 3260 + }, + { + "epoch": 5.26603516904318, + "grad_norm": 0.88532954454422, + "learning_rate": 0.0001, + "loss": 67.5781, + "step": 3270 + }, + { + "epoch": 5.282158512621555, + "grad_norm": 0.7986133694648743, + "learning_rate": 0.0001, + "loss": 69.759, + "step": 3280 + }, + { + "epoch": 5.29828185619993, + "grad_norm": 0.9100566506385803, + "learning_rate": 0.0001, + "loss": 68.6368, + "step": 3290 + }, + { + "epoch": 5.314405199778304, + "grad_norm": 0.8861745595932007, + "learning_rate": 0.0001, + "loss": 70.5756, + "step": 3300 + }, + { + "epoch": 5.330528543356679, + "grad_norm": 1.0729680061340332, + "learning_rate": 0.0001, + "loss": 70.3178, + "step": 3310 + }, + { + "epoch": 5.346651886935053, + "grad_norm": 1.0288280248641968, + "learning_rate": 0.0001, + "loss": 69.422, + "step": 3320 + }, + { + "epoch": 5.362775230513428, + "grad_norm": 0.8335140347480774, + "learning_rate": 0.0001, + "loss": 69.8281, + "step": 3330 + }, + { + "epoch": 5.378898574091802, + "grad_norm": 0.9789717197418213, + "learning_rate": 0.0001, + "loss": 71.1532, + "step": 3340 + }, + { + "epoch": 5.395021917670177, + "grad_norm": 0.9884677529335022, + "learning_rate": 0.0001, + "loss": 69.5673, + "step": 3350 + }, + { + "epoch": 5.411145261248551, + "grad_norm": 1.050154447555542, + "learning_rate": 0.0001, + "loss": 70.8015, + "step": 3360 + }, + { + "epoch": 5.427268604826926, + "grad_norm": 0.9645342230796814, + "learning_rate": 0.0001, + "loss": 69.1615, + "step": 3370 + }, + { + "epoch": 5.4433919484053, + "grad_norm": 0.8383159041404724, + "learning_rate": 0.0001, + "loss": 70.6299, + "step": 3380 + }, + { + "epoch": 5.459515291983675, + "grad_norm": 0.9250556826591492, + "learning_rate": 0.0001, + "loss": 69.231, + "step": 3390 + }, + { + "epoch": 5.475638635562049, + "grad_norm": 0.8108687400817871, + "learning_rate": 0.0001, + "loss": 70.5858, + "step": 3400 + }, + { + "epoch": 5.491761979140424, + "grad_norm": 0.8748818039894104, + "learning_rate": 0.0001, + "loss": 69.9001, + "step": 3410 + }, + { + "epoch": 5.507885322718799, + "grad_norm": 0.7411381006240845, + "learning_rate": 0.0001, + "loss": 68.3248, + "step": 3420 + }, + { + "epoch": 5.524008666297173, + "grad_norm": 1.0753326416015625, + "learning_rate": 0.0001, + "loss": 69.5831, + "step": 3430 + }, + { + "epoch": 5.540132009875548, + "grad_norm": 0.875254213809967, + "learning_rate": 0.0001, + "loss": 69.7516, + "step": 3440 + }, + { + "epoch": 5.556255353453922, + "grad_norm": 0.7637733817100525, + "learning_rate": 0.0001, + "loss": 68.0078, + "step": 3450 + }, + { + "epoch": 5.572378697032297, + "grad_norm": 0.968203604221344, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 3460 + }, + { + "epoch": 5.588502040610671, + "grad_norm": 0.8903141617774963, + "learning_rate": 0.0001, + "loss": 68.5835, + "step": 3470 + }, + { + "epoch": 5.604625384189046, + "grad_norm": 0.9536195397377014, + "learning_rate": 0.0001, + "loss": 71.0259, + "step": 3480 + }, + { + "epoch": 5.62074872776742, + "grad_norm": 0.8951714038848877, + "learning_rate": 0.0001, + "loss": 69.4422, + "step": 3490 + }, + { + "epoch": 5.636872071345795, + "grad_norm": 0.9639522433280945, + "learning_rate": 0.0001, + "loss": 66.5173, + "step": 3500 + }, + { + "epoch": 5.65299541492417, + "grad_norm": 1.0022730827331543, + "learning_rate": 0.0001, + "loss": 68.3443, + "step": 3510 + }, + { + "epoch": 5.669118758502544, + "grad_norm": 1.0654487609863281, + "learning_rate": 0.0001, + "loss": 69.0835, + "step": 3520 + }, + { + "epoch": 5.685242102080919, + "grad_norm": 0.9934629797935486, + "learning_rate": 0.0001, + "loss": 69.2278, + "step": 3530 + }, + { + "epoch": 5.7013654456592935, + "grad_norm": 0.8773566484451294, + "learning_rate": 0.0001, + "loss": 68.9339, + "step": 3540 + }, + { + "epoch": 5.7174887892376685, + "grad_norm": 0.7756998538970947, + "learning_rate": 0.0001, + "loss": 69.2683, + "step": 3550 + }, + { + "epoch": 5.7336121328160425, + "grad_norm": 0.8819359540939331, + "learning_rate": 0.0001, + "loss": 68.5071, + "step": 3560 + }, + { + "epoch": 5.7497354763944175, + "grad_norm": 1.1069825887680054, + "learning_rate": 0.0001, + "loss": 68.5236, + "step": 3570 + }, + { + "epoch": 5.765858819972792, + "grad_norm": 0.8006312847137451, + "learning_rate": 0.0001, + "loss": 68.7192, + "step": 3580 + }, + { + "epoch": 5.781982163551167, + "grad_norm": 1.0278719663619995, + "learning_rate": 0.0001, + "loss": 69.2545, + "step": 3590 + }, + { + "epoch": 5.798105507129541, + "grad_norm": 0.9205041527748108, + "learning_rate": 0.0001, + "loss": 69.7698, + "step": 3600 + }, + { + "epoch": 5.814228850707916, + "grad_norm": 1.0769038200378418, + "learning_rate": 0.0001, + "loss": 67.6568, + "step": 3610 + }, + { + "epoch": 5.83035219428629, + "grad_norm": 1.0565663576126099, + "learning_rate": 0.0001, + "loss": 68.9166, + "step": 3620 + }, + { + "epoch": 5.846475537864665, + "grad_norm": 0.9008663296699524, + "learning_rate": 0.0001, + "loss": 69.5837, + "step": 3630 + }, + { + "epoch": 5.86259888144304, + "grad_norm": 0.8369411826133728, + "learning_rate": 0.0001, + "loss": 68.5661, + "step": 3640 + }, + { + "epoch": 5.878722225021414, + "grad_norm": 1.0472379922866821, + "learning_rate": 0.0001, + "loss": 68.3122, + "step": 3650 + }, + { + "epoch": 5.894845568599789, + "grad_norm": 0.965277373790741, + "learning_rate": 0.0001, + "loss": 69.8062, + "step": 3660 + }, + { + "epoch": 5.910968912178163, + "grad_norm": 0.8936495780944824, + "learning_rate": 0.0001, + "loss": 67.1016, + "step": 3670 + }, + { + "epoch": 5.927092255756538, + "grad_norm": 0.8559162020683289, + "learning_rate": 0.0001, + "loss": 69.9986, + "step": 3680 + }, + { + "epoch": 5.943215599334912, + "grad_norm": 1.4815773963928223, + "learning_rate": 0.0001, + "loss": 66.7648, + "step": 3690 + }, + { + "epoch": 5.959338942913287, + "grad_norm": 0.9034469723701477, + "learning_rate": 0.0001, + "loss": 68.0767, + "step": 3700 + }, + { + "epoch": 5.975462286491661, + "grad_norm": 1.838799238204956, + "learning_rate": 0.0001, + "loss": 66.5848, + "step": 3710 + }, + { + "epoch": 5.991585630070036, + "grad_norm": 0.9274787902832031, + "learning_rate": 0.0001, + "loss": 70.0396, + "step": 3720 + }, + { + "epoch": 6.0064493374313495, + "grad_norm": 1.016974687576294, + "learning_rate": 0.0001, + "loss": 62.2268, + "step": 3730 + }, + { + "epoch": 6.022572681009724, + "grad_norm": 1.0522642135620117, + "learning_rate": 0.0001, + "loss": 69.3381, + "step": 3740 + }, + { + "epoch": 6.0386960245880985, + "grad_norm": 1.1157186031341553, + "learning_rate": 0.0001, + "loss": 68.8969, + "step": 3750 + }, + { + "epoch": 6.0548193681664735, + "grad_norm": 0.9913288950920105, + "learning_rate": 0.0001, + "loss": 67.3634, + "step": 3760 + }, + { + "epoch": 6.0709427117448485, + "grad_norm": 0.8823490142822266, + "learning_rate": 0.0001, + "loss": 70.978, + "step": 3770 + }, + { + "epoch": 6.0870660553232225, + "grad_norm": 0.853837788105011, + "learning_rate": 0.0001, + "loss": 68.9924, + "step": 3780 + }, + { + "epoch": 6.1031893989015975, + "grad_norm": 0.7937766909599304, + "learning_rate": 0.0001, + "loss": 68.1994, + "step": 3790 + }, + { + "epoch": 6.119312742479972, + "grad_norm": 1.7638057470321655, + "learning_rate": 0.0001, + "loss": 68.5078, + "step": 3800 + }, + { + "epoch": 6.135436086058347, + "grad_norm": 1.0542887449264526, + "learning_rate": 0.0001, + "loss": 68.9362, + "step": 3810 + }, + { + "epoch": 6.151559429636721, + "grad_norm": 1.06019926071167, + "learning_rate": 0.0001, + "loss": 67.2493, + "step": 3820 + }, + { + "epoch": 6.167682773215096, + "grad_norm": 0.9765111804008484, + "learning_rate": 0.0001, + "loss": 70.4797, + "step": 3830 + }, + { + "epoch": 6.18380611679347, + "grad_norm": 1.006372332572937, + "learning_rate": 0.0001, + "loss": 69.4233, + "step": 3840 + }, + { + "epoch": 6.199929460371845, + "grad_norm": 0.8274241089820862, + "learning_rate": 0.0001, + "loss": 67.9907, + "step": 3850 + }, + { + "epoch": 6.216052803950219, + "grad_norm": 0.9905765056610107, + "learning_rate": 0.0001, + "loss": 69.6988, + "step": 3860 + }, + { + "epoch": 6.232176147528594, + "grad_norm": 1.0065921545028687, + "learning_rate": 0.0001, + "loss": 68.7678, + "step": 3870 + }, + { + "epoch": 6.248299491106969, + "grad_norm": 0.8845255970954895, + "learning_rate": 0.0001, + "loss": 68.3039, + "step": 3880 + }, + { + "epoch": 6.264422834685343, + "grad_norm": 0.9030968546867371, + "learning_rate": 0.0001, + "loss": 68.671, + "step": 3890 + }, + { + "epoch": 6.280546178263718, + "grad_norm": 0.8650258183479309, + "learning_rate": 0.0001, + "loss": 68.5309, + "step": 3900 + }, + { + "epoch": 6.296669521842092, + "grad_norm": 0.8961272835731506, + "learning_rate": 0.0001, + "loss": 67.5611, + "step": 3910 + }, + { + "epoch": 6.312792865420467, + "grad_norm": 1.1702407598495483, + "learning_rate": 0.0001, + "loss": 68.2768, + "step": 3920 + }, + { + "epoch": 6.328916208998841, + "grad_norm": 0.8598962426185608, + "learning_rate": 0.0001, + "loss": 69.1961, + "step": 3930 + }, + { + "epoch": 6.345039552577216, + "grad_norm": 0.8714122176170349, + "learning_rate": 0.0001, + "loss": 66.8628, + "step": 3940 + }, + { + "epoch": 6.36116289615559, + "grad_norm": 0.7757895588874817, + "learning_rate": 0.0001, + "loss": 68.6908, + "step": 3950 + }, + { + "epoch": 6.377286239733965, + "grad_norm": 0.9189490675926208, + "learning_rate": 0.0001, + "loss": 67.8685, + "step": 3960 + }, + { + "epoch": 6.393409583312339, + "grad_norm": 0.8161299228668213, + "learning_rate": 0.0001, + "loss": 66.816, + "step": 3970 + }, + { + "epoch": 6.409532926890714, + "grad_norm": 0.8559791445732117, + "learning_rate": 0.0001, + "loss": 68.4302, + "step": 3980 + }, + { + "epoch": 6.425656270469089, + "grad_norm": 0.8087984919548035, + "learning_rate": 0.0001, + "loss": 66.2646, + "step": 3990 + }, + { + "epoch": 6.441779614047463, + "grad_norm": 0.7733110785484314, + "learning_rate": 0.0001, + "loss": 69.2769, + "step": 4000 + }, + { + "epoch": 6.457902957625838, + "grad_norm": 0.9335984587669373, + "learning_rate": 0.0001, + "loss": 67.1199, + "step": 4010 + }, + { + "epoch": 6.474026301204212, + "grad_norm": 0.9546418786048889, + "learning_rate": 0.0001, + "loss": 68.8706, + "step": 4020 + }, + { + "epoch": 6.490149644782587, + "grad_norm": 0.9670593738555908, + "learning_rate": 0.0001, + "loss": 68.2329, + "step": 4030 + }, + { + "epoch": 6.506272988360961, + "grad_norm": 0.8428588509559631, + "learning_rate": 0.0001, + "loss": 68.0105, + "step": 4040 + }, + { + "epoch": 6.522396331939336, + "grad_norm": 0.9755622744560242, + "learning_rate": 0.0001, + "loss": 68.656, + "step": 4050 + }, + { + "epoch": 6.53851967551771, + "grad_norm": 0.8972346186637878, + "learning_rate": 0.0001, + "loss": 69.3665, + "step": 4060 + }, + { + "epoch": 6.554643019096085, + "grad_norm": 0.9716668128967285, + "learning_rate": 0.0001, + "loss": 68.1053, + "step": 4070 + }, + { + "epoch": 6.570766362674459, + "grad_norm": 0.8147910833358765, + "learning_rate": 0.0001, + "loss": 67.9778, + "step": 4080 + }, + { + "epoch": 6.586889706252834, + "grad_norm": 0.8304234147071838, + "learning_rate": 0.0001, + "loss": 67.7069, + "step": 4090 + }, + { + "epoch": 6.603013049831208, + "grad_norm": 0.9356856346130371, + "learning_rate": 0.0001, + "loss": 68.1942, + "step": 4100 + }, + { + "epoch": 6.619136393409583, + "grad_norm": 1.017134189605713, + "learning_rate": 0.0001, + "loss": 67.4211, + "step": 4110 + }, + { + "epoch": 6.635259736987958, + "grad_norm": 1.142558217048645, + "learning_rate": 0.0001, + "loss": 68.4413, + "step": 4120 + }, + { + "epoch": 6.651383080566332, + "grad_norm": 0.9071153998374939, + "learning_rate": 0.0001, + "loss": 69.3666, + "step": 4130 + }, + { + "epoch": 6.667506424144707, + "grad_norm": 0.9620397686958313, + "learning_rate": 0.0001, + "loss": 68.208, + "step": 4140 + }, + { + "epoch": 6.683629767723081, + "grad_norm": 0.8282128572463989, + "learning_rate": 0.0001, + "loss": 67.1702, + "step": 4150 + }, + { + "epoch": 6.699753111301456, + "grad_norm": 0.8838199973106384, + "learning_rate": 0.0001, + "loss": 69.0074, + "step": 4160 + }, + { + "epoch": 6.71587645487983, + "grad_norm": 0.8864356279373169, + "learning_rate": 0.0001, + "loss": 67.8052, + "step": 4170 + }, + { + "epoch": 6.731999798458205, + "grad_norm": 0.9373385906219482, + "learning_rate": 0.0001, + "loss": 67.446, + "step": 4180 + }, + { + "epoch": 6.7481231420365795, + "grad_norm": 0.8308956027030945, + "learning_rate": 0.0001, + "loss": 68.716, + "step": 4190 + }, + { + "epoch": 6.7642464856149545, + "grad_norm": 0.8031372427940369, + "learning_rate": 0.0001, + "loss": 68.0482, + "step": 4200 + }, + { + "epoch": 6.780369829193329, + "grad_norm": 0.9258586764335632, + "learning_rate": 0.0001, + "loss": 68.5607, + "step": 4210 + }, + { + "epoch": 6.7964931727717035, + "grad_norm": 0.8147327899932861, + "learning_rate": 0.0001, + "loss": 67.7265, + "step": 4220 + }, + { + "epoch": 6.8126165163500785, + "grad_norm": 0.8717535734176636, + "learning_rate": 0.0001, + "loss": 67.8097, + "step": 4230 + }, + { + "epoch": 6.828739859928453, + "grad_norm": 0.9876209497451782, + "learning_rate": 0.0001, + "loss": 67.9068, + "step": 4240 + }, + { + "epoch": 6.8448632035068275, + "grad_norm": 0.718376874923706, + "learning_rate": 0.0001, + "loss": 68.1762, + "step": 4250 + }, + { + "epoch": 6.860986547085202, + "grad_norm": 0.8264320492744446, + "learning_rate": 0.0001, + "loss": 67.8391, + "step": 4260 + }, + { + "epoch": 6.877109890663577, + "grad_norm": 0.7478179335594177, + "learning_rate": 0.0001, + "loss": 67.6635, + "step": 4270 + }, + { + "epoch": 6.893233234241951, + "grad_norm": 0.8394957184791565, + "learning_rate": 0.0001, + "loss": 68.6073, + "step": 4280 + }, + { + "epoch": 6.909356577820326, + "grad_norm": 0.8534071445465088, + "learning_rate": 0.0001, + "loss": 69.6206, + "step": 4290 + }, + { + "epoch": 6.9254799213987, + "grad_norm": 0.9137582778930664, + "learning_rate": 0.0001, + "loss": 67.4362, + "step": 4300 + }, + { + "epoch": 6.941603264977075, + "grad_norm": 0.8869971036911011, + "learning_rate": 0.0001, + "loss": 67.6811, + "step": 4310 + }, + { + "epoch": 6.957726608555449, + "grad_norm": 0.889025866985321, + "learning_rate": 0.0001, + "loss": 65.3354, + "step": 4320 + }, + { + "epoch": 6.973849952133824, + "grad_norm": 0.7968549728393555, + "learning_rate": 0.0001, + "loss": 66.192, + "step": 4330 + }, + { + "epoch": 6.989973295712199, + "grad_norm": 1.0077433586120605, + "learning_rate": 0.0001, + "loss": 67.5216, + "step": 4340 + }, + { + "epoch": 7.004837003073512, + "grad_norm": 0.7900649309158325, + "learning_rate": 0.0001, + "loss": 62.2421, + "step": 4350 + }, + { + "epoch": 7.020960346651887, + "grad_norm": 1.0160410404205322, + "learning_rate": 0.0001, + "loss": 68.0978, + "step": 4360 + }, + { + "epoch": 7.037083690230261, + "grad_norm": 0.841387927532196, + "learning_rate": 0.0001, + "loss": 68.1008, + "step": 4370 + }, + { + "epoch": 7.053207033808636, + "grad_norm": 0.87884122133255, + "learning_rate": 0.0001, + "loss": 66.4667, + "step": 4380 + }, + { + "epoch": 7.06933037738701, + "grad_norm": 0.8580564856529236, + "learning_rate": 0.0001, + "loss": 67.2002, + "step": 4390 + }, + { + "epoch": 7.085453720965385, + "grad_norm": 0.7560063600540161, + "learning_rate": 0.0001, + "loss": 67.6743, + "step": 4400 + }, + { + "epoch": 7.1015770645437595, + "grad_norm": 0.855478048324585, + "learning_rate": 0.0001, + "loss": 67.822, + "step": 4410 + }, + { + "epoch": 7.1177004081221344, + "grad_norm": 0.7994911670684814, + "learning_rate": 0.0001, + "loss": 66.0094, + "step": 4420 + }, + { + "epoch": 7.1338237517005085, + "grad_norm": 0.8728616833686829, + "learning_rate": 0.0001, + "loss": 66.405, + "step": 4430 + }, + { + "epoch": 7.1499470952788835, + "grad_norm": 0.8434081673622131, + "learning_rate": 0.0001, + "loss": 67.0535, + "step": 4440 + }, + { + "epoch": 7.166070438857258, + "grad_norm": 0.8543094396591187, + "learning_rate": 0.0001, + "loss": 66.2417, + "step": 4450 + }, + { + "epoch": 7.182193782435633, + "grad_norm": 0.7209319472312927, + "learning_rate": 0.0001, + "loss": 67.3958, + "step": 4460 + }, + { + "epoch": 7.1983171260140075, + "grad_norm": 0.6735154390335083, + "learning_rate": 0.0001, + "loss": 67.4086, + "step": 4470 + }, + { + "epoch": 7.214440469592382, + "grad_norm": 0.9126104116439819, + "learning_rate": 0.0001, + "loss": 68.4191, + "step": 4480 + }, + { + "epoch": 7.230563813170757, + "grad_norm": 0.9334300756454468, + "learning_rate": 0.0001, + "loss": 68.3247, + "step": 4490 + }, + { + "epoch": 7.246687156749131, + "grad_norm": 1.5956038236618042, + "learning_rate": 0.0001, + "loss": 68.2779, + "step": 4500 + }, + { + "epoch": 7.262810500327506, + "grad_norm": 1.5012775659561157, + "learning_rate": 0.0001, + "loss": 68.3533, + "step": 4510 + }, + { + "epoch": 7.27893384390588, + "grad_norm": 1.1376632452011108, + "learning_rate": 0.0001, + "loss": 65.3627, + "step": 4520 + }, + { + "epoch": 7.295057187484255, + "grad_norm": 0.8597291111946106, + "learning_rate": 0.0001, + "loss": 67.8653, + "step": 4530 + }, + { + "epoch": 7.311180531062629, + "grad_norm": 0.7645048499107361, + "learning_rate": 0.0001, + "loss": 65.4592, + "step": 4540 + }, + { + "epoch": 7.327303874641004, + "grad_norm": 0.7717318534851074, + "learning_rate": 0.0001, + "loss": 66.902, + "step": 4550 + }, + { + "epoch": 7.343427218219378, + "grad_norm": 0.8884950876235962, + "learning_rate": 0.0001, + "loss": 67.1858, + "step": 4560 + }, + { + "epoch": 7.359550561797753, + "grad_norm": 0.7826898694038391, + "learning_rate": 0.0001, + "loss": 68.4673, + "step": 4570 + }, + { + "epoch": 7.375673905376128, + "grad_norm": 0.7484182119369507, + "learning_rate": 0.0001, + "loss": 67.4859, + "step": 4580 + }, + { + "epoch": 7.391797248954502, + "grad_norm": 0.918194591999054, + "learning_rate": 0.0001, + "loss": 67.7393, + "step": 4590 + }, + { + "epoch": 7.407920592532877, + "grad_norm": 1.0696896314620972, + "learning_rate": 0.0001, + "loss": 67.0101, + "step": 4600 + }, + { + "epoch": 7.424043936111251, + "grad_norm": 0.8190692067146301, + "learning_rate": 0.0001, + "loss": 68.5358, + "step": 4610 + }, + { + "epoch": 7.440167279689626, + "grad_norm": 0.9063783884048462, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 4620 + }, + { + "epoch": 7.456290623268, + "grad_norm": 0.7588100433349609, + "learning_rate": 0.0001, + "loss": 68.0066, + "step": 4630 + }, + { + "epoch": 7.472413966846375, + "grad_norm": 0.897281289100647, + "learning_rate": 0.0001, + "loss": 67.2499, + "step": 4640 + }, + { + "epoch": 7.488537310424749, + "grad_norm": 0.9505569338798523, + "learning_rate": 0.0001, + "loss": 67.5452, + "step": 4650 + }, + { + "epoch": 7.504660654003124, + "grad_norm": 0.8409159183502197, + "learning_rate": 0.0001, + "loss": 66.6293, + "step": 4660 + }, + { + "epoch": 7.520783997581498, + "grad_norm": 0.9878574013710022, + "learning_rate": 0.0001, + "loss": 65.3379, + "step": 4670 + }, + { + "epoch": 7.536907341159873, + "grad_norm": 1.0410685539245605, + "learning_rate": 0.0001, + "loss": 65.8028, + "step": 4680 + }, + { + "epoch": 7.553030684738248, + "grad_norm": 0.8417725563049316, + "learning_rate": 0.0001, + "loss": 65.4283, + "step": 4690 + }, + { + "epoch": 7.569154028316622, + "grad_norm": 0.7660863399505615, + "learning_rate": 0.0001, + "loss": 66.6765, + "step": 4700 + }, + { + "epoch": 7.585277371894997, + "grad_norm": 0.8074233531951904, + "learning_rate": 0.0001, + "loss": 67.1082, + "step": 4710 + }, + { + "epoch": 7.601400715473371, + "grad_norm": 0.7653326988220215, + "learning_rate": 0.0001, + "loss": 66.028, + "step": 4720 + }, + { + "epoch": 7.617524059051746, + "grad_norm": 1.1764041185379028, + "learning_rate": 0.0001, + "loss": 65.2587, + "step": 4730 + }, + { + "epoch": 7.63364740263012, + "grad_norm": 1.7604950666427612, + "learning_rate": 0.0001, + "loss": 66.0543, + "step": 4740 + }, + { + "epoch": 7.649770746208495, + "grad_norm": 0.941438615322113, + "learning_rate": 0.0001, + "loss": 65.6954, + "step": 4750 + }, + { + "epoch": 7.665894089786869, + "grad_norm": 0.740069568157196, + "learning_rate": 0.0001, + "loss": 67.9701, + "step": 4760 + }, + { + "epoch": 7.682017433365244, + "grad_norm": 0.9867332577705383, + "learning_rate": 0.0001, + "loss": 66.1445, + "step": 4770 + }, + { + "epoch": 7.698140776943618, + "grad_norm": 0.8156166672706604, + "learning_rate": 0.0001, + "loss": 67.2647, + "step": 4780 + }, + { + "epoch": 7.714264120521993, + "grad_norm": 0.7689579129219055, + "learning_rate": 0.0001, + "loss": 67.0961, + "step": 4790 + }, + { + "epoch": 7.730387464100367, + "grad_norm": 0.8159700036048889, + "learning_rate": 0.0001, + "loss": 66.4974, + "step": 4800 + }, + { + "epoch": 7.746510807678742, + "grad_norm": 0.8554222583770752, + "learning_rate": 0.0001, + "loss": 64.5057, + "step": 4810 + }, + { + "epoch": 7.762634151257117, + "grad_norm": 0.856629490852356, + "learning_rate": 0.0001, + "loss": 67.7819, + "step": 4820 + }, + { + "epoch": 7.778757494835491, + "grad_norm": 1.157179832458496, + "learning_rate": 0.0001, + "loss": 66.0122, + "step": 4830 + }, + { + "epoch": 7.794880838413866, + "grad_norm": 0.9597711563110352, + "learning_rate": 0.0001, + "loss": 66.8248, + "step": 4840 + }, + { + "epoch": 7.8110041819922404, + "grad_norm": 0.9175744652748108, + "learning_rate": 0.0001, + "loss": 65.7998, + "step": 4850 + }, + { + "epoch": 7.827127525570615, + "grad_norm": 0.8467298150062561, + "learning_rate": 0.0001, + "loss": 67.5363, + "step": 4860 + }, + { + "epoch": 7.8432508691489895, + "grad_norm": 0.863922655582428, + "learning_rate": 0.0001, + "loss": 67.7219, + "step": 4870 + }, + { + "epoch": 7.8593742127273645, + "grad_norm": 0.9896290302276611, + "learning_rate": 0.0001, + "loss": 66.7098, + "step": 4880 + }, + { + "epoch": 7.8754975563057386, + "grad_norm": 0.8828520774841309, + "learning_rate": 0.0001, + "loss": 66.0192, + "step": 4890 + }, + { + "epoch": 7.8916208998841135, + "grad_norm": 0.9093656539916992, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 4900 + }, + { + "epoch": 7.9077442434624885, + "grad_norm": 0.8665091395378113, + "learning_rate": 0.0001, + "loss": 66.5851, + "step": 4910 + }, + { + "epoch": 7.923867587040863, + "grad_norm": 0.8027398586273193, + "learning_rate": 0.0001, + "loss": 64.8607, + "step": 4920 + }, + { + "epoch": 7.9399909306192376, + "grad_norm": 0.7808436751365662, + "learning_rate": 0.0001, + "loss": 65.052, + "step": 4930 + }, + { + "epoch": 7.956114274197612, + "grad_norm": 0.7246070504188538, + "learning_rate": 0.0001, + "loss": 64.789, + "step": 4940 + }, + { + "epoch": 7.972237617775987, + "grad_norm": 1.0405027866363525, + "learning_rate": 0.0001, + "loss": 66.7735, + "step": 4950 + }, + { + "epoch": 7.988360961354361, + "grad_norm": 0.812921404838562, + "learning_rate": 0.0001, + "loss": 64.1286, + "step": 4960 + }, + { + "epoch": 8.003224668715674, + "grad_norm": 0.884215772151947, + "learning_rate": 0.0001, + "loss": 61.0812, + "step": 4970 + }, + { + "epoch": 8.01934801229405, + "grad_norm": 0.8519641757011414, + "learning_rate": 0.0001, + "loss": 66.2784, + "step": 4980 + }, + { + "epoch": 8.035471355872424, + "grad_norm": 0.8331199884414673, + "learning_rate": 0.0001, + "loss": 67.1405, + "step": 4990 + }, + { + "epoch": 8.0515946994508, + "grad_norm": 0.8568746447563171, + "learning_rate": 0.0001, + "loss": 66.7835, + "step": 5000 + }, + { + "epoch": 8.067718043029172, + "grad_norm": 1.170482873916626, + "learning_rate": 0.0001, + "loss": 64.2269, + "step": 5010 + }, + { + "epoch": 8.083841386607547, + "grad_norm": 0.9171113967895508, + "learning_rate": 0.0001, + "loss": 66.5567, + "step": 5020 + }, + { + "epoch": 8.099964730185922, + "grad_norm": 0.8988714814186096, + "learning_rate": 0.0001, + "loss": 66.0471, + "step": 5030 + }, + { + "epoch": 8.116088073764297, + "grad_norm": 0.7651671171188354, + "learning_rate": 0.0001, + "loss": 65.7185, + "step": 5040 + }, + { + "epoch": 8.132211417342672, + "grad_norm": 0.6890843510627747, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 5050 + }, + { + "epoch": 8.148334760921045, + "grad_norm": 0.7683818340301514, + "learning_rate": 0.0001, + "loss": 65.895, + "step": 5060 + }, + { + "epoch": 8.16445810449942, + "grad_norm": 0.9398573637008667, + "learning_rate": 0.0001, + "loss": 64.674, + "step": 5070 + }, + { + "epoch": 8.180581448077795, + "grad_norm": 0.7708531022071838, + "learning_rate": 0.0001, + "loss": 65.0637, + "step": 5080 + }, + { + "epoch": 8.19670479165617, + "grad_norm": 0.8840665221214294, + "learning_rate": 0.0001, + "loss": 65.6449, + "step": 5090 + }, + { + "epoch": 8.212828135234544, + "grad_norm": 0.8390524983406067, + "learning_rate": 0.0001, + "loss": 65.4738, + "step": 5100 + }, + { + "epoch": 8.228951478812919, + "grad_norm": 1.7559128999710083, + "learning_rate": 0.0001, + "loss": 68.911, + "step": 5110 + }, + { + "epoch": 8.245074822391294, + "grad_norm": 0.8293817043304443, + "learning_rate": 0.0001, + "loss": 65.2601, + "step": 5120 + }, + { + "epoch": 8.261198165969668, + "grad_norm": 0.8947123289108276, + "learning_rate": 0.0001, + "loss": 65.8611, + "step": 5130 + }, + { + "epoch": 8.277321509548042, + "grad_norm": 0.8526618480682373, + "learning_rate": 0.0001, + "loss": 65.559, + "step": 5140 + }, + { + "epoch": 8.293444853126417, + "grad_norm": 1.7423408031463623, + "learning_rate": 0.0001, + "loss": 64.5007, + "step": 5150 + }, + { + "epoch": 8.309568196704792, + "grad_norm": 1.0512502193450928, + "learning_rate": 0.0001, + "loss": 66.2593, + "step": 5160 + }, + { + "epoch": 8.325691540283167, + "grad_norm": 0.8129869103431702, + "learning_rate": 0.0001, + "loss": 64.7989, + "step": 5170 + }, + { + "epoch": 8.341814883861542, + "grad_norm": 0.852199912071228, + "learning_rate": 0.0001, + "loss": 65.9627, + "step": 5180 + }, + { + "epoch": 8.357938227439915, + "grad_norm": 0.8932474851608276, + "learning_rate": 0.0001, + "loss": 64.1035, + "step": 5190 + }, + { + "epoch": 8.37406157101829, + "grad_norm": 1.2631597518920898, + "learning_rate": 0.0001, + "loss": 65.3406, + "step": 5200 + }, + { + "epoch": 8.390184914596665, + "grad_norm": 0.9062619209289551, + "learning_rate": 0.0001, + "loss": 65.1841, + "step": 5210 + }, + { + "epoch": 8.40630825817504, + "grad_norm": 0.9153940081596375, + "learning_rate": 0.0001, + "loss": 66.3852, + "step": 5220 + }, + { + "epoch": 8.422431601753413, + "grad_norm": 0.8056164979934692, + "learning_rate": 0.0001, + "loss": 66.511, + "step": 5230 + }, + { + "epoch": 8.438554945331788, + "grad_norm": 0.7506036162376404, + "learning_rate": 0.0001, + "loss": 65.275, + "step": 5240 + }, + { + "epoch": 8.454678288910163, + "grad_norm": 0.7994081377983093, + "learning_rate": 0.0001, + "loss": 64.9946, + "step": 5250 + }, + { + "epoch": 8.470801632488538, + "grad_norm": 0.9887873530387878, + "learning_rate": 0.0001, + "loss": 67.2057, + "step": 5260 + }, + { + "epoch": 8.486924976066913, + "grad_norm": 0.8495044112205505, + "learning_rate": 0.0001, + "loss": 66.2481, + "step": 5270 + }, + { + "epoch": 8.503048319645286, + "grad_norm": 0.7353820204734802, + "learning_rate": 0.0001, + "loss": 65.6163, + "step": 5280 + }, + { + "epoch": 8.519171663223661, + "grad_norm": 0.795216977596283, + "learning_rate": 0.0001, + "loss": 64.808, + "step": 5290 + }, + { + "epoch": 8.535295006802036, + "grad_norm": 0.8121851682662964, + "learning_rate": 0.0001, + "loss": 65.9755, + "step": 5300 + }, + { + "epoch": 8.55141835038041, + "grad_norm": 0.8164243698120117, + "learning_rate": 0.0001, + "loss": 66.0583, + "step": 5310 + }, + { + "epoch": 8.567541693958784, + "grad_norm": 0.9893137216567993, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 5320 + }, + { + "epoch": 8.583665037537159, + "grad_norm": 1.3616434335708618, + "learning_rate": 0.0001, + "loss": 64.1444, + "step": 5330 + }, + { + "epoch": 8.599788381115534, + "grad_norm": 0.7002740502357483, + "learning_rate": 0.0001, + "loss": 65.0906, + "step": 5340 + }, + { + "epoch": 8.615911724693909, + "grad_norm": 0.838424563407898, + "learning_rate": 0.0001, + "loss": 65.4578, + "step": 5350 + }, + { + "epoch": 8.632035068272282, + "grad_norm": 0.988902747631073, + "learning_rate": 0.0001, + "loss": 64.3405, + "step": 5360 + }, + { + "epoch": 8.648158411850657, + "grad_norm": 0.7941252589225769, + "learning_rate": 0.0001, + "loss": 67.0146, + "step": 5370 + }, + { + "epoch": 8.664281755429032, + "grad_norm": 0.7727184295654297, + "learning_rate": 0.0001, + "loss": 64.8011, + "step": 5380 + }, + { + "epoch": 8.680405099007407, + "grad_norm": 0.9514367580413818, + "learning_rate": 0.0001, + "loss": 65.8165, + "step": 5390 + }, + { + "epoch": 8.69652844258578, + "grad_norm": 0.870544970035553, + "learning_rate": 0.0001, + "loss": 65.803, + "step": 5400 + }, + { + "epoch": 8.712651786164155, + "grad_norm": 0.7348082065582275, + "learning_rate": 0.0001, + "loss": 65.8409, + "step": 5410 + }, + { + "epoch": 8.72877512974253, + "grad_norm": 0.7543814778327942, + "learning_rate": 0.0001, + "loss": 65.5958, + "step": 5420 + }, + { + "epoch": 8.744898473320905, + "grad_norm": 0.7520368099212646, + "learning_rate": 0.0001, + "loss": 65.3323, + "step": 5430 + }, + { + "epoch": 8.76102181689928, + "grad_norm": 0.7610031366348267, + "learning_rate": 0.0001, + "loss": 65.4386, + "step": 5440 + }, + { + "epoch": 8.777145160477653, + "grad_norm": 0.8854930400848389, + "learning_rate": 0.0001, + "loss": 66.181, + "step": 5450 + }, + { + "epoch": 8.793268504056028, + "grad_norm": 0.7659342885017395, + "learning_rate": 0.0001, + "loss": 65.2586, + "step": 5460 + }, + { + "epoch": 8.809391847634403, + "grad_norm": 0.8592572212219238, + "learning_rate": 0.0001, + "loss": 66.3306, + "step": 5470 + }, + { + "epoch": 8.825515191212778, + "grad_norm": 0.8169172406196594, + "learning_rate": 0.0001, + "loss": 65.8283, + "step": 5480 + }, + { + "epoch": 8.841638534791151, + "grad_norm": 0.7380614876747131, + "learning_rate": 0.0001, + "loss": 65.8711, + "step": 5490 + }, + { + "epoch": 8.857761878369526, + "grad_norm": 0.9993331432342529, + "learning_rate": 0.0001, + "loss": 64.9378, + "step": 5500 + }, + { + "epoch": 8.873885221947901, + "grad_norm": 0.6252862811088562, + "learning_rate": 0.0001, + "loss": 64.1655, + "step": 5510 + }, + { + "epoch": 8.890008565526276, + "grad_norm": 0.8441852331161499, + "learning_rate": 0.0001, + "loss": 64.9795, + "step": 5520 + }, + { + "epoch": 8.906131909104651, + "grad_norm": 0.7075499892234802, + "learning_rate": 0.0001, + "loss": 65.5843, + "step": 5530 + }, + { + "epoch": 8.922255252683025, + "grad_norm": 0.7374144196510315, + "learning_rate": 0.0001, + "loss": 67.5064, + "step": 5540 + }, + { + "epoch": 8.9383785962614, + "grad_norm": 0.8073632121086121, + "learning_rate": 0.0001, + "loss": 67.0182, + "step": 5550 + }, + { + "epoch": 8.954501939839774, + "grad_norm": 0.7576195001602173, + "learning_rate": 0.0001, + "loss": 65.9139, + "step": 5560 + }, + { + "epoch": 8.97062528341815, + "grad_norm": 1.6479626893997192, + "learning_rate": 0.0001, + "loss": 66.3517, + "step": 5570 + }, + { + "epoch": 8.986748626996523, + "grad_norm": 0.846513569355011, + "learning_rate": 0.0001, + "loss": 64.2908, + "step": 5580 + }, + { + "epoch": 9.001612334357837, + "grad_norm": 0.9551037549972534, + "learning_rate": 0.0001, + "loss": 61.7312, + "step": 5590 + }, + { + "epoch": 9.017735677936212, + "grad_norm": 0.8250304460525513, + "learning_rate": 0.0001, + "loss": 66.3351, + "step": 5600 + }, + { + "epoch": 9.033859021514587, + "grad_norm": 0.7932960391044617, + "learning_rate": 0.0001, + "loss": 66.7753, + "step": 5610 + }, + { + "epoch": 9.04998236509296, + "grad_norm": 0.9839881658554077, + "learning_rate": 0.0001, + "loss": 64.5146, + "step": 5620 + }, + { + "epoch": 9.066105708671335, + "grad_norm": 0.7337850332260132, + "learning_rate": 0.0001, + "loss": 65.0544, + "step": 5630 + }, + { + "epoch": 9.08222905224971, + "grad_norm": 0.7496634721755981, + "learning_rate": 0.0001, + "loss": 64.1451, + "step": 5640 + }, + { + "epoch": 9.098352395828085, + "grad_norm": 0.9457297921180725, + "learning_rate": 0.0001, + "loss": 65.3036, + "step": 5650 + }, + { + "epoch": 9.11447573940646, + "grad_norm": 0.8315348625183105, + "learning_rate": 0.0001, + "loss": 66.0585, + "step": 5660 + }, + { + "epoch": 9.130599082984833, + "grad_norm": 0.9666195511817932, + "learning_rate": 0.0001, + "loss": 64.6802, + "step": 5670 + }, + { + "epoch": 9.146722426563208, + "grad_norm": 0.7416576147079468, + "learning_rate": 0.0001, + "loss": 65.1557, + "step": 5680 + }, + { + "epoch": 9.162845770141583, + "grad_norm": 0.7723799347877502, + "learning_rate": 0.0001, + "loss": 66.2871, + "step": 5690 + }, + { + "epoch": 9.178969113719958, + "grad_norm": 0.7461305856704712, + "learning_rate": 0.0001, + "loss": 65.5795, + "step": 5700 + }, + { + "epoch": 9.195092457298331, + "grad_norm": 0.6663411259651184, + "learning_rate": 0.0001, + "loss": 66.4094, + "step": 5710 + }, + { + "epoch": 9.211215800876706, + "grad_norm": 0.9340134859085083, + "learning_rate": 0.0001, + "loss": 63.5753, + "step": 5720 + }, + { + "epoch": 9.227339144455081, + "grad_norm": 0.7643963694572449, + "learning_rate": 0.0001, + "loss": 65.3359, + "step": 5730 + }, + { + "epoch": 9.243462488033456, + "grad_norm": 0.6931570768356323, + "learning_rate": 0.0001, + "loss": 65.0755, + "step": 5740 + }, + { + "epoch": 9.259585831611831, + "grad_norm": 0.9096988439559937, + "learning_rate": 0.0001, + "loss": 64.7983, + "step": 5750 + }, + { + "epoch": 9.275709175190205, + "grad_norm": 0.8360128402709961, + "learning_rate": 0.0001, + "loss": 64.3604, + "step": 5760 + }, + { + "epoch": 9.29183251876858, + "grad_norm": 0.9302591681480408, + "learning_rate": 0.0001, + "loss": 66.2754, + "step": 5770 + }, + { + "epoch": 9.307955862346954, + "grad_norm": 0.7000735998153687, + "learning_rate": 0.0001, + "loss": 65.2241, + "step": 5780 + }, + { + "epoch": 9.32407920592533, + "grad_norm": 0.7321175932884216, + "learning_rate": 0.0001, + "loss": 65.3558, + "step": 5790 + }, + { + "epoch": 9.340202549503703, + "grad_norm": 0.7948715090751648, + "learning_rate": 0.0001, + "loss": 64.2793, + "step": 5800 + }, + { + "epoch": 9.356325893082078, + "grad_norm": 0.7676129341125488, + "learning_rate": 0.0001, + "loss": 66.4778, + "step": 5810 + }, + { + "epoch": 9.372449236660453, + "grad_norm": 0.977236270904541, + "learning_rate": 0.0001, + "loss": 64.5587, + "step": 5820 + }, + { + "epoch": 9.388572580238828, + "grad_norm": 0.86808842420578, + "learning_rate": 0.0001, + "loss": 65.8026, + "step": 5830 + }, + { + "epoch": 9.4046959238172, + "grad_norm": 0.7455324530601501, + "learning_rate": 0.0001, + "loss": 65.066, + "step": 5840 + }, + { + "epoch": 9.420819267395576, + "grad_norm": 0.6784212589263916, + "learning_rate": 0.0001, + "loss": 65.4167, + "step": 5850 + }, + { + "epoch": 9.43694261097395, + "grad_norm": 1.033731460571289, + "learning_rate": 0.0001, + "loss": 66.4502, + "step": 5860 + }, + { + "epoch": 9.453065954552326, + "grad_norm": 0.7946107983589172, + "learning_rate": 0.0001, + "loss": 63.7567, + "step": 5870 + }, + { + "epoch": 9.4691892981307, + "grad_norm": 0.7826364636421204, + "learning_rate": 0.0001, + "loss": 65.4484, + "step": 5880 + }, + { + "epoch": 9.485312641709074, + "grad_norm": 0.7605016827583313, + "learning_rate": 0.0001, + "loss": 66.8459, + "step": 5890 + }, + { + "epoch": 9.501435985287449, + "grad_norm": 0.7383126020431519, + "learning_rate": 0.0001, + "loss": 64.6873, + "step": 5900 + }, + { + "epoch": 9.517559328865824, + "grad_norm": 0.8355686664581299, + "learning_rate": 0.0001, + "loss": 63.5145, + "step": 5910 + }, + { + "epoch": 9.533682672444199, + "grad_norm": 0.8827091455459595, + "learning_rate": 0.0001, + "loss": 67.2207, + "step": 5920 + }, + { + "epoch": 9.549806016022572, + "grad_norm": 0.6988142728805542, + "learning_rate": 0.0001, + "loss": 64.2289, + "step": 5930 + }, + { + "epoch": 9.565929359600947, + "grad_norm": 0.7609000205993652, + "learning_rate": 0.0001, + "loss": 63.9324, + "step": 5940 + }, + { + "epoch": 9.582052703179322, + "grad_norm": 0.8053853511810303, + "learning_rate": 0.0001, + "loss": 62.5564, + "step": 5950 + }, + { + "epoch": 9.598176046757697, + "grad_norm": 0.7684671878814697, + "learning_rate": 0.0001, + "loss": 66.2669, + "step": 5960 + }, + { + "epoch": 9.614299390336072, + "grad_norm": 0.8411065340042114, + "learning_rate": 0.0001, + "loss": 64.5087, + "step": 5970 + }, + { + "epoch": 9.630422733914445, + "grad_norm": 0.963498592376709, + "learning_rate": 0.0001, + "loss": 68.1736, + "step": 5980 + }, + { + "epoch": 9.64654607749282, + "grad_norm": 0.8715981841087341, + "learning_rate": 0.0001, + "loss": 64.1686, + "step": 5990 + }, + { + "epoch": 9.662669421071195, + "grad_norm": 0.8217800855636597, + "learning_rate": 0.0001, + "loss": 64.2211, + "step": 6000 + }, + { + "epoch": 9.67879276464957, + "grad_norm": 0.7824742197990417, + "learning_rate": 0.0001, + "loss": 65.9052, + "step": 6010 + }, + { + "epoch": 9.694916108227943, + "grad_norm": 0.7440009117126465, + "learning_rate": 0.0001, + "loss": 63.3749, + "step": 6020 + }, + { + "epoch": 9.711039451806318, + "grad_norm": 0.8759479522705078, + "learning_rate": 0.0001, + "loss": 63.4488, + "step": 6030 + }, + { + "epoch": 9.727162795384693, + "grad_norm": 3.662144184112549, + "learning_rate": 0.0001, + "loss": 62.8787, + "step": 6040 + }, + { + "epoch": 9.743286138963068, + "grad_norm": 1.007149577140808, + "learning_rate": 0.0001, + "loss": 66.0053, + "step": 6050 + }, + { + "epoch": 9.759409482541441, + "grad_norm": 0.8369242548942566, + "learning_rate": 0.0001, + "loss": 67.233, + "step": 6060 + }, + { + "epoch": 9.775532826119816, + "grad_norm": 0.8160434365272522, + "learning_rate": 0.0001, + "loss": 65.2089, + "step": 6070 + }, + { + "epoch": 9.791656169698191, + "grad_norm": 0.8482319712638855, + "learning_rate": 0.0001, + "loss": 65.0718, + "step": 6080 + }, + { + "epoch": 9.807779513276566, + "grad_norm": 0.8800129890441895, + "learning_rate": 0.0001, + "loss": 65.145, + "step": 6090 + }, + { + "epoch": 9.82390285685494, + "grad_norm": 0.8911405801773071, + "learning_rate": 0.0001, + "loss": 65.7952, + "step": 6100 + }, + { + "epoch": 9.840026200433314, + "grad_norm": 0.8108358979225159, + "learning_rate": 0.0001, + "loss": 65.1928, + "step": 6110 + }, + { + "epoch": 9.85614954401169, + "grad_norm": 0.8919743895530701, + "learning_rate": 0.0001, + "loss": 65.8129, + "step": 6120 + }, + { + "epoch": 9.872272887590064, + "grad_norm": 0.73152756690979, + "learning_rate": 0.0001, + "loss": 65.7141, + "step": 6130 + }, + { + "epoch": 9.88839623116844, + "grad_norm": 0.8583903312683105, + "learning_rate": 0.0001, + "loss": 64.8231, + "step": 6140 + }, + { + "epoch": 9.904519574746812, + "grad_norm": 0.798543393611908, + "learning_rate": 0.0001, + "loss": 64.5073, + "step": 6150 + }, + { + "epoch": 9.920642918325187, + "grad_norm": 0.8657358288764954, + "learning_rate": 0.0001, + "loss": 64.1207, + "step": 6160 + }, + { + "epoch": 9.936766261903562, + "grad_norm": 0.9324677586555481, + "learning_rate": 0.0001, + "loss": 64.7129, + "step": 6170 + }, + { + "epoch": 9.952889605481937, + "grad_norm": 0.7371154427528381, + "learning_rate": 0.0001, + "loss": 64.0251, + "step": 6180 + }, + { + "epoch": 9.96901294906031, + "grad_norm": 0.8273237347602844, + "learning_rate": 0.0001, + "loss": 65.8633, + "step": 6190 + }, + { + "epoch": 9.985136292638686, + "grad_norm": 0.7219365239143372, + "learning_rate": 0.0001, + "loss": 63.4029, + "step": 6200 + }, + { + "epoch": 10.0, + "grad_norm": 0.76242995262146, + "learning_rate": 0.0001, + "loss": 60.9994, + "step": 6210 + }, + { + "epoch": 10.016123343578375, + "grad_norm": 0.8677287697792053, + "learning_rate": 0.0001, + "loss": 65.9687, + "step": 6220 + }, + { + "epoch": 10.03224668715675, + "grad_norm": 0.7073521018028259, + "learning_rate": 0.0001, + "loss": 63.0083, + "step": 6230 + }, + { + "epoch": 10.048370030735123, + "grad_norm": 0.794535756111145, + "learning_rate": 0.0001, + "loss": 63.0616, + "step": 6240 + }, + { + "epoch": 10.064493374313498, + "grad_norm": 0.6974631547927856, + "learning_rate": 0.0001, + "loss": 64.6579, + "step": 6250 + }, + { + "epoch": 10.080616717891873, + "grad_norm": 0.8279150128364563, + "learning_rate": 0.0001, + "loss": 65.1069, + "step": 6260 + }, + { + "epoch": 10.096740061470248, + "grad_norm": 0.8682932257652283, + "learning_rate": 0.0001, + "loss": 64.5019, + "step": 6270 + }, + { + "epoch": 10.112863405048621, + "grad_norm": 1.0158190727233887, + "learning_rate": 0.0001, + "loss": 65.8854, + "step": 6280 + }, + { + "epoch": 10.128986748626996, + "grad_norm": 0.6911011934280396, + "learning_rate": 0.0001, + "loss": 62.4879, + "step": 6290 + }, + { + "epoch": 10.145110092205371, + "grad_norm": 0.7394022941589355, + "learning_rate": 0.0001, + "loss": 64.2604, + "step": 6300 + }, + { + "epoch": 10.161233435783746, + "grad_norm": 0.7941621541976929, + "learning_rate": 0.0001, + "loss": 64.5133, + "step": 6310 + }, + { + "epoch": 10.17735677936212, + "grad_norm": 1.0238994359970093, + "learning_rate": 0.0001, + "loss": 63.8594, + "step": 6320 + }, + { + "epoch": 10.193480122940494, + "grad_norm": 0.7332146763801575, + "learning_rate": 0.0001, + "loss": 66.5434, + "step": 6330 + }, + { + "epoch": 10.20960346651887, + "grad_norm": 0.9375078082084656, + "learning_rate": 0.0001, + "loss": 63.4793, + "step": 6340 + }, + { + "epoch": 10.225726810097244, + "grad_norm": 0.715931236743927, + "learning_rate": 0.0001, + "loss": 65.8812, + "step": 6350 + }, + { + "epoch": 10.24185015367562, + "grad_norm": 1.291536808013916, + "learning_rate": 0.0001, + "loss": 64.9125, + "step": 6360 + }, + { + "epoch": 10.257973497253992, + "grad_norm": 0.8942781686782837, + "learning_rate": 0.0001, + "loss": 67.1706, + "step": 6370 + }, + { + "epoch": 10.274096840832367, + "grad_norm": 0.8272124528884888, + "learning_rate": 0.0001, + "loss": 62.1721, + "step": 6380 + }, + { + "epoch": 10.290220184410742, + "grad_norm": 0.8191152215003967, + "learning_rate": 0.0001, + "loss": 64.8931, + "step": 6390 + }, + { + "epoch": 10.306343527989117, + "grad_norm": 1.7500241994857788, + "learning_rate": 0.0001, + "loss": 62.9693, + "step": 6400 + }, + { + "epoch": 10.32246687156749, + "grad_norm": 1.0159430503845215, + "learning_rate": 0.0001, + "loss": 63.0889, + "step": 6410 + }, + { + "epoch": 10.338590215145866, + "grad_norm": 0.7300348281860352, + "learning_rate": 0.0001, + "loss": 65.1742, + "step": 6420 + }, + { + "epoch": 10.35471355872424, + "grad_norm": 1.3132387399673462, + "learning_rate": 0.0001, + "loss": 64.8867, + "step": 6430 + }, + { + "epoch": 10.370836902302615, + "grad_norm": 0.7502451539039612, + "learning_rate": 0.0001, + "loss": 66.5094, + "step": 6440 + }, + { + "epoch": 10.38696024588099, + "grad_norm": 0.9408686757087708, + "learning_rate": 0.0001, + "loss": 65.1515, + "step": 6450 + }, + { + "epoch": 10.403083589459364, + "grad_norm": 0.769598126411438, + "learning_rate": 0.0001, + "loss": 65.1478, + "step": 6460 + }, + { + "epoch": 10.419206933037739, + "grad_norm": 0.9086162447929382, + "learning_rate": 0.0001, + "loss": 63.5263, + "step": 6470 + }, + { + "epoch": 10.435330276616114, + "grad_norm": 0.7899322509765625, + "learning_rate": 0.0001, + "loss": 63.3939, + "step": 6480 + }, + { + "epoch": 10.451453620194489, + "grad_norm": 0.9135434031486511, + "learning_rate": 0.0001, + "loss": 63.867, + "step": 6490 + }, + { + "epoch": 10.467576963772862, + "grad_norm": 0.9312737584114075, + "learning_rate": 0.0001, + "loss": 63.6939, + "step": 6500 + }, + { + "epoch": 10.483700307351237, + "grad_norm": 0.861764132976532, + "learning_rate": 0.0001, + "loss": 63.4523, + "step": 6510 + }, + { + "epoch": 10.499823650929612, + "grad_norm": 0.7056366801261902, + "learning_rate": 0.0001, + "loss": 63.2889, + "step": 6520 + }, + { + "epoch": 10.515946994507987, + "grad_norm": 0.7007710337638855, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 6530 + }, + { + "epoch": 10.53207033808636, + "grad_norm": 0.7091413736343384, + "learning_rate": 0.0001, + "loss": 63.4185, + "step": 6540 + }, + { + "epoch": 10.548193681664735, + "grad_norm": 0.7147234678268433, + "learning_rate": 0.0001, + "loss": 65.0726, + "step": 6550 + }, + { + "epoch": 10.56431702524311, + "grad_norm": 0.7954673767089844, + "learning_rate": 0.0001, + "loss": 65.0334, + "step": 6560 + }, + { + "epoch": 10.580440368821485, + "grad_norm": 0.725352942943573, + "learning_rate": 0.0001, + "loss": 62.6663, + "step": 6570 + }, + { + "epoch": 10.59656371239986, + "grad_norm": 0.9563508629798889, + "learning_rate": 0.0001, + "loss": 64.0705, + "step": 6580 + }, + { + "epoch": 10.612687055978233, + "grad_norm": 0.7778734564781189, + "learning_rate": 0.0001, + "loss": 63.7619, + "step": 6590 + }, + { + "epoch": 10.628810399556608, + "grad_norm": 0.8414369821548462, + "learning_rate": 0.0001, + "loss": 63.9238, + "step": 6600 + }, + { + "epoch": 10.644933743134983, + "grad_norm": 0.8070562481880188, + "learning_rate": 0.0001, + "loss": 63.9591, + "step": 6610 + }, + { + "epoch": 10.661057086713358, + "grad_norm": 0.6493734121322632, + "learning_rate": 0.0001, + "loss": 63.7237, + "step": 6620 + }, + { + "epoch": 10.677180430291731, + "grad_norm": 0.8263372778892517, + "learning_rate": 0.0001, + "loss": 64.2262, + "step": 6630 + }, + { + "epoch": 10.693303773870106, + "grad_norm": 1.0090227127075195, + "learning_rate": 0.0001, + "loss": 63.5469, + "step": 6640 + }, + { + "epoch": 10.709427117448481, + "grad_norm": 0.998319685459137, + "learning_rate": 0.0001, + "loss": 62.9113, + "step": 6650 + }, + { + "epoch": 10.725550461026856, + "grad_norm": 1.006489634513855, + "learning_rate": 0.0001, + "loss": 66.1458, + "step": 6660 + }, + { + "epoch": 10.741673804605231, + "grad_norm": 0.9575757384300232, + "learning_rate": 0.0001, + "loss": 63.0913, + "step": 6670 + }, + { + "epoch": 10.757797148183604, + "grad_norm": 0.6832238435745239, + "learning_rate": 0.0001, + "loss": 66.7501, + "step": 6680 + }, + { + "epoch": 10.773920491761979, + "grad_norm": 0.8075717687606812, + "learning_rate": 0.0001, + "loss": 64.2839, + "step": 6690 + }, + { + "epoch": 10.790043835340354, + "grad_norm": 0.9402624368667603, + "learning_rate": 0.0001, + "loss": 64.9296, + "step": 6700 + }, + { + "epoch": 10.806167178918729, + "grad_norm": 0.7681158185005188, + "learning_rate": 0.0001, + "loss": 63.5686, + "step": 6710 + }, + { + "epoch": 10.822290522497102, + "grad_norm": 0.8451285362243652, + "learning_rate": 0.0001, + "loss": 64.5404, + "step": 6720 + }, + { + "epoch": 10.838413866075477, + "grad_norm": 0.7278769016265869, + "learning_rate": 0.0001, + "loss": 64.1341, + "step": 6730 + }, + { + "epoch": 10.854537209653852, + "grad_norm": 0.8524075746536255, + "learning_rate": 0.0001, + "loss": 64.0483, + "step": 6740 + }, + { + "epoch": 10.870660553232227, + "grad_norm": 0.703598141670227, + "learning_rate": 0.0001, + "loss": 64.0072, + "step": 6750 + }, + { + "epoch": 10.8867838968106, + "grad_norm": 0.7740384936332703, + "learning_rate": 0.0001, + "loss": 63.4171, + "step": 6760 + }, + { + "epoch": 10.902907240388975, + "grad_norm": 0.8302356004714966, + "learning_rate": 0.0001, + "loss": 65.5159, + "step": 6770 + }, + { + "epoch": 10.91903058396735, + "grad_norm": 1.0289733409881592, + "learning_rate": 0.0001, + "loss": 64.9785, + "step": 6780 + }, + { + "epoch": 10.935153927545725, + "grad_norm": 0.6721987128257751, + "learning_rate": 0.0001, + "loss": 63.965, + "step": 6790 + }, + { + "epoch": 10.951277271124098, + "grad_norm": 0.7614948749542236, + "learning_rate": 0.0001, + "loss": 64.1375, + "step": 6800 + }, + { + "epoch": 10.967400614702473, + "grad_norm": 0.7342216372489929, + "learning_rate": 0.0001, + "loss": 64.4342, + "step": 6810 + }, + { + "epoch": 10.983523958280848, + "grad_norm": 0.8808079957962036, + "learning_rate": 0.0001, + "loss": 65.2864, + "step": 6820 + }, + { + "epoch": 10.999647301859223, + "grad_norm": 0.818477988243103, + "learning_rate": 0.0001, + "loss": 63.9931, + "step": 6830 + }, + { + "epoch": 11.014511009220538, + "grad_norm": 0.7867474555969238, + "learning_rate": 0.0001, + "loss": 59.2426, + "step": 6840 + }, + { + "epoch": 11.030634352798911, + "grad_norm": 0.6163678765296936, + "learning_rate": 0.0001, + "loss": 65.0149, + "step": 6850 + }, + { + "epoch": 11.046757696377286, + "grad_norm": 0.8334609866142273, + "learning_rate": 0.0001, + "loss": 63.5017, + "step": 6860 + }, + { + "epoch": 11.062881039955661, + "grad_norm": 0.7335264086723328, + "learning_rate": 0.0001, + "loss": 64.6307, + "step": 6870 + }, + { + "epoch": 11.079004383534036, + "grad_norm": 0.7723846435546875, + "learning_rate": 0.0001, + "loss": 64.316, + "step": 6880 + }, + { + "epoch": 11.09512772711241, + "grad_norm": 0.7972145080566406, + "learning_rate": 0.0001, + "loss": 63.1469, + "step": 6890 + }, + { + "epoch": 11.111251070690784, + "grad_norm": 0.7696133852005005, + "learning_rate": 0.0001, + "loss": 64.0338, + "step": 6900 + }, + { + "epoch": 11.127374414269159, + "grad_norm": 0.8972133994102478, + "learning_rate": 0.0001, + "loss": 65.5727, + "step": 6910 + }, + { + "epoch": 11.143497757847534, + "grad_norm": 0.6776295900344849, + "learning_rate": 0.0001, + "loss": 66.176, + "step": 6920 + }, + { + "epoch": 11.159621101425909, + "grad_norm": 0.7345874905586243, + "learning_rate": 0.0001, + "loss": 64.0949, + "step": 6930 + }, + { + "epoch": 11.175744445004282, + "grad_norm": 0.7988374829292297, + "learning_rate": 0.0001, + "loss": 64.7721, + "step": 6940 + }, + { + "epoch": 11.191867788582657, + "grad_norm": 0.7552387714385986, + "learning_rate": 0.0001, + "loss": 63.1053, + "step": 6950 + }, + { + "epoch": 11.207991132161032, + "grad_norm": 0.6388214230537415, + "learning_rate": 0.0001, + "loss": 63.7871, + "step": 6960 + }, + { + "epoch": 11.224114475739407, + "grad_norm": 0.6837960481643677, + "learning_rate": 0.0001, + "loss": 63.906, + "step": 6970 + }, + { + "epoch": 11.24023781931778, + "grad_norm": 0.7050679326057434, + "learning_rate": 0.0001, + "loss": 64.2803, + "step": 6980 + }, + { + "epoch": 11.256361162896155, + "grad_norm": 0.8619478940963745, + "learning_rate": 0.0001, + "loss": 62.8786, + "step": 6990 + }, + { + "epoch": 11.27248450647453, + "grad_norm": 0.9759813547134399, + "learning_rate": 0.0001, + "loss": 65.2299, + "step": 7000 + }, + { + "epoch": 11.288607850052905, + "grad_norm": 0.7584514021873474, + "learning_rate": 0.0001, + "loss": 61.4178, + "step": 7010 + }, + { + "epoch": 11.304731193631278, + "grad_norm": 0.8467874526977539, + "learning_rate": 0.0001, + "loss": 65.6229, + "step": 7020 + }, + { + "epoch": 11.320854537209653, + "grad_norm": 0.7121654152870178, + "learning_rate": 0.0001, + "loss": 63.9061, + "step": 7030 + }, + { + "epoch": 11.336977880788028, + "grad_norm": 0.8885557055473328, + "learning_rate": 0.0001, + "loss": 65.1436, + "step": 7040 + }, + { + "epoch": 11.353101224366403, + "grad_norm": 0.6320177912712097, + "learning_rate": 0.0001, + "loss": 62.8535, + "step": 7050 + }, + { + "epoch": 11.369224567944778, + "grad_norm": 0.9287629723548889, + "learning_rate": 0.0001, + "loss": 62.7407, + "step": 7060 + }, + { + "epoch": 11.385347911523152, + "grad_norm": 0.7705608606338501, + "learning_rate": 0.0001, + "loss": 63.469, + "step": 7070 + }, + { + "epoch": 11.401471255101526, + "grad_norm": 0.6687246561050415, + "learning_rate": 0.0001, + "loss": 62.722, + "step": 7080 + }, + { + "epoch": 11.417594598679901, + "grad_norm": 0.7774648666381836, + "learning_rate": 0.0001, + "loss": 63.1145, + "step": 7090 + }, + { + "epoch": 11.433717942258276, + "grad_norm": 0.7808363437652588, + "learning_rate": 0.0001, + "loss": 63.9342, + "step": 7100 + }, + { + "epoch": 11.44984128583665, + "grad_norm": 0.8033562302589417, + "learning_rate": 0.0001, + "loss": 65.7876, + "step": 7110 + }, + { + "epoch": 11.465964629415025, + "grad_norm": 0.8143486976623535, + "learning_rate": 0.0001, + "loss": 64.8782, + "step": 7120 + }, + { + "epoch": 11.4820879729934, + "grad_norm": 0.7855561971664429, + "learning_rate": 0.0001, + "loss": 63.4944, + "step": 7130 + }, + { + "epoch": 11.498211316571775, + "grad_norm": 0.7525351047515869, + "learning_rate": 0.0001, + "loss": 62.4088, + "step": 7140 + }, + { + "epoch": 11.514334660150148, + "grad_norm": 0.7193494439125061, + "learning_rate": 0.0001, + "loss": 64.2386, + "step": 7150 + }, + { + "epoch": 11.530458003728523, + "grad_norm": 0.892281174659729, + "learning_rate": 0.0001, + "loss": 63.8075, + "step": 7160 + }, + { + "epoch": 11.546581347306898, + "grad_norm": 0.720287561416626, + "learning_rate": 0.0001, + "loss": 63.3259, + "step": 7170 + }, + { + "epoch": 11.562704690885273, + "grad_norm": 1.4404689073562622, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 7180 + }, + { + "epoch": 11.578828034463648, + "grad_norm": 0.9082726240158081, + "learning_rate": 0.0001, + "loss": 64.6877, + "step": 7190 + }, + { + "epoch": 11.59495137804202, + "grad_norm": 0.7137670516967773, + "learning_rate": 0.0001, + "loss": 63.4832, + "step": 7200 + }, + { + "epoch": 11.611074721620396, + "grad_norm": 0.6848503947257996, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 7210 + }, + { + "epoch": 11.62719806519877, + "grad_norm": 0.7483687996864319, + "learning_rate": 0.0001, + "loss": 63.294, + "step": 7220 + }, + { + "epoch": 11.643321408777146, + "grad_norm": 0.935321569442749, + "learning_rate": 0.0001, + "loss": 62.8669, + "step": 7230 + }, + { + "epoch": 11.659444752355519, + "grad_norm": 0.8359326124191284, + "learning_rate": 0.0001, + "loss": 64.4482, + "step": 7240 + }, + { + "epoch": 11.675568095933894, + "grad_norm": 0.7838515043258667, + "learning_rate": 0.0001, + "loss": 64.1975, + "step": 7250 + }, + { + "epoch": 11.691691439512269, + "grad_norm": 0.9128443598747253, + "learning_rate": 0.0001, + "loss": 63.1822, + "step": 7260 + }, + { + "epoch": 11.707814783090644, + "grad_norm": 1.0545419454574585, + "learning_rate": 0.0001, + "loss": 63.3112, + "step": 7270 + }, + { + "epoch": 11.723938126669019, + "grad_norm": 0.8572486639022827, + "learning_rate": 0.0001, + "loss": 64.9605, + "step": 7280 + }, + { + "epoch": 11.740061470247392, + "grad_norm": 1.0894116163253784, + "learning_rate": 0.0001, + "loss": 64.3671, + "step": 7290 + }, + { + "epoch": 11.756184813825767, + "grad_norm": 0.7500355839729309, + "learning_rate": 0.0001, + "loss": 62.6381, + "step": 7300 + }, + { + "epoch": 11.772308157404142, + "grad_norm": 0.7315185070037842, + "learning_rate": 0.0001, + "loss": 66.1438, + "step": 7310 + }, + { + "epoch": 11.788431500982517, + "grad_norm": 0.8123629093170166, + "learning_rate": 0.0001, + "loss": 62.6012, + "step": 7320 + }, + { + "epoch": 11.80455484456089, + "grad_norm": 0.8072763085365295, + "learning_rate": 0.0001, + "loss": 63.4424, + "step": 7330 + }, + { + "epoch": 11.820678188139265, + "grad_norm": 1.1077874898910522, + "learning_rate": 0.0001, + "loss": 63.2812, + "step": 7340 + }, + { + "epoch": 11.83680153171764, + "grad_norm": 0.7116726636886597, + "learning_rate": 0.0001, + "loss": 63.6077, + "step": 7350 + }, + { + "epoch": 11.852924875296015, + "grad_norm": 0.8817790746688843, + "learning_rate": 0.0001, + "loss": 63.1938, + "step": 7360 + }, + { + "epoch": 11.86904821887439, + "grad_norm": 1.021730661392212, + "learning_rate": 0.0001, + "loss": 65.2215, + "step": 7370 + }, + { + "epoch": 11.885171562452763, + "grad_norm": 0.9871250987052917, + "learning_rate": 0.0001, + "loss": 64.6468, + "step": 7380 + }, + { + "epoch": 11.901294906031138, + "grad_norm": 1.3591430187225342, + "learning_rate": 0.0001, + "loss": 61.5546, + "step": 7390 + }, + { + "epoch": 11.917418249609513, + "grad_norm": 0.8114227056503296, + "learning_rate": 0.0001, + "loss": 61.8215, + "step": 7400 + }, + { + "epoch": 11.933541593187888, + "grad_norm": 0.7151288986206055, + "learning_rate": 0.0001, + "loss": 61.4856, + "step": 7410 + }, + { + "epoch": 11.949664936766261, + "grad_norm": 0.7494287490844727, + "learning_rate": 0.0001, + "loss": 63.6451, + "step": 7420 + }, + { + "epoch": 11.965788280344636, + "grad_norm": 0.7377745509147644, + "learning_rate": 0.0001, + "loss": 63.002, + "step": 7430 + }, + { + "epoch": 11.981911623923011, + "grad_norm": 0.6534659266471863, + "learning_rate": 0.0001, + "loss": 63.6404, + "step": 7440 + }, + { + "epoch": 11.998034967501386, + "grad_norm": 0.7548736333847046, + "learning_rate": 0.0001, + "loss": 64.1986, + "step": 7450 + }, + { + "epoch": 12.012898674862699, + "grad_norm": 0.7227553129196167, + "learning_rate": 0.0001, + "loss": 56.8327, + "step": 7460 + }, + { + "epoch": 12.029022018441074, + "grad_norm": 0.6348695158958435, + "learning_rate": 0.0001, + "loss": 62.4472, + "step": 7470 + }, + { + "epoch": 12.045145362019449, + "grad_norm": 0.8249921202659607, + "learning_rate": 0.0001, + "loss": 65.1877, + "step": 7480 + }, + { + "epoch": 12.061268705597824, + "grad_norm": 0.7152777314186096, + "learning_rate": 0.0001, + "loss": 64.2609, + "step": 7490 + }, + { + "epoch": 12.077392049176197, + "grad_norm": 0.7399184703826904, + "learning_rate": 0.0001, + "loss": 63.3997, + "step": 7500 + }, + { + "epoch": 12.093515392754572, + "grad_norm": 0.6972354650497437, + "learning_rate": 0.0001, + "loss": 61.3573, + "step": 7510 + }, + { + "epoch": 12.109638736332947, + "grad_norm": 0.7666720151901245, + "learning_rate": 0.0001, + "loss": 63.8438, + "step": 7520 + }, + { + "epoch": 12.125762079911322, + "grad_norm": 0.7309302687644958, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 7530 + }, + { + "epoch": 12.141885423489697, + "grad_norm": 1.3805177211761475, + "learning_rate": 0.0001, + "loss": 64.68, + "step": 7540 + }, + { + "epoch": 12.15800876706807, + "grad_norm": 1.0315922498703003, + "learning_rate": 0.0001, + "loss": 63.6372, + "step": 7550 + }, + { + "epoch": 12.174132110646445, + "grad_norm": 1.2012155055999756, + "learning_rate": 0.0001, + "loss": 62.5765, + "step": 7560 + }, + { + "epoch": 12.19025545422482, + "grad_norm": 0.7769846320152283, + "learning_rate": 0.0001, + "loss": 63.9737, + "step": 7570 + }, + { + "epoch": 12.206378797803195, + "grad_norm": 0.6587379574775696, + "learning_rate": 0.0001, + "loss": 63.382, + "step": 7580 + }, + { + "epoch": 12.222502141381568, + "grad_norm": 0.8309435248374939, + "learning_rate": 0.0001, + "loss": 64.4027, + "step": 7590 + }, + { + "epoch": 12.238625484959943, + "grad_norm": 0.8221393823623657, + "learning_rate": 0.0001, + "loss": 62.8561, + "step": 7600 + }, + { + "epoch": 12.254748828538318, + "grad_norm": 0.8629311323165894, + "learning_rate": 0.0001, + "loss": 64.3673, + "step": 7610 + }, + { + "epoch": 12.270872172116693, + "grad_norm": 0.698113739490509, + "learning_rate": 0.0001, + "loss": 63.6153, + "step": 7620 + }, + { + "epoch": 12.286995515695068, + "grad_norm": 0.72906094789505, + "learning_rate": 0.0001, + "loss": 64.8439, + "step": 7630 + }, + { + "epoch": 12.303118859273441, + "grad_norm": 0.7402384877204895, + "learning_rate": 0.0001, + "loss": 64.2103, + "step": 7640 + }, + { + "epoch": 12.319242202851816, + "grad_norm": 0.9972009062767029, + "learning_rate": 0.0001, + "loss": 63.803, + "step": 7650 + }, + { + "epoch": 12.335365546430191, + "grad_norm": 0.7852274775505066, + "learning_rate": 0.0001, + "loss": 64.8144, + "step": 7660 + }, + { + "epoch": 12.351488890008566, + "grad_norm": 0.7433444857597351, + "learning_rate": 0.0001, + "loss": 63.0123, + "step": 7670 + }, + { + "epoch": 12.36761223358694, + "grad_norm": 0.8548759818077087, + "learning_rate": 0.0001, + "loss": 63.2215, + "step": 7680 + }, + { + "epoch": 12.383735577165314, + "grad_norm": 0.6759565472602844, + "learning_rate": 0.0001, + "loss": 62.1246, + "step": 7690 + }, + { + "epoch": 12.39985892074369, + "grad_norm": 0.9079536199569702, + "learning_rate": 0.0001, + "loss": 64.1692, + "step": 7700 + }, + { + "epoch": 12.415982264322064, + "grad_norm": 0.6890071034431458, + "learning_rate": 0.0001, + "loss": 62.5602, + "step": 7710 + }, + { + "epoch": 12.432105607900438, + "grad_norm": 0.8096348643302917, + "learning_rate": 0.0001, + "loss": 61.0103, + "step": 7720 + }, + { + "epoch": 12.448228951478812, + "grad_norm": 0.7359873056411743, + "learning_rate": 0.0001, + "loss": 62.1914, + "step": 7730 + }, + { + "epoch": 12.464352295057187, + "grad_norm": 1.0701091289520264, + "learning_rate": 0.0001, + "loss": 61.7295, + "step": 7740 + }, + { + "epoch": 12.480475638635562, + "grad_norm": 0.791928768157959, + "learning_rate": 0.0001, + "loss": 62.6812, + "step": 7750 + }, + { + "epoch": 12.496598982213937, + "grad_norm": 0.7719446420669556, + "learning_rate": 0.0001, + "loss": 61.9114, + "step": 7760 + }, + { + "epoch": 12.51272232579231, + "grad_norm": 0.8720338344573975, + "learning_rate": 0.0001, + "loss": 64.8107, + "step": 7770 + }, + { + "epoch": 12.528845669370686, + "grad_norm": 0.9999920129776001, + "learning_rate": 0.0001, + "loss": 61.198, + "step": 7780 + }, + { + "epoch": 12.54496901294906, + "grad_norm": 0.710754930973053, + "learning_rate": 0.0001, + "loss": 64.1462, + "step": 7790 + }, + { + "epoch": 12.561092356527435, + "grad_norm": 0.7401363849639893, + "learning_rate": 0.0001, + "loss": 61.8988, + "step": 7800 + }, + { + "epoch": 12.577215700105809, + "grad_norm": 0.9183729290962219, + "learning_rate": 0.0001, + "loss": 62.6392, + "step": 7810 + }, + { + "epoch": 12.593339043684184, + "grad_norm": 0.7341399788856506, + "learning_rate": 0.0001, + "loss": 62.7068, + "step": 7820 + }, + { + "epoch": 12.609462387262559, + "grad_norm": 0.8066722750663757, + "learning_rate": 0.0001, + "loss": 64.0625, + "step": 7830 + }, + { + "epoch": 12.625585730840934, + "grad_norm": 0.6088835597038269, + "learning_rate": 0.0001, + "loss": 64.3208, + "step": 7840 + }, + { + "epoch": 12.641709074419307, + "grad_norm": 0.6969302892684937, + "learning_rate": 0.0001, + "loss": 62.5817, + "step": 7850 + }, + { + "epoch": 12.657832417997682, + "grad_norm": 0.82635098695755, + "learning_rate": 0.0001, + "loss": 62.0061, + "step": 7860 + }, + { + "epoch": 12.673955761576057, + "grad_norm": 0.7016922235488892, + "learning_rate": 0.0001, + "loss": 62.5166, + "step": 7870 + }, + { + "epoch": 12.690079105154432, + "grad_norm": 0.8035777807235718, + "learning_rate": 0.0001, + "loss": 63.6543, + "step": 7880 + }, + { + "epoch": 12.706202448732807, + "grad_norm": 0.8573809862136841, + "learning_rate": 0.0001, + "loss": 62.5258, + "step": 7890 + }, + { + "epoch": 12.72232579231118, + "grad_norm": 0.9497472643852234, + "learning_rate": 0.0001, + "loss": 63.1133, + "step": 7900 + }, + { + "epoch": 12.738449135889555, + "grad_norm": 0.7494457960128784, + "learning_rate": 0.0001, + "loss": 63.8157, + "step": 7910 + }, + { + "epoch": 12.75457247946793, + "grad_norm": 0.7657570242881775, + "learning_rate": 0.0001, + "loss": 62.6255, + "step": 7920 + }, + { + "epoch": 12.770695823046305, + "grad_norm": 0.7848380208015442, + "learning_rate": 0.0001, + "loss": 62.9095, + "step": 7930 + }, + { + "epoch": 12.786819166624678, + "grad_norm": 0.8386877775192261, + "learning_rate": 0.0001, + "loss": 63.2254, + "step": 7940 + }, + { + "epoch": 12.802942510203053, + "grad_norm": 0.6774776577949524, + "learning_rate": 0.0001, + "loss": 64.3929, + "step": 7950 + }, + { + "epoch": 12.819065853781428, + "grad_norm": 0.798547625541687, + "learning_rate": 0.0001, + "loss": 62.9884, + "step": 7960 + }, + { + "epoch": 12.835189197359803, + "grad_norm": 0.9340888261795044, + "learning_rate": 0.0001, + "loss": 62.6197, + "step": 7970 + }, + { + "epoch": 12.851312540938178, + "grad_norm": 0.8211432695388794, + "learning_rate": 0.0001, + "loss": 62.7756, + "step": 7980 + }, + { + "epoch": 12.867435884516551, + "grad_norm": 0.7612001299858093, + "learning_rate": 0.0001, + "loss": 62.0792, + "step": 7990 + }, + { + "epoch": 12.883559228094926, + "grad_norm": 0.7743386626243591, + "learning_rate": 0.0001, + "loss": 63.9203, + "step": 8000 + }, + { + "epoch": 12.899682571673301, + "grad_norm": 0.7419672608375549, + "learning_rate": 0.0001, + "loss": 62.1705, + "step": 8010 + }, + { + "epoch": 12.915805915251676, + "grad_norm": 0.6556256413459778, + "learning_rate": 0.0001, + "loss": 64.5863, + "step": 8020 + }, + { + "epoch": 12.93192925883005, + "grad_norm": 1.136175274848938, + "learning_rate": 0.0001, + "loss": 63.2619, + "step": 8030 + }, + { + "epoch": 12.948052602408424, + "grad_norm": 0.8631499409675598, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8040 + }, + { + "epoch": 12.9641759459868, + "grad_norm": 0.8386172652244568, + "learning_rate": 0.0001, + "loss": 63.0598, + "step": 8050 + }, + { + "epoch": 12.980299289565174, + "grad_norm": 0.8078306913375854, + "learning_rate": 0.0001, + "loss": 62.6877, + "step": 8060 + }, + { + "epoch": 12.996422633143549, + "grad_norm": 0.8486189842224121, + "learning_rate": 0.0001, + "loss": 62.8643, + "step": 8070 + }, + { + "epoch": 13.011286340504862, + "grad_norm": 0.8717893958091736, + "learning_rate": 0.0001, + "loss": 57.6947, + "step": 8080 + }, + { + "epoch": 13.027409684083237, + "grad_norm": 1.2023789882659912, + "learning_rate": 0.0001, + "loss": 60.4728, + "step": 8090 + }, + { + "epoch": 13.043533027661612, + "grad_norm": 0.6879557967185974, + "learning_rate": 0.0001, + "loss": 61.7643, + "step": 8100 + }, + { + "epoch": 13.059656371239987, + "grad_norm": 1.0918916463851929, + "learning_rate": 0.0001, + "loss": 65.3682, + "step": 8110 + }, + { + "epoch": 13.07577971481836, + "grad_norm": 0.85753333568573, + "learning_rate": 0.0001, + "loss": 63.0815, + "step": 8120 + }, + { + "epoch": 13.091903058396735, + "grad_norm": 0.6568583846092224, + "learning_rate": 0.0001, + "loss": 64.7167, + "step": 8130 + }, + { + "epoch": 13.10802640197511, + "grad_norm": 0.6178144812583923, + "learning_rate": 0.0001, + "loss": 62.4294, + "step": 8140 + }, + { + "epoch": 13.124149745553485, + "grad_norm": 0.7930784821510315, + "learning_rate": 0.0001, + "loss": 64.542, + "step": 8150 + }, + { + "epoch": 13.140273089131858, + "grad_norm": 0.7675037980079651, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8160 + }, + { + "epoch": 13.156396432710233, + "grad_norm": 0.786997377872467, + "learning_rate": 0.0001, + "loss": 64.1495, + "step": 8170 + }, + { + "epoch": 13.172519776288608, + "grad_norm": 0.7752560377120972, + "learning_rate": 0.0001, + "loss": 61.6483, + "step": 8180 + }, + { + "epoch": 13.188643119866983, + "grad_norm": 0.8455203771591187, + "learning_rate": 0.0001, + "loss": 63.3551, + "step": 8190 + }, + { + "epoch": 13.204766463445356, + "grad_norm": 0.7223097681999207, + "learning_rate": 0.0001, + "loss": 63.1473, + "step": 8200 + }, + { + "epoch": 13.220889807023731, + "grad_norm": 0.7688442468643188, + "learning_rate": 0.0001, + "loss": 62.6723, + "step": 8210 + }, + { + "epoch": 13.237013150602106, + "grad_norm": 0.7730395793914795, + "learning_rate": 0.0001, + "loss": 64.3424, + "step": 8220 + }, + { + "epoch": 13.253136494180481, + "grad_norm": 0.8479727506637573, + "learning_rate": 0.0001, + "loss": 61.4336, + "step": 8230 + }, + { + "epoch": 13.269259837758856, + "grad_norm": 0.9070560932159424, + "learning_rate": 0.0001, + "loss": 63.1221, + "step": 8240 + }, + { + "epoch": 13.28538318133723, + "grad_norm": 0.8924600481987, + "learning_rate": 0.0001, + "loss": 63.9172, + "step": 8250 + }, + { + "epoch": 13.301506524915604, + "grad_norm": 1.0688443183898926, + "learning_rate": 0.0001, + "loss": 63.7944, + "step": 8260 + }, + { + "epoch": 13.31762986849398, + "grad_norm": 0.6866253018379211, + "learning_rate": 0.0001, + "loss": 65.5847, + "step": 8270 + }, + { + "epoch": 13.333753212072354, + "grad_norm": 0.711171567440033, + "learning_rate": 0.0001, + "loss": 62.698, + "step": 8280 + }, + { + "epoch": 13.349876555650727, + "grad_norm": 0.8128448724746704, + "learning_rate": 0.0001, + "loss": 61.5992, + "step": 8290 + }, + { + "epoch": 13.365999899229102, + "grad_norm": 0.7483755350112915, + "learning_rate": 0.0001, + "loss": 63.782, + "step": 8300 + }, + { + "epoch": 13.382123242807477, + "grad_norm": 0.7872599363327026, + "learning_rate": 0.0001, + "loss": 65.4922, + "step": 8310 + }, + { + "epoch": 13.398246586385852, + "grad_norm": 0.6861543655395508, + "learning_rate": 0.0001, + "loss": 62.6946, + "step": 8320 + }, + { + "epoch": 13.414369929964227, + "grad_norm": 0.7192599773406982, + "learning_rate": 0.0001, + "loss": 62.894, + "step": 8330 + }, + { + "epoch": 13.4304932735426, + "grad_norm": 0.7665286064147949, + "learning_rate": 0.0001, + "loss": 62.6253, + "step": 8340 + }, + { + "epoch": 13.446616617120975, + "grad_norm": 0.7909309267997742, + "learning_rate": 0.0001, + "loss": 62.5355, + "step": 8350 + }, + { + "epoch": 13.46273996069935, + "grad_norm": 0.7339541912078857, + "learning_rate": 0.0001, + "loss": 62.8514, + "step": 8360 + }, + { + "epoch": 13.478863304277725, + "grad_norm": 0.9893004298210144, + "learning_rate": 0.0001, + "loss": 63.7751, + "step": 8370 + }, + { + "epoch": 13.494986647856098, + "grad_norm": 0.9109547734260559, + "learning_rate": 0.0001, + "loss": 62.6697, + "step": 8380 + }, + { + "epoch": 13.511109991434473, + "grad_norm": 0.6807034015655518, + "learning_rate": 0.0001, + "loss": 62.1418, + "step": 8390 + }, + { + "epoch": 13.527233335012848, + "grad_norm": 0.915533185005188, + "learning_rate": 0.0001, + "loss": 61.7645, + "step": 8400 + }, + { + "epoch": 13.543356678591223, + "grad_norm": 0.8662852048873901, + "learning_rate": 0.0001, + "loss": 62.5546, + "step": 8410 + }, + { + "epoch": 13.559480022169598, + "grad_norm": 0.8678365349769592, + "learning_rate": 0.0001, + "loss": 63.8795, + "step": 8420 + }, + { + "epoch": 13.575603365747972, + "grad_norm": 0.6808590888977051, + "learning_rate": 0.0001, + "loss": 64.158, + "step": 8430 + }, + { + "epoch": 13.591726709326347, + "grad_norm": 0.7896480560302734, + "learning_rate": 0.0001, + "loss": 62.1052, + "step": 8440 + }, + { + "epoch": 13.607850052904721, + "grad_norm": 0.8973979949951172, + "learning_rate": 0.0001, + "loss": 64.478, + "step": 8450 + }, + { + "epoch": 13.623973396483096, + "grad_norm": 0.8228474259376526, + "learning_rate": 0.0001, + "loss": 61.2416, + "step": 8460 + }, + { + "epoch": 13.64009674006147, + "grad_norm": 0.9414492845535278, + "learning_rate": 0.0001, + "loss": 62.816, + "step": 8470 + }, + { + "epoch": 13.656220083639845, + "grad_norm": 0.9369436502456665, + "learning_rate": 0.0001, + "loss": 63.4951, + "step": 8480 + }, + { + "epoch": 13.67234342721822, + "grad_norm": 0.7152880430221558, + "learning_rate": 0.0001, + "loss": 62.4737, + "step": 8490 + }, + { + "epoch": 13.688466770796595, + "grad_norm": 0.7364594340324402, + "learning_rate": 0.0001, + "loss": 63.8688, + "step": 8500 + }, + { + "epoch": 13.704590114374968, + "grad_norm": 0.8499143123626709, + "learning_rate": 0.0001, + "loss": 64.5654, + "step": 8510 + }, + { + "epoch": 13.720713457953343, + "grad_norm": 0.7699156999588013, + "learning_rate": 0.0001, + "loss": 63.3726, + "step": 8520 + }, + { + "epoch": 13.736836801531718, + "grad_norm": 0.9146804809570312, + "learning_rate": 0.0001, + "loss": 63.6565, + "step": 8530 + }, + { + "epoch": 13.752960145110093, + "grad_norm": 0.773318350315094, + "learning_rate": 0.0001, + "loss": 65.0719, + "step": 8540 + }, + { + "epoch": 13.769083488688466, + "grad_norm": 0.6158450245857239, + "learning_rate": 0.0001, + "loss": 60.7291, + "step": 8550 + }, + { + "epoch": 13.78520683226684, + "grad_norm": 0.8053523302078247, + "learning_rate": 0.0001, + "loss": 62.5622, + "step": 8560 + }, + { + "epoch": 13.801330175845216, + "grad_norm": 0.8983457684516907, + "learning_rate": 0.0001, + "loss": 64.2921, + "step": 8570 + }, + { + "epoch": 13.81745351942359, + "grad_norm": 0.8214695453643799, + "learning_rate": 0.0001, + "loss": 62.2649, + "step": 8580 + }, + { + "epoch": 13.833576863001966, + "grad_norm": 0.8079172372817993, + "learning_rate": 0.0001, + "loss": 64.0828, + "step": 8590 + }, + { + "epoch": 13.849700206580339, + "grad_norm": 1.126568078994751, + "learning_rate": 0.0001, + "loss": 62.3927, + "step": 8600 + }, + { + "epoch": 13.865823550158714, + "grad_norm": 0.7597793936729431, + "learning_rate": 0.0001, + "loss": 62.5922, + "step": 8610 + }, + { + "epoch": 13.881946893737089, + "grad_norm": 0.671531617641449, + "learning_rate": 0.0001, + "loss": 61.9885, + "step": 8620 + }, + { + "epoch": 13.898070237315464, + "grad_norm": 0.823423445224762, + "learning_rate": 0.0001, + "loss": 62.0632, + "step": 8630 + }, + { + "epoch": 13.914193580893837, + "grad_norm": 0.7694658637046814, + "learning_rate": 0.0001, + "loss": 64.5485, + "step": 8640 + }, + { + "epoch": 13.930316924472212, + "grad_norm": 0.7394612431526184, + "learning_rate": 0.0001, + "loss": 61.9626, + "step": 8650 + }, + { + "epoch": 13.946440268050587, + "grad_norm": 0.724054217338562, + "learning_rate": 0.0001, + "loss": 62.3761, + "step": 8660 + }, + { + "epoch": 13.962563611628962, + "grad_norm": 0.7160128951072693, + "learning_rate": 0.0001, + "loss": 60.5142, + "step": 8670 + }, + { + "epoch": 13.978686955207337, + "grad_norm": 1.0045746564865112, + "learning_rate": 0.0001, + "loss": 62.7059, + "step": 8680 + }, + { + "epoch": 13.99481029878571, + "grad_norm": 0.6980776190757751, + "learning_rate": 0.0001, + "loss": 62.9427, + "step": 8690 + }, + { + "epoch": 14.009674006147025, + "grad_norm": 0.810961127281189, + "learning_rate": 0.0001, + "loss": 58.7552, + "step": 8700 + }, + { + "epoch": 14.0257973497254, + "grad_norm": 0.9210551381111145, + "learning_rate": 0.0001, + "loss": 63.7412, + "step": 8710 + }, + { + "epoch": 14.041920693303775, + "grad_norm": 0.7805785536766052, + "learning_rate": 0.0001, + "loss": 61.477, + "step": 8720 + }, + { + "epoch": 14.058044036882148, + "grad_norm": 0.8528208136558533, + "learning_rate": 0.0001, + "loss": 62.6746, + "step": 8730 + }, + { + "epoch": 14.074167380460523, + "grad_norm": 0.7467619776725769, + "learning_rate": 0.0001, + "loss": 63.8744, + "step": 8740 + }, + { + "epoch": 14.090290724038898, + "grad_norm": 0.6434402465820312, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 8750 + }, + { + "epoch": 14.106414067617273, + "grad_norm": 0.7217757701873779, + "learning_rate": 0.0001, + "loss": 64.0428, + "step": 8760 + }, + { + "epoch": 14.122537411195646, + "grad_norm": 0.8665125966072083, + "learning_rate": 0.0001, + "loss": 62.2772, + "step": 8770 + }, + { + "epoch": 14.13866075477402, + "grad_norm": 0.7868321537971497, + "learning_rate": 0.0001, + "loss": 62.0509, + "step": 8780 + }, + { + "epoch": 14.154784098352396, + "grad_norm": 0.8716760873794556, + "learning_rate": 0.0001, + "loss": 61.6021, + "step": 8790 + }, + { + "epoch": 14.17090744193077, + "grad_norm": 1.0922521352767944, + "learning_rate": 0.0001, + "loss": 64.4197, + "step": 8800 + }, + { + "epoch": 14.187030785509146, + "grad_norm": 0.6545577049255371, + "learning_rate": 0.0001, + "loss": 62.9259, + "step": 8810 + }, + { + "epoch": 14.203154129087519, + "grad_norm": 0.7634063959121704, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 8820 + }, + { + "epoch": 14.219277472665894, + "grad_norm": 0.9075300097465515, + "learning_rate": 0.0001, + "loss": 62.9128, + "step": 8830 + }, + { + "epoch": 14.235400816244269, + "grad_norm": 3.475320816040039, + "learning_rate": 0.0001, + "loss": 62.4841, + "step": 8840 + }, + { + "epoch": 14.251524159822644, + "grad_norm": 0.6839455962181091, + "learning_rate": 0.0001, + "loss": 61.5898, + "step": 8850 + }, + { + "epoch": 14.267647503401017, + "grad_norm": 0.8024231791496277, + "learning_rate": 0.0001, + "loss": 61.1875, + "step": 8860 + }, + { + "epoch": 14.283770846979392, + "grad_norm": 0.8948174118995667, + "learning_rate": 0.0001, + "loss": 62.3386, + "step": 8870 + }, + { + "epoch": 14.299894190557767, + "grad_norm": 0.7571985721588135, + "learning_rate": 0.0001, + "loss": 60.579, + "step": 8880 + }, + { + "epoch": 14.316017534136142, + "grad_norm": 1.432371973991394, + "learning_rate": 0.0001, + "loss": 62.4405, + "step": 8890 + }, + { + "epoch": 14.332140877714515, + "grad_norm": 0.6696875691413879, + "learning_rate": 0.0001, + "loss": 62.9984, + "step": 8900 + }, + { + "epoch": 14.34826422129289, + "grad_norm": 0.749396026134491, + "learning_rate": 0.0001, + "loss": 61.492, + "step": 8910 + }, + { + "epoch": 14.364387564871265, + "grad_norm": 0.664193332195282, + "learning_rate": 0.0001, + "loss": 62.9893, + "step": 8920 + }, + { + "epoch": 14.38051090844964, + "grad_norm": 0.7032697796821594, + "learning_rate": 0.0001, + "loss": 61.1682, + "step": 8930 + }, + { + "epoch": 14.396634252028015, + "grad_norm": 0.813119649887085, + "learning_rate": 0.0001, + "loss": 62.906, + "step": 8940 + }, + { + "epoch": 14.412757595606388, + "grad_norm": 0.7699596881866455, + "learning_rate": 0.0001, + "loss": 64.8087, + "step": 8950 + }, + { + "epoch": 14.428880939184763, + "grad_norm": 0.6424354910850525, + "learning_rate": 0.0001, + "loss": 62.2418, + "step": 8960 + }, + { + "epoch": 14.445004282763138, + "grad_norm": 0.8741731643676758, + "learning_rate": 0.0001, + "loss": 62.656, + "step": 8970 + }, + { + "epoch": 14.461127626341513, + "grad_norm": 0.710617184638977, + "learning_rate": 0.0001, + "loss": 60.8979, + "step": 8980 + }, + { + "epoch": 14.477250969919886, + "grad_norm": 0.6493995785713196, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 8990 + }, + { + "epoch": 14.493374313498261, + "grad_norm": 0.6655835509300232, + "learning_rate": 0.0001, + "loss": 64.151, + "step": 9000 + }, + { + "epoch": 14.509497657076636, + "grad_norm": 0.8140881657600403, + "learning_rate": 0.0001, + "loss": 62.4447, + "step": 9010 + }, + { + "epoch": 14.525621000655011, + "grad_norm": 0.7288498878479004, + "learning_rate": 0.0001, + "loss": 62.5189, + "step": 9020 + }, + { + "epoch": 14.541744344233386, + "grad_norm": 0.6972017884254456, + "learning_rate": 0.0001, + "loss": 61.5441, + "step": 9030 + }, + { + "epoch": 14.55786768781176, + "grad_norm": 0.7444366216659546, + "learning_rate": 0.0001, + "loss": 62.3233, + "step": 9040 + }, + { + "epoch": 14.573991031390134, + "grad_norm": 0.6602845191955566, + "learning_rate": 0.0001, + "loss": 61.4176, + "step": 9050 + }, + { + "epoch": 14.59011437496851, + "grad_norm": 0.7149770855903625, + "learning_rate": 0.0001, + "loss": 62.159, + "step": 9060 + }, + { + "epoch": 14.606237718546884, + "grad_norm": 0.6975772976875305, + "learning_rate": 0.0001, + "loss": 63.1857, + "step": 9070 + }, + { + "epoch": 14.622361062125258, + "grad_norm": 0.7835262417793274, + "learning_rate": 0.0001, + "loss": 61.6795, + "step": 9080 + }, + { + "epoch": 14.638484405703633, + "grad_norm": 5.790656566619873, + "learning_rate": 0.0001, + "loss": 62.7087, + "step": 9090 + }, + { + "epoch": 14.654607749282007, + "grad_norm": 1.0082288980484009, + "learning_rate": 0.0001, + "loss": 61.6808, + "step": 9100 + }, + { + "epoch": 14.670731092860382, + "grad_norm": 0.7758098840713501, + "learning_rate": 0.0001, + "loss": 61.0734, + "step": 9110 + }, + { + "epoch": 14.686854436438756, + "grad_norm": 0.7841722965240479, + "learning_rate": 0.0001, + "loss": 62.8882, + "step": 9120 + }, + { + "epoch": 14.70297778001713, + "grad_norm": 0.7695890069007874, + "learning_rate": 0.0001, + "loss": 60.7811, + "step": 9130 + }, + { + "epoch": 14.719101123595506, + "grad_norm": 0.7444761395454407, + "learning_rate": 0.0001, + "loss": 61.692, + "step": 9140 + }, + { + "epoch": 14.73522446717388, + "grad_norm": 0.7036821246147156, + "learning_rate": 0.0001, + "loss": 63.1698, + "step": 9150 + }, + { + "epoch": 14.751347810752256, + "grad_norm": 0.7891719937324524, + "learning_rate": 0.0001, + "loss": 63.8575, + "step": 9160 + }, + { + "epoch": 14.767471154330629, + "grad_norm": 0.6371109485626221, + "learning_rate": 0.0001, + "loss": 61.3358, + "step": 9170 + }, + { + "epoch": 14.783594497909004, + "grad_norm": 0.6551769375801086, + "learning_rate": 0.0001, + "loss": 62.2705, + "step": 9180 + }, + { + "epoch": 14.799717841487379, + "grad_norm": 0.6922310590744019, + "learning_rate": 0.0001, + "loss": 61.6442, + "step": 9190 + }, + { + "epoch": 14.815841185065754, + "grad_norm": 0.9615464210510254, + "learning_rate": 0.0001, + "loss": 60.932, + "step": 9200 + }, + { + "epoch": 14.831964528644127, + "grad_norm": 0.6367932558059692, + "learning_rate": 0.0001, + "loss": 61.3475, + "step": 9210 + }, + { + "epoch": 14.848087872222502, + "grad_norm": 0.5634505748748779, + "learning_rate": 0.0001, + "loss": 61.7562, + "step": 9220 + }, + { + "epoch": 14.864211215800877, + "grad_norm": 0.7021980285644531, + "learning_rate": 0.0001, + "loss": 62.0672, + "step": 9230 + }, + { + "epoch": 14.880334559379252, + "grad_norm": 0.8832064867019653, + "learning_rate": 0.0001, + "loss": 60.9882, + "step": 9240 + }, + { + "epoch": 14.896457902957625, + "grad_norm": 0.9043561220169067, + "learning_rate": 0.0001, + "loss": 63.0968, + "step": 9250 + }, + { + "epoch": 14.912581246536, + "grad_norm": 0.8958032131195068, + "learning_rate": 0.0001, + "loss": 61.5431, + "step": 9260 + }, + { + "epoch": 14.928704590114375, + "grad_norm": 0.7181615829467773, + "learning_rate": 0.0001, + "loss": 62.3626, + "step": 9270 + }, + { + "epoch": 14.94482793369275, + "grad_norm": 0.7541075348854065, + "learning_rate": 0.0001, + "loss": 61.7186, + "step": 9280 + }, + { + "epoch": 14.960951277271125, + "grad_norm": 1.5627169609069824, + "learning_rate": 0.0001, + "loss": 61.654, + "step": 9290 + }, + { + "epoch": 14.977074620849498, + "grad_norm": 0.7126879096031189, + "learning_rate": 0.0001, + "loss": 61.6806, + "step": 9300 + }, + { + "epoch": 14.993197964427873, + "grad_norm": 0.7397864460945129, + "learning_rate": 0.0001, + "loss": 61.2669, + "step": 9310 + }, + { + "epoch": 15.008061671789187, + "grad_norm": 0.7282766699790955, + "learning_rate": 0.0001, + "loss": 56.2289, + "step": 9320 + }, + { + "epoch": 15.024185015367562, + "grad_norm": 0.8346580862998962, + "learning_rate": 0.0001, + "loss": 61.4335, + "step": 9330 + }, + { + "epoch": 15.040308358945936, + "grad_norm": 0.9117271304130554, + "learning_rate": 0.0001, + "loss": 60.4636, + "step": 9340 + }, + { + "epoch": 15.05643170252431, + "grad_norm": 0.8483272194862366, + "learning_rate": 0.0001, + "loss": 61.3003, + "step": 9350 + }, + { + "epoch": 15.072555046102686, + "grad_norm": 1.1898932456970215, + "learning_rate": 0.0001, + "loss": 62.7169, + "step": 9360 + }, + { + "epoch": 15.08867838968106, + "grad_norm": 0.7258167862892151, + "learning_rate": 0.0001, + "loss": 63.5285, + "step": 9370 + }, + { + "epoch": 15.104801733259436, + "grad_norm": 0.7810865044593811, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 9380 + }, + { + "epoch": 15.120925076837809, + "grad_norm": 1.0359688997268677, + "learning_rate": 0.0001, + "loss": 62.5435, + "step": 9390 + }, + { + "epoch": 15.137048420416184, + "grad_norm": 0.7371546030044556, + "learning_rate": 0.0001, + "loss": 60.5583, + "step": 9400 + }, + { + "epoch": 15.153171763994559, + "grad_norm": 0.7578926086425781, + "learning_rate": 0.0001, + "loss": 61.2853, + "step": 9410 + }, + { + "epoch": 15.169295107572934, + "grad_norm": 0.6161713004112244, + "learning_rate": 0.0001, + "loss": 62.9725, + "step": 9420 + }, + { + "epoch": 15.185418451151307, + "grad_norm": 0.8325598835945129, + "learning_rate": 0.0001, + "loss": 62.857, + "step": 9430 + }, + { + "epoch": 15.201541794729682, + "grad_norm": 1.1438454389572144, + "learning_rate": 0.0001, + "loss": 61.8167, + "step": 9440 + }, + { + "epoch": 15.217665138308057, + "grad_norm": 1.1820837259292603, + "learning_rate": 0.0001, + "loss": 60.9899, + "step": 9450 + }, + { + "epoch": 15.233788481886432, + "grad_norm": 0.6057641506195068, + "learning_rate": 0.0001, + "loss": 62.5144, + "step": 9460 + }, + { + "epoch": 15.249911825464805, + "grad_norm": 0.6661948561668396, + "learning_rate": 0.0001, + "loss": 62.0835, + "step": 9470 + }, + { + "epoch": 15.26603516904318, + "grad_norm": 0.7623168230056763, + "learning_rate": 0.0001, + "loss": 61.9491, + "step": 9480 + }, + { + "epoch": 15.282158512621555, + "grad_norm": 0.6784029006958008, + "learning_rate": 0.0001, + "loss": 63.6006, + "step": 9490 + }, + { + "epoch": 15.29828185619993, + "grad_norm": 0.9433781504631042, + "learning_rate": 0.0001, + "loss": 61.6677, + "step": 9500 + }, + { + "epoch": 15.314405199778305, + "grad_norm": 0.6544079184532166, + "learning_rate": 0.0001, + "loss": 63.1439, + "step": 9510 + }, + { + "epoch": 15.330528543356678, + "grad_norm": 0.7979472279548645, + "learning_rate": 0.0001, + "loss": 62.719, + "step": 9520 + }, + { + "epoch": 15.346651886935053, + "grad_norm": 0.6641252040863037, + "learning_rate": 0.0001, + "loss": 63.2543, + "step": 9530 + }, + { + "epoch": 15.362775230513428, + "grad_norm": 0.7780525088310242, + "learning_rate": 0.0001, + "loss": 62.5659, + "step": 9540 + }, + { + "epoch": 15.378898574091803, + "grad_norm": 0.842349648475647, + "learning_rate": 0.0001, + "loss": 60.041, + "step": 9550 + }, + { + "epoch": 15.395021917670176, + "grad_norm": 0.8991017937660217, + "learning_rate": 0.0001, + "loss": 62.9075, + "step": 9560 + }, + { + "epoch": 15.411145261248551, + "grad_norm": 0.6734052300453186, + "learning_rate": 0.0001, + "loss": 60.8647, + "step": 9570 + }, + { + "epoch": 15.427268604826926, + "grad_norm": 0.7585113644599915, + "learning_rate": 0.0001, + "loss": 62.105, + "step": 9580 + }, + { + "epoch": 15.443391948405301, + "grad_norm": 0.7915840148925781, + "learning_rate": 0.0001, + "loss": 62.2741, + "step": 9590 + }, + { + "epoch": 15.459515291983674, + "grad_norm": 0.8676218390464783, + "learning_rate": 0.0001, + "loss": 62.4548, + "step": 9600 + }, + { + "epoch": 15.47563863556205, + "grad_norm": 0.7698650360107422, + "learning_rate": 0.0001, + "loss": 61.3412, + "step": 9610 + }, + { + "epoch": 15.491761979140424, + "grad_norm": 0.6600908637046814, + "learning_rate": 0.0001, + "loss": 61.1561, + "step": 9620 + }, + { + "epoch": 15.5078853227188, + "grad_norm": 0.5995001792907715, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 9630 + }, + { + "epoch": 15.524008666297174, + "grad_norm": 0.7258557677268982, + "learning_rate": 0.0001, + "loss": 62.4659, + "step": 9640 + }, + { + "epoch": 15.540132009875547, + "grad_norm": 0.6562618613243103, + "learning_rate": 0.0001, + "loss": 63.288, + "step": 9650 + }, + { + "epoch": 15.556255353453922, + "grad_norm": 0.9224995374679565, + "learning_rate": 0.0001, + "loss": 61.4861, + "step": 9660 + }, + { + "epoch": 15.572378697032297, + "grad_norm": 0.7053484916687012, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 9670 + }, + { + "epoch": 15.588502040610672, + "grad_norm": 1.095229148864746, + "learning_rate": 0.0001, + "loss": 62.1542, + "step": 9680 + }, + { + "epoch": 15.604625384189045, + "grad_norm": 0.7563759088516235, + "learning_rate": 0.0001, + "loss": 61.1644, + "step": 9690 + }, + { + "epoch": 15.62074872776742, + "grad_norm": 0.6508835554122925, + "learning_rate": 0.0001, + "loss": 60.1374, + "step": 9700 + }, + { + "epoch": 15.636872071345795, + "grad_norm": 0.664799153804779, + "learning_rate": 0.0001, + "loss": 60.2331, + "step": 9710 + }, + { + "epoch": 15.65299541492417, + "grad_norm": 0.6525771617889404, + "learning_rate": 0.0001, + "loss": 60.6142, + "step": 9720 + }, + { + "epoch": 15.669118758502545, + "grad_norm": 0.7395381331443787, + "learning_rate": 0.0001, + "loss": 63.5623, + "step": 9730 + }, + { + "epoch": 15.685242102080919, + "grad_norm": 0.7063724398612976, + "learning_rate": 0.0001, + "loss": 63.7133, + "step": 9740 + }, + { + "epoch": 15.701365445659293, + "grad_norm": 0.7918921113014221, + "learning_rate": 0.0001, + "loss": 62.5957, + "step": 9750 + }, + { + "epoch": 15.717488789237668, + "grad_norm": 0.8135486245155334, + "learning_rate": 0.0001, + "loss": 62.9146, + "step": 9760 + }, + { + "epoch": 15.733612132816043, + "grad_norm": 0.8422091007232666, + "learning_rate": 0.0001, + "loss": 61.1466, + "step": 9770 + }, + { + "epoch": 15.749735476394417, + "grad_norm": 0.6329447627067566, + "learning_rate": 0.0001, + "loss": 61.7247, + "step": 9780 + }, + { + "epoch": 15.765858819972792, + "grad_norm": 1.0627039670944214, + "learning_rate": 0.0001, + "loss": 60.6789, + "step": 9790 + }, + { + "epoch": 15.781982163551167, + "grad_norm": 0.7316949367523193, + "learning_rate": 0.0001, + "loss": 61.9896, + "step": 9800 + }, + { + "epoch": 15.798105507129542, + "grad_norm": 0.6968954205513, + "learning_rate": 0.0001, + "loss": 60.8571, + "step": 9810 + }, + { + "epoch": 15.814228850707915, + "grad_norm": 0.7226986289024353, + "learning_rate": 0.0001, + "loss": 62.2664, + "step": 9820 + }, + { + "epoch": 15.83035219428629, + "grad_norm": 0.6542951464653015, + "learning_rate": 0.0001, + "loss": 61.3316, + "step": 9830 + }, + { + "epoch": 15.846475537864665, + "grad_norm": 0.7662289142608643, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 9840 + }, + { + "epoch": 15.86259888144304, + "grad_norm": 0.7060043215751648, + "learning_rate": 0.0001, + "loss": 62.2647, + "step": 9850 + }, + { + "epoch": 15.878722225021415, + "grad_norm": 0.6948843002319336, + "learning_rate": 0.0001, + "loss": 62.8678, + "step": 9860 + }, + { + "epoch": 15.894845568599788, + "grad_norm": 0.7787327766418457, + "learning_rate": 0.0001, + "loss": 62.9154, + "step": 9870 + }, + { + "epoch": 15.910968912178163, + "grad_norm": 0.7077026963233948, + "learning_rate": 0.0001, + "loss": 62.9537, + "step": 9880 + }, + { + "epoch": 15.927092255756538, + "grad_norm": 0.9957109093666077, + "learning_rate": 0.0001, + "loss": 60.9461, + "step": 9890 + }, + { + "epoch": 15.943215599334913, + "grad_norm": 0.9767161011695862, + "learning_rate": 0.0001, + "loss": 62.5944, + "step": 9900 + }, + { + "epoch": 15.959338942913286, + "grad_norm": 0.6016744375228882, + "learning_rate": 0.0001, + "loss": 62.5889, + "step": 9910 + }, + { + "epoch": 15.975462286491661, + "grad_norm": 0.5464991331100464, + "learning_rate": 0.0001, + "loss": 61.3138, + "step": 9920 + }, + { + "epoch": 15.991585630070036, + "grad_norm": 0.8242359757423401, + "learning_rate": 0.0001, + "loss": 61.8091, + "step": 9930 + }, + { + "epoch": 16.00644933743135, + "grad_norm": 0.7031534314155579, + "learning_rate": 0.0001, + "loss": 57.4847, + "step": 9940 + }, + { + "epoch": 16.022572681009724, + "grad_norm": 0.8429487943649292, + "learning_rate": 0.0001, + "loss": 61.7286, + "step": 9950 + }, + { + "epoch": 16.0386960245881, + "grad_norm": 0.7200850248336792, + "learning_rate": 0.0001, + "loss": 62.6968, + "step": 9960 + }, + { + "epoch": 16.054819368166473, + "grad_norm": 0.7412559390068054, + "learning_rate": 0.0001, + "loss": 62.413, + "step": 9970 + }, + { + "epoch": 16.07094271174485, + "grad_norm": 0.8179170489311218, + "learning_rate": 0.0001, + "loss": 62.7044, + "step": 9980 + }, + { + "epoch": 16.087066055323223, + "grad_norm": 0.6939452886581421, + "learning_rate": 0.0001, + "loss": 61.9062, + "step": 9990 + }, + { + "epoch": 16.1031893989016, + "grad_norm": 0.7145045399665833, + "learning_rate": 0.0001, + "loss": 61.6201, + "step": 10000 + }, + { + "epoch": 16.119312742479973, + "grad_norm": 0.7874398231506348, + "learning_rate": 0.0001, + "loss": 60.4299, + "step": 10010 + }, + { + "epoch": 16.135436086058345, + "grad_norm": 0.6640581488609314, + "learning_rate": 0.0001, + "loss": 62.0195, + "step": 10020 + }, + { + "epoch": 16.15155942963672, + "grad_norm": 0.7903703451156616, + "learning_rate": 0.0001, + "loss": 62.501, + "step": 10030 + }, + { + "epoch": 16.167682773215095, + "grad_norm": 0.6669818162918091, + "learning_rate": 0.0001, + "loss": 59.1489, + "step": 10040 + }, + { + "epoch": 16.18380611679347, + "grad_norm": 0.8380771279335022, + "learning_rate": 0.0001, + "loss": 61.0985, + "step": 10050 + }, + { + "epoch": 16.199929460371845, + "grad_norm": 0.7925304174423218, + "learning_rate": 0.0001, + "loss": 62.9251, + "step": 10060 + }, + { + "epoch": 16.21605280395022, + "grad_norm": 1.2176182270050049, + "learning_rate": 0.0001, + "loss": 61.6772, + "step": 10070 + }, + { + "epoch": 16.232176147528595, + "grad_norm": 0.828171968460083, + "learning_rate": 0.0001, + "loss": 61.3296, + "step": 10080 + }, + { + "epoch": 16.24829949110697, + "grad_norm": 0.6884972453117371, + "learning_rate": 0.0001, + "loss": 61.5371, + "step": 10090 + }, + { + "epoch": 16.264422834685345, + "grad_norm": 0.6455051302909851, + "learning_rate": 0.0001, + "loss": 60.6548, + "step": 10100 + }, + { + "epoch": 16.280546178263716, + "grad_norm": 0.687982976436615, + "learning_rate": 0.0001, + "loss": 62.0357, + "step": 10110 + }, + { + "epoch": 16.29666952184209, + "grad_norm": 0.811744213104248, + "learning_rate": 0.0001, + "loss": 60.5664, + "step": 10120 + }, + { + "epoch": 16.312792865420466, + "grad_norm": 0.7140675187110901, + "learning_rate": 0.0001, + "loss": 62.9491, + "step": 10130 + }, + { + "epoch": 16.32891620899884, + "grad_norm": 0.9068045020103455, + "learning_rate": 0.0001, + "loss": 61.118, + "step": 10140 + }, + { + "epoch": 16.345039552577216, + "grad_norm": 0.9028809666633606, + "learning_rate": 0.0001, + "loss": 61.4308, + "step": 10150 + }, + { + "epoch": 16.36116289615559, + "grad_norm": 0.7502166628837585, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10160 + }, + { + "epoch": 16.377286239733966, + "grad_norm": 0.807391345500946, + "learning_rate": 0.0001, + "loss": 61.6825, + "step": 10170 + }, + { + "epoch": 16.39340958331234, + "grad_norm": 0.762725830078125, + "learning_rate": 0.0001, + "loss": 62.7879, + "step": 10180 + }, + { + "epoch": 16.409532926890712, + "grad_norm": 0.7973660230636597, + "learning_rate": 0.0001, + "loss": 61.0727, + "step": 10190 + }, + { + "epoch": 16.425656270469087, + "grad_norm": 1.004622459411621, + "learning_rate": 0.0001, + "loss": 59.8048, + "step": 10200 + }, + { + "epoch": 16.441779614047462, + "grad_norm": 0.8243395686149597, + "learning_rate": 0.0001, + "loss": 63.9312, + "step": 10210 + }, + { + "epoch": 16.457902957625837, + "grad_norm": 0.7006281614303589, + "learning_rate": 0.0001, + "loss": 61.3766, + "step": 10220 + }, + { + "epoch": 16.474026301204212, + "grad_norm": 0.8672197461128235, + "learning_rate": 0.0001, + "loss": 61.6659, + "step": 10230 + }, + { + "epoch": 16.490149644782587, + "grad_norm": 0.7836477160453796, + "learning_rate": 0.0001, + "loss": 61.3686, + "step": 10240 + }, + { + "epoch": 16.506272988360962, + "grad_norm": 0.5842607617378235, + "learning_rate": 0.0001, + "loss": 62.616, + "step": 10250 + }, + { + "epoch": 16.522396331939337, + "grad_norm": 0.6777102947235107, + "learning_rate": 0.0001, + "loss": 62.1135, + "step": 10260 + }, + { + "epoch": 16.538519675517712, + "grad_norm": 0.856697678565979, + "learning_rate": 0.0001, + "loss": 61.849, + "step": 10270 + }, + { + "epoch": 16.554643019096083, + "grad_norm": 0.7300705313682556, + "learning_rate": 0.0001, + "loss": 60.7678, + "step": 10280 + }, + { + "epoch": 16.57076636267446, + "grad_norm": 0.799802839756012, + "learning_rate": 0.0001, + "loss": 62.4888, + "step": 10290 + }, + { + "epoch": 16.586889706252833, + "grad_norm": 0.791251540184021, + "learning_rate": 0.0001, + "loss": 61.7589, + "step": 10300 + }, + { + "epoch": 16.60301304983121, + "grad_norm": 0.6498433947563171, + "learning_rate": 0.0001, + "loss": 61.087, + "step": 10310 + }, + { + "epoch": 16.619136393409583, + "grad_norm": 0.702925443649292, + "learning_rate": 0.0001, + "loss": 59.3493, + "step": 10320 + }, + { + "epoch": 16.63525973698796, + "grad_norm": 0.8508023619651794, + "learning_rate": 0.0001, + "loss": 61.2188, + "step": 10330 + }, + { + "epoch": 16.651383080566333, + "grad_norm": 0.8325870633125305, + "learning_rate": 0.0001, + "loss": 62.9164, + "step": 10340 + }, + { + "epoch": 16.667506424144708, + "grad_norm": 0.7140557765960693, + "learning_rate": 0.0001, + "loss": 62.5838, + "step": 10350 + }, + { + "epoch": 16.683629767723083, + "grad_norm": 0.5574952363967896, + "learning_rate": 0.0001, + "loss": 59.767, + "step": 10360 + }, + { + "epoch": 16.699753111301455, + "grad_norm": 0.8022965788841248, + "learning_rate": 0.0001, + "loss": 60.2198, + "step": 10370 + }, + { + "epoch": 16.71587645487983, + "grad_norm": 1.1291698217391968, + "learning_rate": 0.0001, + "loss": 61.4908, + "step": 10380 + }, + { + "epoch": 16.731999798458205, + "grad_norm": 0.8651410937309265, + "learning_rate": 0.0001, + "loss": 60.5228, + "step": 10390 + }, + { + "epoch": 16.74812314203658, + "grad_norm": 0.8426535129547119, + "learning_rate": 0.0001, + "loss": 62.3786, + "step": 10400 + }, + { + "epoch": 16.764246485614954, + "grad_norm": 0.6329453587532043, + "learning_rate": 0.0001, + "loss": 60.9621, + "step": 10410 + }, + { + "epoch": 16.78036982919333, + "grad_norm": 0.6745129227638245, + "learning_rate": 0.0001, + "loss": 59.3405, + "step": 10420 + }, + { + "epoch": 16.796493172771704, + "grad_norm": 0.8483593463897705, + "learning_rate": 0.0001, + "loss": 61.1527, + "step": 10430 + }, + { + "epoch": 16.81261651635008, + "grad_norm": 0.8649617433547974, + "learning_rate": 0.0001, + "loss": 61.051, + "step": 10440 + }, + { + "epoch": 16.828739859928454, + "grad_norm": 0.7309728264808655, + "learning_rate": 0.0001, + "loss": 60.3626, + "step": 10450 + }, + { + "epoch": 16.844863203506826, + "grad_norm": 0.6467884182929993, + "learning_rate": 0.0001, + "loss": 63.8254, + "step": 10460 + }, + { + "epoch": 16.8609865470852, + "grad_norm": 0.7628102898597717, + "learning_rate": 0.0001, + "loss": 59.5604, + "step": 10470 + }, + { + "epoch": 16.877109890663576, + "grad_norm": 0.78266841173172, + "learning_rate": 0.0001, + "loss": 60.9387, + "step": 10480 + }, + { + "epoch": 16.89323323424195, + "grad_norm": 0.6936920285224915, + "learning_rate": 0.0001, + "loss": 62.5492, + "step": 10490 + }, + { + "epoch": 16.909356577820326, + "grad_norm": 0.8343939185142517, + "learning_rate": 0.0001, + "loss": 60.9975, + "step": 10500 + }, + { + "epoch": 16.9254799213987, + "grad_norm": 0.798984169960022, + "learning_rate": 0.0001, + "loss": 59.5835, + "step": 10510 + }, + { + "epoch": 16.941603264977076, + "grad_norm": 1.369275450706482, + "learning_rate": 0.0001, + "loss": 61.749, + "step": 10520 + }, + { + "epoch": 16.95772660855545, + "grad_norm": 0.5993408560752869, + "learning_rate": 0.0001, + "loss": 62.1093, + "step": 10530 + }, + { + "epoch": 16.973849952133826, + "grad_norm": 0.8153544664382935, + "learning_rate": 0.0001, + "loss": 61.857, + "step": 10540 + }, + { + "epoch": 16.989973295712197, + "grad_norm": 0.7042689919471741, + "learning_rate": 0.0001, + "loss": 60.5178, + "step": 10550 + }, + { + "epoch": 17.00483700307351, + "grad_norm": 1.1931812763214111, + "learning_rate": 0.0001, + "loss": 56.4815, + "step": 10560 + }, + { + "epoch": 17.020960346651886, + "grad_norm": 0.9051854610443115, + "learning_rate": 0.0001, + "loss": 60.7729, + "step": 10570 + }, + { + "epoch": 17.03708369023026, + "grad_norm": 1.093518614768982, + "learning_rate": 0.0001, + "loss": 60.904, + "step": 10580 + }, + { + "epoch": 17.053207033808636, + "grad_norm": 0.7545284032821655, + "learning_rate": 0.0001, + "loss": 59.7165, + "step": 10590 + }, + { + "epoch": 17.06933037738701, + "grad_norm": 0.8542641401290894, + "learning_rate": 0.0001, + "loss": 60.8947, + "step": 10600 + }, + { + "epoch": 17.085453720965386, + "grad_norm": 0.7393799424171448, + "learning_rate": 0.0001, + "loss": 62.8384, + "step": 10610 + }, + { + "epoch": 17.10157706454376, + "grad_norm": 0.6756435632705688, + "learning_rate": 0.0001, + "loss": 61.164, + "step": 10620 + }, + { + "epoch": 17.117700408122133, + "grad_norm": 0.725150465965271, + "learning_rate": 0.0001, + "loss": 60.3789, + "step": 10630 + }, + { + "epoch": 17.133823751700508, + "grad_norm": 1.1529417037963867, + "learning_rate": 0.0001, + "loss": 62.2465, + "step": 10640 + }, + { + "epoch": 17.149947095278883, + "grad_norm": 0.8266693353652954, + "learning_rate": 0.0001, + "loss": 61.6755, + "step": 10650 + }, + { + "epoch": 17.166070438857258, + "grad_norm": 0.7444040775299072, + "learning_rate": 0.0001, + "loss": 62.7084, + "step": 10660 + }, + { + "epoch": 17.182193782435633, + "grad_norm": 0.8164665102958679, + "learning_rate": 0.0001, + "loss": 61.6032, + "step": 10670 + }, + { + "epoch": 17.198317126014008, + "grad_norm": 0.8215296268463135, + "learning_rate": 0.0001, + "loss": 61.033, + "step": 10680 + }, + { + "epoch": 17.214440469592382, + "grad_norm": 0.7939956188201904, + "learning_rate": 0.0001, + "loss": 60.5016, + "step": 10690 + }, + { + "epoch": 17.230563813170757, + "grad_norm": 0.9533108472824097, + "learning_rate": 0.0001, + "loss": 62.6995, + "step": 10700 + }, + { + "epoch": 17.246687156749132, + "grad_norm": 0.6602820754051208, + "learning_rate": 0.0001, + "loss": 61.2709, + "step": 10710 + }, + { + "epoch": 17.262810500327504, + "grad_norm": 0.8369201421737671, + "learning_rate": 0.0001, + "loss": 63.1138, + "step": 10720 + }, + { + "epoch": 17.27893384390588, + "grad_norm": 0.7431164979934692, + "learning_rate": 0.0001, + "loss": 62.8588, + "step": 10730 + }, + { + "epoch": 17.295057187484254, + "grad_norm": 0.7957271337509155, + "learning_rate": 0.0001, + "loss": 60.5119, + "step": 10740 + }, + { + "epoch": 17.31118053106263, + "grad_norm": 0.9407948851585388, + "learning_rate": 0.0001, + "loss": 61.1877, + "step": 10750 + }, + { + "epoch": 17.327303874641004, + "grad_norm": 0.6641581654548645, + "learning_rate": 0.0001, + "loss": 60.542, + "step": 10760 + }, + { + "epoch": 17.34342721821938, + "grad_norm": 0.7786923050880432, + "learning_rate": 0.0001, + "loss": 62.1582, + "step": 10770 + }, + { + "epoch": 17.359550561797754, + "grad_norm": 0.8150411248207092, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10780 + }, + { + "epoch": 17.37567390537613, + "grad_norm": 0.8315487504005432, + "learning_rate": 0.0001, + "loss": 60.4382, + "step": 10790 + }, + { + "epoch": 17.391797248954504, + "grad_norm": 0.8305006623268127, + "learning_rate": 0.0001, + "loss": 61.3262, + "step": 10800 + }, + { + "epoch": 17.407920592532875, + "grad_norm": 0.897731363773346, + "learning_rate": 0.0001, + "loss": 60.7809, + "step": 10810 + }, + { + "epoch": 17.42404393611125, + "grad_norm": 0.7461619973182678, + "learning_rate": 0.0001, + "loss": 61.3858, + "step": 10820 + }, + { + "epoch": 17.440167279689625, + "grad_norm": 0.6750472187995911, + "learning_rate": 0.0001, + "loss": 60.2692, + "step": 10830 + }, + { + "epoch": 17.456290623268, + "grad_norm": 0.5412933230400085, + "learning_rate": 0.0001, + "loss": 61.7872, + "step": 10840 + }, + { + "epoch": 17.472413966846375, + "grad_norm": 0.6485120058059692, + "learning_rate": 0.0001, + "loss": 59.5407, + "step": 10850 + }, + { + "epoch": 17.48853731042475, + "grad_norm": 0.818708598613739, + "learning_rate": 0.0001, + "loss": 61.8066, + "step": 10860 + }, + { + "epoch": 17.504660654003125, + "grad_norm": 0.7121092081069946, + "learning_rate": 0.0001, + "loss": 61.8086, + "step": 10870 + }, + { + "epoch": 17.5207839975815, + "grad_norm": 0.7031684517860413, + "learning_rate": 0.0001, + "loss": 60.0706, + "step": 10880 + }, + { + "epoch": 17.536907341159875, + "grad_norm": 0.6259323358535767, + "learning_rate": 0.0001, + "loss": 61.9625, + "step": 10890 + }, + { + "epoch": 17.553030684738246, + "grad_norm": 0.8526529669761658, + "learning_rate": 0.0001, + "loss": 61.3299, + "step": 10900 + }, + { + "epoch": 17.56915402831662, + "grad_norm": 0.6860859990119934, + "learning_rate": 0.0001, + "loss": 61.3193, + "step": 10910 + }, + { + "epoch": 17.585277371894996, + "grad_norm": 0.7715994119644165, + "learning_rate": 0.0001, + "loss": 61.9111, + "step": 10920 + }, + { + "epoch": 17.60140071547337, + "grad_norm": 0.7171186804771423, + "learning_rate": 0.0001, + "loss": 60.3638, + "step": 10930 + }, + { + "epoch": 17.617524059051746, + "grad_norm": 0.7796649932861328, + "learning_rate": 0.0001, + "loss": 61.9437, + "step": 10940 + }, + { + "epoch": 17.63364740263012, + "grad_norm": 0.7930101752281189, + "learning_rate": 0.0001, + "loss": 60.1085, + "step": 10950 + }, + { + "epoch": 17.649770746208496, + "grad_norm": 0.6370466351509094, + "learning_rate": 0.0001, + "loss": 61.8433, + "step": 10960 + }, + { + "epoch": 17.66589408978687, + "grad_norm": 0.7830466628074646, + "learning_rate": 0.0001, + "loss": 61.3403, + "step": 10970 + }, + { + "epoch": 17.682017433365242, + "grad_norm": 0.873456597328186, + "learning_rate": 0.0001, + "loss": 62.4567, + "step": 10980 + }, + { + "epoch": 17.698140776943617, + "grad_norm": 0.7560754418373108, + "learning_rate": 0.0001, + "loss": 60.9808, + "step": 10990 + }, + { + "epoch": 17.714264120521992, + "grad_norm": 0.6832230091094971, + "learning_rate": 0.0001, + "loss": 61.5059, + "step": 11000 + }, + { + "epoch": 17.730387464100367, + "grad_norm": 0.7701742649078369, + "learning_rate": 0.0001, + "loss": 62.4963, + "step": 11010 + }, + { + "epoch": 17.746510807678742, + "grad_norm": 0.7088269591331482, + "learning_rate": 0.0001, + "loss": 62.6467, + "step": 11020 + }, + { + "epoch": 17.762634151257117, + "grad_norm": 0.8424612879753113, + "learning_rate": 0.0001, + "loss": 60.1213, + "step": 11030 + }, + { + "epoch": 17.778757494835492, + "grad_norm": 0.9974421858787537, + "learning_rate": 0.0001, + "loss": 61.6051, + "step": 11040 + }, + { + "epoch": 17.794880838413867, + "grad_norm": 0.701970636844635, + "learning_rate": 0.0001, + "loss": 62.4155, + "step": 11050 + }, + { + "epoch": 17.811004181992242, + "grad_norm": 0.6826842427253723, + "learning_rate": 0.0001, + "loss": 60.2127, + "step": 11060 + }, + { + "epoch": 17.827127525570614, + "grad_norm": 0.7019745707511902, + "learning_rate": 0.0001, + "loss": 60.9281, + "step": 11070 + }, + { + "epoch": 17.84325086914899, + "grad_norm": 0.836280345916748, + "learning_rate": 0.0001, + "loss": 61.1217, + "step": 11080 + }, + { + "epoch": 17.859374212727364, + "grad_norm": 0.6405923366546631, + "learning_rate": 0.0001, + "loss": 61.6393, + "step": 11090 + }, + { + "epoch": 17.87549755630574, + "grad_norm": 0.6809528470039368, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 11100 + }, + { + "epoch": 17.891620899884114, + "grad_norm": 0.6983410120010376, + "learning_rate": 0.0001, + "loss": 59.933, + "step": 11110 + }, + { + "epoch": 17.90774424346249, + "grad_norm": 1.0240110158920288, + "learning_rate": 0.0001, + "loss": 61.1964, + "step": 11120 + }, + { + "epoch": 17.923867587040863, + "grad_norm": 0.860657811164856, + "learning_rate": 0.0001, + "loss": 61.1613, + "step": 11130 + }, + { + "epoch": 17.93999093061924, + "grad_norm": 0.6434107422828674, + "learning_rate": 0.0001, + "loss": 60.2435, + "step": 11140 + }, + { + "epoch": 17.956114274197613, + "grad_norm": 0.7400848269462585, + "learning_rate": 0.0001, + "loss": 62.452, + "step": 11150 + }, + { + "epoch": 17.972237617775985, + "grad_norm": 0.6526095867156982, + "learning_rate": 0.0001, + "loss": 62.3344, + "step": 11160 + }, + { + "epoch": 17.98836096135436, + "grad_norm": 0.6971097588539124, + "learning_rate": 0.0001, + "loss": 60.2415, + "step": 11170 + }, + { + "epoch": 18.003224668715674, + "grad_norm": 0.724881649017334, + "learning_rate": 0.0001, + "loss": 56.0513, + "step": 11180 + }, + { + "epoch": 18.01934801229405, + "grad_norm": 0.653710663318634, + "learning_rate": 0.0001, + "loss": 60.8364, + "step": 11190 + }, + { + "epoch": 18.035471355872424, + "grad_norm": 0.7356674075126648, + "learning_rate": 0.0001, + "loss": 61.6681, + "step": 11200 + }, + { + "epoch": 18.0515946994508, + "grad_norm": 0.6726879477500916, + "learning_rate": 0.0001, + "loss": 60.6682, + "step": 11210 + }, + { + "epoch": 18.067718043029174, + "grad_norm": 0.7367722392082214, + "learning_rate": 0.0001, + "loss": 63.0462, + "step": 11220 + }, + { + "epoch": 18.08384138660755, + "grad_norm": 0.6687121987342834, + "learning_rate": 0.0001, + "loss": 60.8707, + "step": 11230 + }, + { + "epoch": 18.09996473018592, + "grad_norm": 1.0027060508728027, + "learning_rate": 0.0001, + "loss": 62.4703, + "step": 11240 + }, + { + "epoch": 18.116088073764296, + "grad_norm": 1.0447720289230347, + "learning_rate": 0.0001, + "loss": 62.1746, + "step": 11250 + }, + { + "epoch": 18.13221141734267, + "grad_norm": 0.7687053084373474, + "learning_rate": 0.0001, + "loss": 60.1413, + "step": 11260 + }, + { + "epoch": 18.148334760921045, + "grad_norm": 0.7250781059265137, + "learning_rate": 0.0001, + "loss": 60.1143, + "step": 11270 + }, + { + "epoch": 18.16445810449942, + "grad_norm": 0.7356208562850952, + "learning_rate": 0.0001, + "loss": 62.0788, + "step": 11280 + }, + { + "epoch": 18.180581448077795, + "grad_norm": 0.7838225364685059, + "learning_rate": 0.0001, + "loss": 61.7046, + "step": 11290 + }, + { + "epoch": 18.19670479165617, + "grad_norm": 0.7600042819976807, + "learning_rate": 0.0001, + "loss": 61.4646, + "step": 11300 + }, + { + "epoch": 18.212828135234545, + "grad_norm": 0.7339571714401245, + "learning_rate": 0.0001, + "loss": 61.0983, + "step": 11310 + }, + { + "epoch": 18.22895147881292, + "grad_norm": 0.6982674598693848, + "learning_rate": 0.0001, + "loss": 60.2853, + "step": 11320 + }, + { + "epoch": 18.24507482239129, + "grad_norm": 0.7572301030158997, + "learning_rate": 0.0001, + "loss": 61.8967, + "step": 11330 + }, + { + "epoch": 18.261198165969667, + "grad_norm": 0.6206921935081482, + "learning_rate": 0.0001, + "loss": 62.1467, + "step": 11340 + }, + { + "epoch": 18.27732150954804, + "grad_norm": 0.658437967300415, + "learning_rate": 0.0001, + "loss": 61.2117, + "step": 11350 + }, + { + "epoch": 18.293444853126417, + "grad_norm": 0.7880764603614807, + "learning_rate": 0.0001, + "loss": 61.6246, + "step": 11360 + }, + { + "epoch": 18.30956819670479, + "grad_norm": 0.6601800918579102, + "learning_rate": 0.0001, + "loss": 61.4818, + "step": 11370 + }, + { + "epoch": 18.325691540283167, + "grad_norm": 0.6816290616989136, + "learning_rate": 0.0001, + "loss": 60.4834, + "step": 11380 + }, + { + "epoch": 18.34181488386154, + "grad_norm": 0.7667511105537415, + "learning_rate": 0.0001, + "loss": 60.9106, + "step": 11390 + }, + { + "epoch": 18.357938227439917, + "grad_norm": 0.6490039825439453, + "learning_rate": 0.0001, + "loss": 59.6968, + "step": 11400 + }, + { + "epoch": 18.37406157101829, + "grad_norm": 0.6865658760070801, + "learning_rate": 0.0001, + "loss": 61.7898, + "step": 11410 + }, + { + "epoch": 18.390184914596663, + "grad_norm": 0.8605737090110779, + "learning_rate": 0.0001, + "loss": 60.924, + "step": 11420 + }, + { + "epoch": 18.406308258175038, + "grad_norm": 1.0474110841751099, + "learning_rate": 0.0001, + "loss": 61.0381, + "step": 11430 + }, + { + "epoch": 18.422431601753413, + "grad_norm": 0.9241340160369873, + "learning_rate": 0.0001, + "loss": 61.2609, + "step": 11440 + }, + { + "epoch": 18.438554945331788, + "grad_norm": 1.1441621780395508, + "learning_rate": 0.0001, + "loss": 60.5495, + "step": 11450 + }, + { + "epoch": 18.454678288910163, + "grad_norm": 0.7277412414550781, + "learning_rate": 0.0001, + "loss": 60.4654, + "step": 11460 + }, + { + "epoch": 18.470801632488538, + "grad_norm": 0.6668717861175537, + "learning_rate": 0.0001, + "loss": 60.057, + "step": 11470 + }, + { + "epoch": 18.486924976066913, + "grad_norm": 0.7335819005966187, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 11480 + }, + { + "epoch": 18.503048319645288, + "grad_norm": 0.7261807918548584, + "learning_rate": 0.0001, + "loss": 61.1508, + "step": 11490 + }, + { + "epoch": 18.519171663223663, + "grad_norm": 0.7715283036231995, + "learning_rate": 0.0001, + "loss": 61.5492, + "step": 11500 + }, + { + "epoch": 18.535295006802034, + "grad_norm": 0.9603212475776672, + "learning_rate": 0.0001, + "loss": 60.4321, + "step": 11510 + }, + { + "epoch": 18.55141835038041, + "grad_norm": 0.6130046844482422, + "learning_rate": 0.0001, + "loss": 61.2942, + "step": 11520 + }, + { + "epoch": 18.567541693958784, + "grad_norm": 0.8147348165512085, + "learning_rate": 0.0001, + "loss": 61.4313, + "step": 11530 + }, + { + "epoch": 18.58366503753716, + "grad_norm": 0.7909948825836182, + "learning_rate": 0.0001, + "loss": 60.3547, + "step": 11540 + }, + { + "epoch": 18.599788381115534, + "grad_norm": 0.6470443606376648, + "learning_rate": 0.0001, + "loss": 61.4144, + "step": 11550 + }, + { + "epoch": 18.61591172469391, + "grad_norm": 0.7132186889648438, + "learning_rate": 0.0001, + "loss": 61.8102, + "step": 11560 + }, + { + "epoch": 18.632035068272284, + "grad_norm": 0.7054323554039001, + "learning_rate": 0.0001, + "loss": 62.1431, + "step": 11570 + }, + { + "epoch": 18.64815841185066, + "grad_norm": 0.8784761428833008, + "learning_rate": 0.0001, + "loss": 61.6653, + "step": 11580 + }, + { + "epoch": 18.66428175542903, + "grad_norm": 0.7143248319625854, + "learning_rate": 0.0001, + "loss": 60.1275, + "step": 11590 + }, + { + "epoch": 18.680405099007405, + "grad_norm": 7.87012243270874, + "learning_rate": 0.0001, + "loss": 61.3484, + "step": 11600 + }, + { + "epoch": 18.69652844258578, + "grad_norm": 0.7263020873069763, + "learning_rate": 0.0001, + "loss": 59.906, + "step": 11610 + }, + { + "epoch": 18.712651786164155, + "grad_norm": 0.6468373537063599, + "learning_rate": 0.0001, + "loss": 61.8781, + "step": 11620 + }, + { + "epoch": 18.72877512974253, + "grad_norm": 3.34307861328125, + "learning_rate": 0.0001, + "loss": 61.0841, + "step": 11630 + }, + { + "epoch": 18.744898473320905, + "grad_norm": 0.7348340153694153, + "learning_rate": 0.0001, + "loss": 61.3201, + "step": 11640 + }, + { + "epoch": 18.76102181689928, + "grad_norm": 0.7646427154541016, + "learning_rate": 0.0001, + "loss": 60.1858, + "step": 11650 + }, + { + "epoch": 18.777145160477655, + "grad_norm": 0.6659495234489441, + "learning_rate": 0.0001, + "loss": 60.8999, + "step": 11660 + }, + { + "epoch": 18.79326850405603, + "grad_norm": 0.9913583993911743, + "learning_rate": 0.0001, + "loss": 59.0732, + "step": 11670 + }, + { + "epoch": 18.8093918476344, + "grad_norm": 0.7480027079582214, + "learning_rate": 0.0001, + "loss": 58.6846, + "step": 11680 + }, + { + "epoch": 18.825515191212777, + "grad_norm": 0.8000630736351013, + "learning_rate": 0.0001, + "loss": 62.8883, + "step": 11690 + }, + { + "epoch": 18.84163853479115, + "grad_norm": 0.8450509309768677, + "learning_rate": 0.0001, + "loss": 63.4962, + "step": 11700 + }, + { + "epoch": 18.857761878369526, + "grad_norm": 0.8365609645843506, + "learning_rate": 0.0001, + "loss": 62.2701, + "step": 11710 + }, + { + "epoch": 18.8738852219479, + "grad_norm": 0.7197144031524658, + "learning_rate": 0.0001, + "loss": 61.3158, + "step": 11720 + }, + { + "epoch": 18.890008565526276, + "grad_norm": 0.6460379958152771, + "learning_rate": 0.0001, + "loss": 62.6645, + "step": 11730 + }, + { + "epoch": 18.90613190910465, + "grad_norm": 0.8203967213630676, + "learning_rate": 0.0001, + "loss": 62.3257, + "step": 11740 + }, + { + "epoch": 18.922255252683026, + "grad_norm": 0.682134211063385, + "learning_rate": 0.0001, + "loss": 59.9144, + "step": 11750 + }, + { + "epoch": 18.9383785962614, + "grad_norm": 0.7080821394920349, + "learning_rate": 0.0001, + "loss": 61.4325, + "step": 11760 + }, + { + "epoch": 18.954501939839773, + "grad_norm": 0.7963712811470032, + "learning_rate": 0.0001, + "loss": 58.6136, + "step": 11770 + }, + { + "epoch": 18.970625283418148, + "grad_norm": 0.7317046523094177, + "learning_rate": 0.0001, + "loss": 61.3507, + "step": 11780 + }, + { + "epoch": 18.986748626996523, + "grad_norm": 0.6586548686027527, + "learning_rate": 0.0001, + "loss": 61.4898, + "step": 11790 + }, + { + "epoch": 19.001612334357837, + "grad_norm": 0.9010424613952637, + "learning_rate": 0.0001, + "loss": 56.8915, + "step": 11800 + }, + { + "epoch": 19.017735677936212, + "grad_norm": 0.7102377414703369, + "learning_rate": 0.0001, + "loss": 62.2097, + "step": 11810 + }, + { + "epoch": 19.033859021514587, + "grad_norm": 0.7959222793579102, + "learning_rate": 0.0001, + "loss": 61.4484, + "step": 11820 + }, + { + "epoch": 19.049982365092962, + "grad_norm": 0.6483829617500305, + "learning_rate": 0.0001, + "loss": 60.3563, + "step": 11830 + }, + { + "epoch": 19.066105708671337, + "grad_norm": 1.1301988363265991, + "learning_rate": 0.0001, + "loss": 61.3698, + "step": 11840 + }, + { + "epoch": 19.082229052249712, + "grad_norm": 0.7304167151451111, + "learning_rate": 0.0001, + "loss": 59.2096, + "step": 11850 + }, + { + "epoch": 19.098352395828083, + "grad_norm": 0.7757825255393982, + "learning_rate": 0.0001, + "loss": 59.914, + "step": 11860 + }, + { + "epoch": 19.11447573940646, + "grad_norm": 0.7009251713752747, + "learning_rate": 0.0001, + "loss": 61.0768, + "step": 11870 + }, + { + "epoch": 19.130599082984833, + "grad_norm": 0.6015574336051941, + "learning_rate": 0.0001, + "loss": 61.2698, + "step": 11880 + }, + { + "epoch": 19.14672242656321, + "grad_norm": 0.8028033971786499, + "learning_rate": 0.0001, + "loss": 61.422, + "step": 11890 + }, + { + "epoch": 19.162845770141583, + "grad_norm": 0.5753505825996399, + "learning_rate": 0.0001, + "loss": 61.5066, + "step": 11900 + }, + { + "epoch": 19.17896911371996, + "grad_norm": 0.7702978849411011, + "learning_rate": 0.0001, + "loss": 60.0737, + "step": 11910 + }, + { + "epoch": 19.195092457298333, + "grad_norm": 0.6469365358352661, + "learning_rate": 0.0001, + "loss": 59.5091, + "step": 11920 + }, + { + "epoch": 19.211215800876708, + "grad_norm": 0.8400216698646545, + "learning_rate": 0.0001, + "loss": 62.3739, + "step": 11930 + }, + { + "epoch": 19.22733914445508, + "grad_norm": 1.9919443130493164, + "learning_rate": 0.0001, + "loss": 59.7271, + "step": 11940 + }, + { + "epoch": 19.243462488033455, + "grad_norm": 1.0247725248336792, + "learning_rate": 0.0001, + "loss": 59.3804, + "step": 11950 + }, + { + "epoch": 19.25958583161183, + "grad_norm": 1.0717105865478516, + "learning_rate": 0.0001, + "loss": 61.3617, + "step": 11960 + }, + { + "epoch": 19.275709175190205, + "grad_norm": 0.6324111223220825, + "learning_rate": 0.0001, + "loss": 59.6094, + "step": 11970 + }, + { + "epoch": 19.29183251876858, + "grad_norm": 0.6665195226669312, + "learning_rate": 0.0001, + "loss": 61.5879, + "step": 11980 + }, + { + "epoch": 19.307955862346954, + "grad_norm": 0.7933892011642456, + "learning_rate": 0.0001, + "loss": 60.4011, + "step": 11990 + }, + { + "epoch": 19.32407920592533, + "grad_norm": 0.8135427236557007, + "learning_rate": 0.0001, + "loss": 61.2936, + "step": 12000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 33, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4675838084611205e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ade09d5233289ba4bc5076af39b8652078582dee --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016a4e9b5d71a1947317d3801b00fe5d0604edbef1f981d7b6c92f219131717c +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-12000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/latest new file mode 100644 index 0000000000000000000000000000000000000000..6c2e99ab39c9aa1d68dfdcb53dc9234015ced784 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/latest @@ -0,0 +1 @@ +global_step14000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c362c48c75bb3926ed1fbba812777bfbff0f7d8e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:880c4c96def44f78197acdd93c61cd7a742b4cf99e2f0a7011e7d55cfa4f0570 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..f48bf03ec979010cde484694e459f85cd4719ce8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dc5c2147c96a8fa55ece0b84c1f6f7cd354c282c4fb9f2356942b7601c8f2c0 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..b81136594bf9d5b454cef37dd8cb1a5cbf5ad610 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c43dd81437e035a799b5012ded2f2ed8c6cc28e2f591f5603213b295761526d +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..dcf307a3309c18ee34619ecbc9db028e088444c5 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38f7233ff15f274fd3e3f29afb592c3ce17f77aa64203652ff883c369c2f08bb +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..015666a9f0bec5722f1880871e22b6d6daf47bc3 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47730dc05d25ce29d21ea5cfa70c9c2ee2fe4f786a0667d156485a51dbcdcb1b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..8c764edbc2390859e37f678fee7ecb0df27b30e8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28fa6417945e4451f8bf5820eae79ae7e46f37ae948620020039ee10886ecd98 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..3714e0245e676a80038d0b820cde153297b8632a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc8559012f6738fc281fe6bfd8eec732c5661709a74972a10d4e1d22b396cf39 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..87b79595fd36eab21e48c8855ed932f96531162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76ed3e2eb7ce35629a2703f26662f099c1482473c8788ac5c95330610e843311 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..f59c9d1b1be9d1adcab9492ac8ab6023756ffff6 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0a2da228d7090a1a3b38002fda6571410fe6c14f5013dc6475a911522cb1778 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..f41b87afdce1fcc0b7b9eff4e6b9589700d352b3 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77cff55491dcd98e210e6c07623399a07e8a123042ff2c25bda661be0e7132f3 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1315f6ccdb09896ec77a47766f19ddee60228299 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/trainer_state.json @@ -0,0 +1,9834 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 22.54496901294906, + "eval_steps": 2000, + "global_step": 14000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.016123343578374565, + "grad_norm": 48.11856460571289, + "learning_rate": 0.0001, + "loss": 299.1088, + "step": 10 + }, + { + "epoch": 0.03224668715674913, + "grad_norm": 9.07711410522461, + "learning_rate": 0.0001, + "loss": 234.0888, + "step": 20 + }, + { + "epoch": 0.0483700307351237, + "grad_norm": 3.5935330390930176, + "learning_rate": 0.0001, + "loss": 231.4619, + "step": 30 + }, + { + "epoch": 0.06449337431349826, + "grad_norm": 3.5784752368927, + "learning_rate": 0.0001, + "loss": 229.7082, + "step": 40 + }, + { + "epoch": 0.08061671789187283, + "grad_norm": 3.5757956504821777, + "learning_rate": 0.0001, + "loss": 230.1062, + "step": 50 + }, + { + "epoch": 0.0967400614702474, + "grad_norm": 3.8763978481292725, + "learning_rate": 0.0001, + "loss": 229.1914, + "step": 60 + }, + { + "epoch": 0.11286340504862195, + "grad_norm": 3.006988525390625, + "learning_rate": 0.0001, + "loss": 229.6482, + "step": 70 + }, + { + "epoch": 0.12898674862699652, + "grad_norm": 2.3527095317840576, + "learning_rate": 0.0001, + "loss": 228.1141, + "step": 80 + }, + { + "epoch": 0.14511009220537108, + "grad_norm": 5.353086471557617, + "learning_rate": 0.0001, + "loss": 228.1448, + "step": 90 + }, + { + "epoch": 0.16123343578374566, + "grad_norm": 2.251854419708252, + "learning_rate": 0.0001, + "loss": 228.3897, + "step": 100 + }, + { + "epoch": 0.17735677936212021, + "grad_norm": 1.5789915323257446, + "learning_rate": 0.0001, + "loss": 227.7662, + "step": 110 + }, + { + "epoch": 0.1934801229404948, + "grad_norm": 3.7646398544311523, + "learning_rate": 0.0001, + "loss": 223.6411, + "step": 120 + }, + { + "epoch": 0.20960346651886935, + "grad_norm": 3.444000720977783, + "learning_rate": 0.0001, + "loss": 221.2073, + "step": 130 + }, + { + "epoch": 0.2257268100972439, + "grad_norm": 2.1871185302734375, + "learning_rate": 0.0001, + "loss": 215.9944, + "step": 140 + }, + { + "epoch": 0.2418501536756185, + "grad_norm": 2.266244411468506, + "learning_rate": 0.0001, + "loss": 212.3066, + "step": 150 + }, + { + "epoch": 0.25797349725399304, + "grad_norm": 2.043832778930664, + "learning_rate": 0.0001, + "loss": 208.1343, + "step": 160 + }, + { + "epoch": 0.2740968408323676, + "grad_norm": 2.2641494274139404, + "learning_rate": 0.0001, + "loss": 203.1485, + "step": 170 + }, + { + "epoch": 0.29022018441074215, + "grad_norm": 2.9727790355682373, + "learning_rate": 0.0001, + "loss": 198.181, + "step": 180 + }, + { + "epoch": 0.30634352798911674, + "grad_norm": 2.751523494720459, + "learning_rate": 0.0001, + "loss": 194.8422, + "step": 190 + }, + { + "epoch": 0.3224668715674913, + "grad_norm": 1.9561562538146973, + "learning_rate": 0.0001, + "loss": 189.9401, + "step": 200 + }, + { + "epoch": 0.33859021514586585, + "grad_norm": 2.3264107704162598, + "learning_rate": 0.0001, + "loss": 186.1309, + "step": 210 + }, + { + "epoch": 0.35471355872424043, + "grad_norm": 5.212426662445068, + "learning_rate": 0.0001, + "loss": 182.2255, + "step": 220 + }, + { + "epoch": 0.370836902302615, + "grad_norm": 2.488032102584839, + "learning_rate": 0.0001, + "loss": 181.0103, + "step": 230 + }, + { + "epoch": 0.3869602458809896, + "grad_norm": 2.6173202991485596, + "learning_rate": 0.0001, + "loss": 177.6162, + "step": 240 + }, + { + "epoch": 0.4030835894593641, + "grad_norm": 2.6245291233062744, + "learning_rate": 0.0001, + "loss": 173.0431, + "step": 250 + }, + { + "epoch": 0.4192069330377387, + "grad_norm": 5.356006145477295, + "learning_rate": 0.0001, + "loss": 172.2371, + "step": 260 + }, + { + "epoch": 0.4353302766161133, + "grad_norm": 2.0453829765319824, + "learning_rate": 0.0001, + "loss": 169.7781, + "step": 270 + }, + { + "epoch": 0.4514536201944878, + "grad_norm": 1.7002843618392944, + "learning_rate": 0.0001, + "loss": 166.9564, + "step": 280 + }, + { + "epoch": 0.4675769637728624, + "grad_norm": 2.181410312652588, + "learning_rate": 0.0001, + "loss": 166.0094, + "step": 290 + }, + { + "epoch": 0.483700307351237, + "grad_norm": 2.2948265075683594, + "learning_rate": 0.0001, + "loss": 162.4388, + "step": 300 + }, + { + "epoch": 0.4998236509296115, + "grad_norm": 3.0272789001464844, + "learning_rate": 0.0001, + "loss": 160.3446, + "step": 310 + }, + { + "epoch": 0.5159469945079861, + "grad_norm": 1.948184847831726, + "learning_rate": 0.0001, + "loss": 158.488, + "step": 320 + }, + { + "epoch": 0.5320703380863606, + "grad_norm": 1.7981538772583008, + "learning_rate": 0.0001, + "loss": 156.2107, + "step": 330 + }, + { + "epoch": 0.5481936816647353, + "grad_norm": 2.049323797225952, + "learning_rate": 0.0001, + "loss": 155.5434, + "step": 340 + }, + { + "epoch": 0.5643170252431098, + "grad_norm": 1.9415442943572998, + "learning_rate": 0.0001, + "loss": 153.8682, + "step": 350 + }, + { + "epoch": 0.5804403688214843, + "grad_norm": 2.2168045043945312, + "learning_rate": 0.0001, + "loss": 150.4087, + "step": 360 + }, + { + "epoch": 0.596563712399859, + "grad_norm": 1.510750651359558, + "learning_rate": 0.0001, + "loss": 151.7655, + "step": 370 + }, + { + "epoch": 0.6126870559782335, + "grad_norm": 2.7469751834869385, + "learning_rate": 0.0001, + "loss": 147.4665, + "step": 380 + }, + { + "epoch": 0.628810399556608, + "grad_norm": 2.0436975955963135, + "learning_rate": 0.0001, + "loss": 147.4405, + "step": 390 + }, + { + "epoch": 0.6449337431349826, + "grad_norm": 1.8219181299209595, + "learning_rate": 0.0001, + "loss": 146.6332, + "step": 400 + }, + { + "epoch": 0.6610570867133572, + "grad_norm": 1.948431134223938, + "learning_rate": 0.0001, + "loss": 143.3298, + "step": 410 + }, + { + "epoch": 0.6771804302917317, + "grad_norm": 1.8887908458709717, + "learning_rate": 0.0001, + "loss": 140.5058, + "step": 420 + }, + { + "epoch": 0.6933037738701063, + "grad_norm": 1.8739320039749146, + "learning_rate": 0.0001, + "loss": 137.6334, + "step": 430 + }, + { + "epoch": 0.7094271174484809, + "grad_norm": 2.057689666748047, + "learning_rate": 0.0001, + "loss": 135.0343, + "step": 440 + }, + { + "epoch": 0.7255504610268555, + "grad_norm": 1.6926478147506714, + "learning_rate": 0.0001, + "loss": 132.8766, + "step": 450 + }, + { + "epoch": 0.74167380460523, + "grad_norm": 1.5039501190185547, + "learning_rate": 0.0001, + "loss": 131.6774, + "step": 460 + }, + { + "epoch": 0.7577971481836046, + "grad_norm": 1.6903547048568726, + "learning_rate": 0.0001, + "loss": 129.3116, + "step": 470 + }, + { + "epoch": 0.7739204917619792, + "grad_norm": 3.7062244415283203, + "learning_rate": 0.0001, + "loss": 127.9314, + "step": 480 + }, + { + "epoch": 0.7900438353403537, + "grad_norm": 2.0586934089660645, + "learning_rate": 0.0001, + "loss": 126.6398, + "step": 490 + }, + { + "epoch": 0.8061671789187282, + "grad_norm": 1.7598215341567993, + "learning_rate": 0.0001, + "loss": 125.1501, + "step": 500 + }, + { + "epoch": 0.8222905224971029, + "grad_norm": 1.5992189645767212, + "learning_rate": 0.0001, + "loss": 124.7039, + "step": 510 + }, + { + "epoch": 0.8384138660754774, + "grad_norm": 2.8735458850860596, + "learning_rate": 0.0001, + "loss": 122.1619, + "step": 520 + }, + { + "epoch": 0.8545372096538519, + "grad_norm": 1.867098093032837, + "learning_rate": 0.0001, + "loss": 122.4506, + "step": 530 + }, + { + "epoch": 0.8706605532322266, + "grad_norm": 2.2088418006896973, + "learning_rate": 0.0001, + "loss": 122.1047, + "step": 540 + }, + { + "epoch": 0.8867838968106011, + "grad_norm": 1.642657995223999, + "learning_rate": 0.0001, + "loss": 120.1141, + "step": 550 + }, + { + "epoch": 0.9029072403889756, + "grad_norm": 1.5567114353179932, + "learning_rate": 0.0001, + "loss": 118.532, + "step": 560 + }, + { + "epoch": 0.9190305839673503, + "grad_norm": 1.667843222618103, + "learning_rate": 0.0001, + "loss": 118.0734, + "step": 570 + }, + { + "epoch": 0.9351539275457248, + "grad_norm": 1.5304442644119263, + "learning_rate": 0.0001, + "loss": 114.4925, + "step": 580 + }, + { + "epoch": 0.9512772711240993, + "grad_norm": 1.6920255422592163, + "learning_rate": 0.0001, + "loss": 114.0729, + "step": 590 + }, + { + "epoch": 0.967400614702474, + "grad_norm": 1.6313318014144897, + "learning_rate": 0.0001, + "loss": 114.2879, + "step": 600 + }, + { + "epoch": 0.9835239582808485, + "grad_norm": 1.9033657312393188, + "learning_rate": 0.0001, + "loss": 110.5299, + "step": 610 + }, + { + "epoch": 0.999647301859223, + "grad_norm": 1.9010754823684692, + "learning_rate": 0.0001, + "loss": 109.1145, + "step": 620 + }, + { + "epoch": 1.0145110092205372, + "grad_norm": 3.5390989780426025, + "learning_rate": 0.0001, + "loss": 101.0258, + "step": 630 + }, + { + "epoch": 1.0306343527989117, + "grad_norm": 2.2925171852111816, + "learning_rate": 0.0001, + "loss": 107.3693, + "step": 640 + }, + { + "epoch": 1.0467576963772862, + "grad_norm": 2.329108238220215, + "learning_rate": 0.0001, + "loss": 104.3778, + "step": 650 + }, + { + "epoch": 1.0628810399556607, + "grad_norm": 1.7325661182403564, + "learning_rate": 0.0001, + "loss": 102.8409, + "step": 660 + }, + { + "epoch": 1.0790043835340353, + "grad_norm": 1.989586353302002, + "learning_rate": 0.0001, + "loss": 104.2577, + "step": 670 + }, + { + "epoch": 1.09512772711241, + "grad_norm": 1.900484323501587, + "learning_rate": 0.0001, + "loss": 102.3051, + "step": 680 + }, + { + "epoch": 1.1112510706907845, + "grad_norm": 2.015859842300415, + "learning_rate": 0.0001, + "loss": 100.893, + "step": 690 + }, + { + "epoch": 1.127374414269159, + "grad_norm": 1.7695508003234863, + "learning_rate": 0.0001, + "loss": 99.215, + "step": 700 + }, + { + "epoch": 1.1434977578475336, + "grad_norm": 2.025923490524292, + "learning_rate": 0.0001, + "loss": 98.8768, + "step": 710 + }, + { + "epoch": 1.1596211014259081, + "grad_norm": 2.946708917617798, + "learning_rate": 0.0001, + "loss": 96.5588, + "step": 720 + }, + { + "epoch": 1.1757444450042827, + "grad_norm": 2.4511258602142334, + "learning_rate": 0.0001, + "loss": 95.0545, + "step": 730 + }, + { + "epoch": 1.1918677885826574, + "grad_norm": 2.3094682693481445, + "learning_rate": 0.0001, + "loss": 97.004, + "step": 740 + }, + { + "epoch": 1.207991132161032, + "grad_norm": 2.1313347816467285, + "learning_rate": 0.0001, + "loss": 94.8819, + "step": 750 + }, + { + "epoch": 1.2241144757394065, + "grad_norm": 1.614182710647583, + "learning_rate": 0.0001, + "loss": 95.5339, + "step": 760 + }, + { + "epoch": 1.240237819317781, + "grad_norm": 2.029953956604004, + "learning_rate": 0.0001, + "loss": 93.7303, + "step": 770 + }, + { + "epoch": 1.2563611628961555, + "grad_norm": 1.7871663570404053, + "learning_rate": 0.0001, + "loss": 93.8354, + "step": 780 + }, + { + "epoch": 1.2724845064745303, + "grad_norm": 1.72455632686615, + "learning_rate": 0.0001, + "loss": 94.7348, + "step": 790 + }, + { + "epoch": 1.2886078500529048, + "grad_norm": 1.9776639938354492, + "learning_rate": 0.0001, + "loss": 93.5957, + "step": 800 + }, + { + "epoch": 1.3047311936312793, + "grad_norm": 1.7187533378601074, + "learning_rate": 0.0001, + "loss": 92.9583, + "step": 810 + }, + { + "epoch": 1.3208545372096538, + "grad_norm": 1.5949162244796753, + "learning_rate": 0.0001, + "loss": 93.3088, + "step": 820 + }, + { + "epoch": 1.3369778807880284, + "grad_norm": 1.4616206884384155, + "learning_rate": 0.0001, + "loss": 93.2125, + "step": 830 + }, + { + "epoch": 1.353101224366403, + "grad_norm": 2.832163095474243, + "learning_rate": 0.0001, + "loss": 90.3301, + "step": 840 + }, + { + "epoch": 1.3692245679447774, + "grad_norm": 2.174501419067383, + "learning_rate": 0.0001, + "loss": 91.6988, + "step": 850 + }, + { + "epoch": 1.3853479115231522, + "grad_norm": 1.2156339883804321, + "learning_rate": 0.0001, + "loss": 92.7419, + "step": 860 + }, + { + "epoch": 1.4014712551015267, + "grad_norm": 1.608261227607727, + "learning_rate": 0.0001, + "loss": 89.8255, + "step": 870 + }, + { + "epoch": 1.4175945986799012, + "grad_norm": 1.6671923398971558, + "learning_rate": 0.0001, + "loss": 90.9683, + "step": 880 + }, + { + "epoch": 1.4337179422582758, + "grad_norm": 1.2346433401107788, + "learning_rate": 0.0001, + "loss": 90.6092, + "step": 890 + }, + { + "epoch": 1.4498412858366505, + "grad_norm": 1.561085820198059, + "learning_rate": 0.0001, + "loss": 88.0842, + "step": 900 + }, + { + "epoch": 1.465964629415025, + "grad_norm": 1.5475201606750488, + "learning_rate": 0.0001, + "loss": 89.8786, + "step": 910 + }, + { + "epoch": 1.4820879729933996, + "grad_norm": 1.6044800281524658, + "learning_rate": 0.0001, + "loss": 88.0364, + "step": 920 + }, + { + "epoch": 1.498211316571774, + "grad_norm": 1.4081908464431763, + "learning_rate": 0.0001, + "loss": 88.8799, + "step": 930 + }, + { + "epoch": 1.5143346601501486, + "grad_norm": 2.341053009033203, + "learning_rate": 0.0001, + "loss": 88.2023, + "step": 940 + }, + { + "epoch": 1.5304580037285231, + "grad_norm": 1.3976799249649048, + "learning_rate": 0.0001, + "loss": 86.6493, + "step": 950 + }, + { + "epoch": 1.5465813473068977, + "grad_norm": 1.2485483884811401, + "learning_rate": 0.0001, + "loss": 88.9224, + "step": 960 + }, + { + "epoch": 1.5627046908852722, + "grad_norm": 1.6824235916137695, + "learning_rate": 0.0001, + "loss": 89.6484, + "step": 970 + }, + { + "epoch": 1.578828034463647, + "grad_norm": 1.64790940284729, + "learning_rate": 0.0001, + "loss": 87.2479, + "step": 980 + }, + { + "epoch": 1.5949513780420215, + "grad_norm": 1.2093443870544434, + "learning_rate": 0.0001, + "loss": 86.8241, + "step": 990 + }, + { + "epoch": 1.611074721620396, + "grad_norm": 1.5306812524795532, + "learning_rate": 0.0001, + "loss": 88.0006, + "step": 1000 + }, + { + "epoch": 1.6271980651987707, + "grad_norm": 1.2465763092041016, + "learning_rate": 0.0001, + "loss": 86.8143, + "step": 1010 + }, + { + "epoch": 1.6433214087771453, + "grad_norm": 1.5224069356918335, + "learning_rate": 0.0001, + "loss": 87.0102, + "step": 1020 + }, + { + "epoch": 1.6594447523555198, + "grad_norm": 1.312179684638977, + "learning_rate": 0.0001, + "loss": 88.7395, + "step": 1030 + }, + { + "epoch": 1.6755680959338943, + "grad_norm": 1.2343158721923828, + "learning_rate": 0.0001, + "loss": 87.2868, + "step": 1040 + }, + { + "epoch": 1.6916914395122689, + "grad_norm": 1.2778265476226807, + "learning_rate": 0.0001, + "loss": 87.043, + "step": 1050 + }, + { + "epoch": 1.7078147830906434, + "grad_norm": 1.2799627780914307, + "learning_rate": 0.0001, + "loss": 87.7843, + "step": 1060 + }, + { + "epoch": 1.723938126669018, + "grad_norm": 1.3688952922821045, + "learning_rate": 0.0001, + "loss": 85.8985, + "step": 1070 + }, + { + "epoch": 1.7400614702473924, + "grad_norm": 1.4317796230316162, + "learning_rate": 0.0001, + "loss": 85.5408, + "step": 1080 + }, + { + "epoch": 1.756184813825767, + "grad_norm": 1.164801001548767, + "learning_rate": 0.0001, + "loss": 85.8484, + "step": 1090 + }, + { + "epoch": 1.7723081574041417, + "grad_norm": 1.3929173946380615, + "learning_rate": 0.0001, + "loss": 86.9031, + "step": 1100 + }, + { + "epoch": 1.7884315009825162, + "grad_norm": 2.116119861602783, + "learning_rate": 0.0001, + "loss": 86.5343, + "step": 1110 + }, + { + "epoch": 1.8045548445608908, + "grad_norm": 1.4117276668548584, + "learning_rate": 0.0001, + "loss": 85.2678, + "step": 1120 + }, + { + "epoch": 1.8206781881392655, + "grad_norm": 1.3213504552841187, + "learning_rate": 0.0001, + "loss": 85.8935, + "step": 1130 + }, + { + "epoch": 1.83680153171764, + "grad_norm": 1.3589770793914795, + "learning_rate": 0.0001, + "loss": 85.032, + "step": 1140 + }, + { + "epoch": 1.8529248752960146, + "grad_norm": 1.3336007595062256, + "learning_rate": 0.0001, + "loss": 84.0993, + "step": 1150 + }, + { + "epoch": 1.869048218874389, + "grad_norm": 1.1081451177597046, + "learning_rate": 0.0001, + "loss": 84.2225, + "step": 1160 + }, + { + "epoch": 1.8851715624527636, + "grad_norm": 1.482038974761963, + "learning_rate": 0.0001, + "loss": 82.1439, + "step": 1170 + }, + { + "epoch": 1.9012949060311382, + "grad_norm": 1.1571478843688965, + "learning_rate": 0.0001, + "loss": 82.5045, + "step": 1180 + }, + { + "epoch": 1.9174182496095127, + "grad_norm": 1.2665468454360962, + "learning_rate": 0.0001, + "loss": 82.8761, + "step": 1190 + }, + { + "epoch": 1.9335415931878872, + "grad_norm": 1.0771509408950806, + "learning_rate": 0.0001, + "loss": 81.5162, + "step": 1200 + }, + { + "epoch": 1.949664936766262, + "grad_norm": 1.2992299795150757, + "learning_rate": 0.0001, + "loss": 82.4038, + "step": 1210 + }, + { + "epoch": 1.9657882803446365, + "grad_norm": 1.2946668863296509, + "learning_rate": 0.0001, + "loss": 83.4837, + "step": 1220 + }, + { + "epoch": 1.981911623923011, + "grad_norm": 1.3505686521530151, + "learning_rate": 0.0001, + "loss": 81.8993, + "step": 1230 + }, + { + "epoch": 1.9980349675013858, + "grad_norm": 1.1247715950012207, + "learning_rate": 0.0001, + "loss": 82.9006, + "step": 1240 + }, + { + "epoch": 2.0128986748627, + "grad_norm": 1.5652127265930176, + "learning_rate": 0.0001, + "loss": 75.8347, + "step": 1250 + }, + { + "epoch": 2.0290220184410743, + "grad_norm": 7.311964988708496, + "learning_rate": 0.0001, + "loss": 80.5435, + "step": 1260 + }, + { + "epoch": 2.045145362019449, + "grad_norm": 1.2229893207550049, + "learning_rate": 0.0001, + "loss": 83.3277, + "step": 1270 + }, + { + "epoch": 2.0612687055978234, + "grad_norm": 1.1387693881988525, + "learning_rate": 0.0001, + "loss": 81.3252, + "step": 1280 + }, + { + "epoch": 2.077392049176198, + "grad_norm": 1.307201623916626, + "learning_rate": 0.0001, + "loss": 82.3311, + "step": 1290 + }, + { + "epoch": 2.0935153927545724, + "grad_norm": 1.1502896547317505, + "learning_rate": 0.0001, + "loss": 81.1682, + "step": 1300 + }, + { + "epoch": 2.109638736332947, + "grad_norm": 0.9291583299636841, + "learning_rate": 0.0001, + "loss": 81.7017, + "step": 1310 + }, + { + "epoch": 2.1257620799113215, + "grad_norm": 1.0334666967391968, + "learning_rate": 0.0001, + "loss": 81.718, + "step": 1320 + }, + { + "epoch": 2.141885423489696, + "grad_norm": 1.1099056005477905, + "learning_rate": 0.0001, + "loss": 81.2232, + "step": 1330 + }, + { + "epoch": 2.1580087670680705, + "grad_norm": 1.289478063583374, + "learning_rate": 0.0001, + "loss": 81.8736, + "step": 1340 + }, + { + "epoch": 2.174132110646445, + "grad_norm": 1.1634794473648071, + "learning_rate": 0.0001, + "loss": 79.9434, + "step": 1350 + }, + { + "epoch": 2.19025545422482, + "grad_norm": 0.9563440680503845, + "learning_rate": 0.0001, + "loss": 80.628, + "step": 1360 + }, + { + "epoch": 2.2063787978031946, + "grad_norm": 1.1268844604492188, + "learning_rate": 0.0001, + "loss": 82.1389, + "step": 1370 + }, + { + "epoch": 2.222502141381569, + "grad_norm": 1.1050443649291992, + "learning_rate": 0.0001, + "loss": 78.9023, + "step": 1380 + }, + { + "epoch": 2.2386254849599436, + "grad_norm": 1.1936142444610596, + "learning_rate": 0.0001, + "loss": 81.2118, + "step": 1390 + }, + { + "epoch": 2.254748828538318, + "grad_norm": 1.7825465202331543, + "learning_rate": 0.0001, + "loss": 80.8804, + "step": 1400 + }, + { + "epoch": 2.2708721721166927, + "grad_norm": 2.437251329421997, + "learning_rate": 0.0001, + "loss": 79.3807, + "step": 1410 + }, + { + "epoch": 2.286995515695067, + "grad_norm": 1.0604854822158813, + "learning_rate": 0.0001, + "loss": 79.5881, + "step": 1420 + }, + { + "epoch": 2.3031188592734417, + "grad_norm": 1.2285529375076294, + "learning_rate": 0.0001, + "loss": 79.5517, + "step": 1430 + }, + { + "epoch": 2.3192422028518163, + "grad_norm": 1.1916660070419312, + "learning_rate": 0.0001, + "loss": 80.5887, + "step": 1440 + }, + { + "epoch": 2.335365546430191, + "grad_norm": 1.224304437637329, + "learning_rate": 0.0001, + "loss": 79.8109, + "step": 1450 + }, + { + "epoch": 2.3514888900085653, + "grad_norm": 0.9228448867797852, + "learning_rate": 0.0001, + "loss": 79.2787, + "step": 1460 + }, + { + "epoch": 2.3676122335869403, + "grad_norm": 1.2085226774215698, + "learning_rate": 0.0001, + "loss": 78.4983, + "step": 1470 + }, + { + "epoch": 2.383735577165315, + "grad_norm": 2.424083948135376, + "learning_rate": 0.0001, + "loss": 79.8657, + "step": 1480 + }, + { + "epoch": 2.3998589207436893, + "grad_norm": 1.0277849435806274, + "learning_rate": 0.0001, + "loss": 80.6915, + "step": 1490 + }, + { + "epoch": 2.415982264322064, + "grad_norm": 1.0226075649261475, + "learning_rate": 0.0001, + "loss": 78.9751, + "step": 1500 + }, + { + "epoch": 2.4321056079004384, + "grad_norm": 0.8860388398170471, + "learning_rate": 0.0001, + "loss": 78.3915, + "step": 1510 + }, + { + "epoch": 2.448228951478813, + "grad_norm": 1.265637755393982, + "learning_rate": 0.0001, + "loss": 78.8737, + "step": 1520 + }, + { + "epoch": 2.4643522950571874, + "grad_norm": 1.2518186569213867, + "learning_rate": 0.0001, + "loss": 76.5169, + "step": 1530 + }, + { + "epoch": 2.480475638635562, + "grad_norm": 1.074277400970459, + "learning_rate": 0.0001, + "loss": 77.9156, + "step": 1540 + }, + { + "epoch": 2.4965989822139365, + "grad_norm": 1.0666711330413818, + "learning_rate": 0.0001, + "loss": 78.4103, + "step": 1550 + }, + { + "epoch": 2.512722325792311, + "grad_norm": 0.9492248892784119, + "learning_rate": 0.0001, + "loss": 76.7393, + "step": 1560 + }, + { + "epoch": 2.5288456693706856, + "grad_norm": 1.127238154411316, + "learning_rate": 0.0001, + "loss": 79.729, + "step": 1570 + }, + { + "epoch": 2.5449690129490605, + "grad_norm": 1.1520992517471313, + "learning_rate": 0.0001, + "loss": 78.5618, + "step": 1580 + }, + { + "epoch": 2.5610923565274346, + "grad_norm": 0.8632035255432129, + "learning_rate": 0.0001, + "loss": 79.032, + "step": 1590 + }, + { + "epoch": 2.5772157001058096, + "grad_norm": 1.0506329536437988, + "learning_rate": 0.0001, + "loss": 77.6204, + "step": 1600 + }, + { + "epoch": 2.593339043684184, + "grad_norm": 1.019895315170288, + "learning_rate": 0.0001, + "loss": 79.2634, + "step": 1610 + }, + { + "epoch": 2.6094623872625586, + "grad_norm": 1.1685571670532227, + "learning_rate": 0.0001, + "loss": 78.8371, + "step": 1620 + }, + { + "epoch": 2.625585730840933, + "grad_norm": 1.147546648979187, + "learning_rate": 0.0001, + "loss": 78.2113, + "step": 1630 + }, + { + "epoch": 2.6417090744193077, + "grad_norm": 1.7336841821670532, + "learning_rate": 0.0001, + "loss": 78.7866, + "step": 1640 + }, + { + "epoch": 2.657832417997682, + "grad_norm": 1.2150614261627197, + "learning_rate": 0.0001, + "loss": 78.5621, + "step": 1650 + }, + { + "epoch": 2.6739557615760567, + "grad_norm": 1.2257503271102905, + "learning_rate": 0.0001, + "loss": 77.5835, + "step": 1660 + }, + { + "epoch": 2.6900791051544313, + "grad_norm": 1.0375311374664307, + "learning_rate": 0.0001, + "loss": 77.7509, + "step": 1670 + }, + { + "epoch": 2.706202448732806, + "grad_norm": 1.2551004886627197, + "learning_rate": 0.0001, + "loss": 77.8615, + "step": 1680 + }, + { + "epoch": 2.7223257923111808, + "grad_norm": 1.0464223623275757, + "learning_rate": 0.0001, + "loss": 77.2837, + "step": 1690 + }, + { + "epoch": 2.738449135889555, + "grad_norm": 1.0254806280136108, + "learning_rate": 0.0001, + "loss": 76.9616, + "step": 1700 + }, + { + "epoch": 2.75457247946793, + "grad_norm": 1.1665682792663574, + "learning_rate": 0.0001, + "loss": 77.9108, + "step": 1710 + }, + { + "epoch": 2.7706958230463044, + "grad_norm": 1.1349453926086426, + "learning_rate": 0.0001, + "loss": 78.7636, + "step": 1720 + }, + { + "epoch": 2.786819166624679, + "grad_norm": 1.4961057901382446, + "learning_rate": 0.0001, + "loss": 76.3862, + "step": 1730 + }, + { + "epoch": 2.8029425102030534, + "grad_norm": 1.10721755027771, + "learning_rate": 0.0001, + "loss": 77.4105, + "step": 1740 + }, + { + "epoch": 2.819065853781428, + "grad_norm": 0.9878576397895813, + "learning_rate": 0.0001, + "loss": 78.4855, + "step": 1750 + }, + { + "epoch": 2.8351891973598025, + "grad_norm": 1.1722687482833862, + "learning_rate": 0.0001, + "loss": 75.1589, + "step": 1760 + }, + { + "epoch": 2.851312540938177, + "grad_norm": 1.6624362468719482, + "learning_rate": 0.0001, + "loss": 77.1647, + "step": 1770 + }, + { + "epoch": 2.8674358845165515, + "grad_norm": 0.9542295932769775, + "learning_rate": 0.0001, + "loss": 77.1656, + "step": 1780 + }, + { + "epoch": 2.883559228094926, + "grad_norm": 2.328542947769165, + "learning_rate": 0.0001, + "loss": 76.4686, + "step": 1790 + }, + { + "epoch": 2.899682571673301, + "grad_norm": 1.2532029151916504, + "learning_rate": 0.0001, + "loss": 76.1727, + "step": 1800 + }, + { + "epoch": 2.915805915251675, + "grad_norm": 0.9604716897010803, + "learning_rate": 0.0001, + "loss": 74.3132, + "step": 1810 + }, + { + "epoch": 2.93192925883005, + "grad_norm": 0.9182595610618591, + "learning_rate": 0.0001, + "loss": 77.8689, + "step": 1820 + }, + { + "epoch": 2.9480526024084246, + "grad_norm": 0.9623042941093445, + "learning_rate": 0.0001, + "loss": 77.7531, + "step": 1830 + }, + { + "epoch": 2.964175945986799, + "grad_norm": 1.0500119924545288, + "learning_rate": 0.0001, + "loss": 77.6007, + "step": 1840 + }, + { + "epoch": 2.9802992895651736, + "grad_norm": 1.0874103307724, + "learning_rate": 0.0001, + "loss": 77.042, + "step": 1850 + }, + { + "epoch": 2.996422633143548, + "grad_norm": 0.9955475330352783, + "learning_rate": 0.0001, + "loss": 75.468, + "step": 1860 + }, + { + "epoch": 3.011286340504862, + "grad_norm": 1.4281760454177856, + "learning_rate": 0.0001, + "loss": 69.5972, + "step": 1870 + }, + { + "epoch": 3.0274096840832367, + "grad_norm": 1.0292123556137085, + "learning_rate": 0.0001, + "loss": 76.0186, + "step": 1880 + }, + { + "epoch": 3.0435330276616113, + "grad_norm": 0.9535816311836243, + "learning_rate": 0.0001, + "loss": 76.0979, + "step": 1890 + }, + { + "epoch": 3.059656371239986, + "grad_norm": 0.9099674820899963, + "learning_rate": 0.0001, + "loss": 75.9641, + "step": 1900 + }, + { + "epoch": 3.0757797148183603, + "grad_norm": 0.9568380117416382, + "learning_rate": 0.0001, + "loss": 75.0656, + "step": 1910 + }, + { + "epoch": 3.091903058396735, + "grad_norm": 0.9701060652732849, + "learning_rate": 0.0001, + "loss": 75.1422, + "step": 1920 + }, + { + "epoch": 3.1080264019751094, + "grad_norm": 1.3879098892211914, + "learning_rate": 0.0001, + "loss": 76.0731, + "step": 1930 + }, + { + "epoch": 3.1241497455534843, + "grad_norm": 0.9782084226608276, + "learning_rate": 0.0001, + "loss": 75.9666, + "step": 1940 + }, + { + "epoch": 3.140273089131859, + "grad_norm": 1.0856964588165283, + "learning_rate": 0.0001, + "loss": 75.0217, + "step": 1950 + }, + { + "epoch": 3.1563964327102334, + "grad_norm": 0.8506444692611694, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 1960 + }, + { + "epoch": 3.172519776288608, + "grad_norm": 1.044980525970459, + "learning_rate": 0.0001, + "loss": 75.3471, + "step": 1970 + }, + { + "epoch": 3.1886431198669825, + "grad_norm": 0.8432291746139526, + "learning_rate": 0.0001, + "loss": 75.3563, + "step": 1980 + }, + { + "epoch": 3.204766463445357, + "grad_norm": 0.9651834964752197, + "learning_rate": 0.0001, + "loss": 76.3991, + "step": 1990 + }, + { + "epoch": 3.2208898070237315, + "grad_norm": 0.957943320274353, + "learning_rate": 0.0001, + "loss": 77.2613, + "step": 2000 + }, + { + "epoch": 3.237013150602106, + "grad_norm": 0.9495745301246643, + "learning_rate": 0.0001, + "loss": 73.3801, + "step": 2010 + }, + { + "epoch": 3.2531364941804806, + "grad_norm": 1.182983160018921, + "learning_rate": 0.0001, + "loss": 74.5695, + "step": 2020 + }, + { + "epoch": 3.269259837758855, + "grad_norm": 0.828452467918396, + "learning_rate": 0.0001, + "loss": 75.4313, + "step": 2030 + }, + { + "epoch": 3.2853831813372296, + "grad_norm": 0.8340104818344116, + "learning_rate": 0.0001, + "loss": 75.1086, + "step": 2040 + }, + { + "epoch": 3.301506524915604, + "grad_norm": 1.1700762510299683, + "learning_rate": 0.0001, + "loss": 75.7496, + "step": 2050 + }, + { + "epoch": 3.317629868493979, + "grad_norm": 0.8709110617637634, + "learning_rate": 0.0001, + "loss": 73.8403, + "step": 2060 + }, + { + "epoch": 3.3337532120723536, + "grad_norm": 0.8928954601287842, + "learning_rate": 0.0001, + "loss": 75.6134, + "step": 2070 + }, + { + "epoch": 3.349876555650728, + "grad_norm": 0.9787353277206421, + "learning_rate": 0.0001, + "loss": 75.8116, + "step": 2080 + }, + { + "epoch": 3.3659998992291027, + "grad_norm": 0.9515123963356018, + "learning_rate": 0.0001, + "loss": 75.5024, + "step": 2090 + }, + { + "epoch": 3.3821232428074772, + "grad_norm": 0.9529262185096741, + "learning_rate": 0.0001, + "loss": 74.1553, + "step": 2100 + }, + { + "epoch": 3.3982465863858518, + "grad_norm": 0.8934710621833801, + "learning_rate": 0.0001, + "loss": 74.1501, + "step": 2110 + }, + { + "epoch": 3.4143699299642263, + "grad_norm": 1.089447021484375, + "learning_rate": 0.0001, + "loss": 74.9123, + "step": 2120 + }, + { + "epoch": 3.430493273542601, + "grad_norm": 1.2569622993469238, + "learning_rate": 0.0001, + "loss": 74.3962, + "step": 2130 + }, + { + "epoch": 3.4466166171209753, + "grad_norm": 1.020094394683838, + "learning_rate": 0.0001, + "loss": 74.6617, + "step": 2140 + }, + { + "epoch": 3.46273996069935, + "grad_norm": 1.1579667329788208, + "learning_rate": 0.0001, + "loss": 74.2745, + "step": 2150 + }, + { + "epoch": 3.4788633042777244, + "grad_norm": 1.448935627937317, + "learning_rate": 0.0001, + "loss": 71.5285, + "step": 2160 + }, + { + "epoch": 3.4949866478560994, + "grad_norm": 1.0030494928359985, + "learning_rate": 0.0001, + "loss": 74.6666, + "step": 2170 + }, + { + "epoch": 3.511109991434474, + "grad_norm": 0.9010888934135437, + "learning_rate": 0.0001, + "loss": 74.9355, + "step": 2180 + }, + { + "epoch": 3.5272333350128484, + "grad_norm": 1.1595124006271362, + "learning_rate": 0.0001, + "loss": 74.5049, + "step": 2190 + }, + { + "epoch": 3.543356678591223, + "grad_norm": 1.066405177116394, + "learning_rate": 0.0001, + "loss": 74.6336, + "step": 2200 + }, + { + "epoch": 3.5594800221695975, + "grad_norm": 1.2301596403121948, + "learning_rate": 0.0001, + "loss": 74.6664, + "step": 2210 + }, + { + "epoch": 3.575603365747972, + "grad_norm": 1.0073511600494385, + "learning_rate": 0.0001, + "loss": 73.1405, + "step": 2220 + }, + { + "epoch": 3.5917267093263465, + "grad_norm": 0.9913995862007141, + "learning_rate": 0.0001, + "loss": 75.7443, + "step": 2230 + }, + { + "epoch": 3.607850052904721, + "grad_norm": 0.9555648565292358, + "learning_rate": 0.0001, + "loss": 74.8036, + "step": 2240 + }, + { + "epoch": 3.6239733964830956, + "grad_norm": 0.9350649118423462, + "learning_rate": 0.0001, + "loss": 73.5783, + "step": 2250 + }, + { + "epoch": 3.64009674006147, + "grad_norm": 0.935840368270874, + "learning_rate": 0.0001, + "loss": 75.1694, + "step": 2260 + }, + { + "epoch": 3.6562200836398446, + "grad_norm": 0.9303803443908691, + "learning_rate": 0.0001, + "loss": 72.8282, + "step": 2270 + }, + { + "epoch": 3.6723434272182196, + "grad_norm": 0.937164843082428, + "learning_rate": 0.0001, + "loss": 70.3305, + "step": 2280 + }, + { + "epoch": 3.6884667707965937, + "grad_norm": 1.027329444885254, + "learning_rate": 0.0001, + "loss": 73.748, + "step": 2290 + }, + { + "epoch": 3.7045901143749687, + "grad_norm": 1.0915215015411377, + "learning_rate": 0.0001, + "loss": 72.2265, + "step": 2300 + }, + { + "epoch": 3.720713457953343, + "grad_norm": 1.1031261682510376, + "learning_rate": 0.0001, + "loss": 71.2795, + "step": 2310 + }, + { + "epoch": 3.7368368015317177, + "grad_norm": 1.1466140747070312, + "learning_rate": 0.0001, + "loss": 72.0116, + "step": 2320 + }, + { + "epoch": 3.7529601451100922, + "grad_norm": 1.0441662073135376, + "learning_rate": 0.0001, + "loss": 73.3735, + "step": 2330 + }, + { + "epoch": 3.7690834886884668, + "grad_norm": 0.8610432744026184, + "learning_rate": 0.0001, + "loss": 73.3433, + "step": 2340 + }, + { + "epoch": 3.7852068322668413, + "grad_norm": 0.885720431804657, + "learning_rate": 0.0001, + "loss": 73.575, + "step": 2350 + }, + { + "epoch": 3.801330175845216, + "grad_norm": 1.0023950338363647, + "learning_rate": 0.0001, + "loss": 73.4166, + "step": 2360 + }, + { + "epoch": 3.8174535194235903, + "grad_norm": 1.3820304870605469, + "learning_rate": 0.0001, + "loss": 72.9599, + "step": 2370 + }, + { + "epoch": 3.833576863001965, + "grad_norm": 0.9645710587501526, + "learning_rate": 0.0001, + "loss": 72.3922, + "step": 2380 + }, + { + "epoch": 3.84970020658034, + "grad_norm": 1.273369312286377, + "learning_rate": 0.0001, + "loss": 72.0158, + "step": 2390 + }, + { + "epoch": 3.865823550158714, + "grad_norm": 0.859863817691803, + "learning_rate": 0.0001, + "loss": 74.6642, + "step": 2400 + }, + { + "epoch": 3.881946893737089, + "grad_norm": 0.9396682381629944, + "learning_rate": 0.0001, + "loss": 71.9003, + "step": 2410 + }, + { + "epoch": 3.8980702373154634, + "grad_norm": 0.9976151585578918, + "learning_rate": 0.0001, + "loss": 73.0716, + "step": 2420 + }, + { + "epoch": 3.914193580893838, + "grad_norm": 1.0730767250061035, + "learning_rate": 0.0001, + "loss": 72.7847, + "step": 2430 + }, + { + "epoch": 3.9303169244722125, + "grad_norm": 1.002018690109253, + "learning_rate": 0.0001, + "loss": 71.7309, + "step": 2440 + }, + { + "epoch": 3.946440268050587, + "grad_norm": 0.9102753400802612, + "learning_rate": 0.0001, + "loss": 71.616, + "step": 2450 + }, + { + "epoch": 3.9625636116289615, + "grad_norm": 0.9827467799186707, + "learning_rate": 0.0001, + "loss": 72.22, + "step": 2460 + }, + { + "epoch": 3.978686955207336, + "grad_norm": 1.2781893014907837, + "learning_rate": 0.0001, + "loss": 73.2048, + "step": 2470 + }, + { + "epoch": 3.9948102987857106, + "grad_norm": 0.9042410850524902, + "learning_rate": 0.0001, + "loss": 72.9843, + "step": 2480 + }, + { + "epoch": 4.009674006147025, + "grad_norm": 0.8662117123603821, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 2490 + }, + { + "epoch": 4.0257973497254, + "grad_norm": 0.9448756575584412, + "learning_rate": 0.0001, + "loss": 73.36, + "step": 2500 + }, + { + "epoch": 4.041920693303774, + "grad_norm": 0.9054566621780396, + "learning_rate": 0.0001, + "loss": 73.94, + "step": 2510 + }, + { + "epoch": 4.058044036882149, + "grad_norm": 0.957398533821106, + "learning_rate": 0.0001, + "loss": 73.2019, + "step": 2520 + }, + { + "epoch": 4.074167380460523, + "grad_norm": 0.9444408416748047, + "learning_rate": 0.0001, + "loss": 70.6553, + "step": 2530 + }, + { + "epoch": 4.090290724038898, + "grad_norm": 0.8716063499450684, + "learning_rate": 0.0001, + "loss": 70.1048, + "step": 2540 + }, + { + "epoch": 4.106414067617272, + "grad_norm": 0.949249267578125, + "learning_rate": 0.0001, + "loss": 72.4453, + "step": 2550 + }, + { + "epoch": 4.122537411195647, + "grad_norm": 0.9843282699584961, + "learning_rate": 0.0001, + "loss": 72.1725, + "step": 2560 + }, + { + "epoch": 4.138660754774021, + "grad_norm": 0.8977480530738831, + "learning_rate": 0.0001, + "loss": 72.5272, + "step": 2570 + }, + { + "epoch": 4.154784098352396, + "grad_norm": 0.9957827925682068, + "learning_rate": 0.0001, + "loss": 72.289, + "step": 2580 + }, + { + "epoch": 4.170907441930771, + "grad_norm": 1.1358791589736938, + "learning_rate": 0.0001, + "loss": 72.1949, + "step": 2590 + }, + { + "epoch": 4.187030785509145, + "grad_norm": 0.9953081607818604, + "learning_rate": 0.0001, + "loss": 72.6004, + "step": 2600 + }, + { + "epoch": 4.20315412908752, + "grad_norm": 1.019061803817749, + "learning_rate": 0.0001, + "loss": 72.3772, + "step": 2610 + }, + { + "epoch": 4.219277472665894, + "grad_norm": 0.9062562584877014, + "learning_rate": 0.0001, + "loss": 72.0566, + "step": 2620 + }, + { + "epoch": 4.235400816244269, + "grad_norm": 1.0877611637115479, + "learning_rate": 0.0001, + "loss": 72.854, + "step": 2630 + }, + { + "epoch": 4.251524159822643, + "grad_norm": 1.2419648170471191, + "learning_rate": 0.0001, + "loss": 72.6498, + "step": 2640 + }, + { + "epoch": 4.267647503401018, + "grad_norm": 0.9507225751876831, + "learning_rate": 0.0001, + "loss": 68.9583, + "step": 2650 + }, + { + "epoch": 4.283770846979392, + "grad_norm": 1.121191382408142, + "learning_rate": 0.0001, + "loss": 72.8464, + "step": 2660 + }, + { + "epoch": 4.299894190557767, + "grad_norm": 1.0543361902236938, + "learning_rate": 0.0001, + "loss": 71.9882, + "step": 2670 + }, + { + "epoch": 4.316017534136141, + "grad_norm": 0.8622955083847046, + "learning_rate": 0.0001, + "loss": 71.7373, + "step": 2680 + }, + { + "epoch": 4.332140877714516, + "grad_norm": 0.911253809928894, + "learning_rate": 0.0001, + "loss": 71.1968, + "step": 2690 + }, + { + "epoch": 4.34826422129289, + "grad_norm": 4.405472755432129, + "learning_rate": 0.0001, + "loss": 70.4128, + "step": 2700 + }, + { + "epoch": 4.364387564871265, + "grad_norm": 0.9142985939979553, + "learning_rate": 0.0001, + "loss": 69.7548, + "step": 2710 + }, + { + "epoch": 4.38051090844964, + "grad_norm": 1.1022522449493408, + "learning_rate": 0.0001, + "loss": 71.8282, + "step": 2720 + }, + { + "epoch": 4.396634252028014, + "grad_norm": 1.018386721611023, + "learning_rate": 0.0001, + "loss": 71.0853, + "step": 2730 + }, + { + "epoch": 4.412757595606389, + "grad_norm": 1.0483211278915405, + "learning_rate": 0.0001, + "loss": 72.1478, + "step": 2740 + }, + { + "epoch": 4.428880939184763, + "grad_norm": 1.026861548423767, + "learning_rate": 0.0001, + "loss": 71.8947, + "step": 2750 + }, + { + "epoch": 4.445004282763138, + "grad_norm": 1.1858552694320679, + "learning_rate": 0.0001, + "loss": 71.7016, + "step": 2760 + }, + { + "epoch": 4.461127626341512, + "grad_norm": 1.1212221384048462, + "learning_rate": 0.0001, + "loss": 71.4454, + "step": 2770 + }, + { + "epoch": 4.477250969919887, + "grad_norm": 0.8062102794647217, + "learning_rate": 0.0001, + "loss": 73.2672, + "step": 2780 + }, + { + "epoch": 4.493374313498261, + "grad_norm": 1.104483962059021, + "learning_rate": 0.0001, + "loss": 70.548, + "step": 2790 + }, + { + "epoch": 4.509497657076636, + "grad_norm": 0.942057728767395, + "learning_rate": 0.0001, + "loss": 72.3584, + "step": 2800 + }, + { + "epoch": 4.525621000655011, + "grad_norm": 0.9028333425521851, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 2810 + }, + { + "epoch": 4.541744344233385, + "grad_norm": 0.9096795916557312, + "learning_rate": 0.0001, + "loss": 70.6596, + "step": 2820 + }, + { + "epoch": 4.55786768781176, + "grad_norm": 0.9038829207420349, + "learning_rate": 0.0001, + "loss": 72.6148, + "step": 2830 + }, + { + "epoch": 4.573991031390134, + "grad_norm": 1.8511613607406616, + "learning_rate": 0.0001, + "loss": 71.7205, + "step": 2840 + }, + { + "epoch": 4.590114374968509, + "grad_norm": 1.3492343425750732, + "learning_rate": 0.0001, + "loss": 72.1492, + "step": 2850 + }, + { + "epoch": 4.6062377185468835, + "grad_norm": 1.1469528675079346, + "learning_rate": 0.0001, + "loss": 70.5892, + "step": 2860 + }, + { + "epoch": 4.622361062125258, + "grad_norm": 1.1374379396438599, + "learning_rate": 0.0001, + "loss": 70.061, + "step": 2870 + }, + { + "epoch": 4.6384844057036325, + "grad_norm": 0.9597565531730652, + "learning_rate": 0.0001, + "loss": 71.0869, + "step": 2880 + }, + { + "epoch": 4.6546077492820075, + "grad_norm": 0.8356521129608154, + "learning_rate": 0.0001, + "loss": 72.1891, + "step": 2890 + }, + { + "epoch": 4.670731092860382, + "grad_norm": 0.8882647752761841, + "learning_rate": 0.0001, + "loss": 71.3845, + "step": 2900 + }, + { + "epoch": 4.6868544364387565, + "grad_norm": 1.1904003620147705, + "learning_rate": 0.0001, + "loss": 72.3086, + "step": 2910 + }, + { + "epoch": 4.702977780017131, + "grad_norm": 0.8010085821151733, + "learning_rate": 0.0001, + "loss": 71.3051, + "step": 2920 + }, + { + "epoch": 4.719101123595506, + "grad_norm": 0.7572135329246521, + "learning_rate": 0.0001, + "loss": 72.5605, + "step": 2930 + }, + { + "epoch": 4.735224467173881, + "grad_norm": 1.0866942405700684, + "learning_rate": 0.0001, + "loss": 70.0935, + "step": 2940 + }, + { + "epoch": 4.751347810752255, + "grad_norm": 0.9358721971511841, + "learning_rate": 0.0001, + "loss": 71.0391, + "step": 2950 + }, + { + "epoch": 4.76747115433063, + "grad_norm": 1.0276161432266235, + "learning_rate": 0.0001, + "loss": 69.8462, + "step": 2960 + }, + { + "epoch": 4.783594497909004, + "grad_norm": 0.9573063850402832, + "learning_rate": 0.0001, + "loss": 69.7089, + "step": 2970 + }, + { + "epoch": 4.799717841487379, + "grad_norm": 1.0638892650604248, + "learning_rate": 0.0001, + "loss": 71.8376, + "step": 2980 + }, + { + "epoch": 4.815841185065753, + "grad_norm": 0.9894681572914124, + "learning_rate": 0.0001, + "loss": 70.1764, + "step": 2990 + }, + { + "epoch": 4.831964528644128, + "grad_norm": 1.1265769004821777, + "learning_rate": 0.0001, + "loss": 71.7918, + "step": 3000 + }, + { + "epoch": 4.848087872222502, + "grad_norm": 2.2792513370513916, + "learning_rate": 0.0001, + "loss": 70.4341, + "step": 3010 + }, + { + "epoch": 4.864211215800877, + "grad_norm": 0.8469329476356506, + "learning_rate": 0.0001, + "loss": 72.1137, + "step": 3020 + }, + { + "epoch": 4.880334559379252, + "grad_norm": 0.9333013296127319, + "learning_rate": 0.0001, + "loss": 71.0789, + "step": 3030 + }, + { + "epoch": 4.896457902957626, + "grad_norm": 0.7888994216918945, + "learning_rate": 0.0001, + "loss": 70.8197, + "step": 3040 + }, + { + "epoch": 4.912581246536, + "grad_norm": 0.9194874167442322, + "learning_rate": 0.0001, + "loss": 72.071, + "step": 3050 + }, + { + "epoch": 4.928704590114375, + "grad_norm": 1.1252421140670776, + "learning_rate": 0.0001, + "loss": 71.6943, + "step": 3060 + }, + { + "epoch": 4.94482793369275, + "grad_norm": 1.1801071166992188, + "learning_rate": 0.0001, + "loss": 70.8937, + "step": 3070 + }, + { + "epoch": 4.960951277271124, + "grad_norm": 0.9162415266036987, + "learning_rate": 0.0001, + "loss": 70.024, + "step": 3080 + }, + { + "epoch": 4.977074620849499, + "grad_norm": 0.8833363652229309, + "learning_rate": 0.0001, + "loss": 71.3835, + "step": 3090 + }, + { + "epoch": 4.993197964427873, + "grad_norm": 0.9985883831977844, + "learning_rate": 0.0001, + "loss": 71.635, + "step": 3100 + }, + { + "epoch": 5.0080616717891875, + "grad_norm": 1.097298264503479, + "learning_rate": 0.0001, + "loss": 65.3542, + "step": 3110 + }, + { + "epoch": 5.024185015367562, + "grad_norm": 0.9703903198242188, + "learning_rate": 0.0001, + "loss": 69.8587, + "step": 3120 + }, + { + "epoch": 5.0403083589459365, + "grad_norm": 0.8894070386886597, + "learning_rate": 0.0001, + "loss": 71.2868, + "step": 3130 + }, + { + "epoch": 5.056431702524311, + "grad_norm": 0.6984127163887024, + "learning_rate": 0.0001, + "loss": 69.0425, + "step": 3140 + }, + { + "epoch": 5.072555046102686, + "grad_norm": 0.8607762455940247, + "learning_rate": 0.0001, + "loss": 69.5354, + "step": 3150 + }, + { + "epoch": 5.08867838968106, + "grad_norm": 1.1215914487838745, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 3160 + }, + { + "epoch": 5.104801733259435, + "grad_norm": 0.9114409685134888, + "learning_rate": 0.0001, + "loss": 68.4186, + "step": 3170 + }, + { + "epoch": 5.12092507683781, + "grad_norm": 1.010400652885437, + "learning_rate": 0.0001, + "loss": 68.3965, + "step": 3180 + }, + { + "epoch": 5.137048420416184, + "grad_norm": 0.8741958141326904, + "learning_rate": 0.0001, + "loss": 70.6728, + "step": 3190 + }, + { + "epoch": 5.153171763994559, + "grad_norm": 0.8760207891464233, + "learning_rate": 0.0001, + "loss": 72.0704, + "step": 3200 + }, + { + "epoch": 5.169295107572933, + "grad_norm": 1.328247308731079, + "learning_rate": 0.0001, + "loss": 69.9082, + "step": 3210 + }, + { + "epoch": 5.185418451151308, + "grad_norm": 0.8263765573501587, + "learning_rate": 0.0001, + "loss": 70.3539, + "step": 3220 + }, + { + "epoch": 5.201541794729682, + "grad_norm": 0.9358834028244019, + "learning_rate": 0.0001, + "loss": 70.4221, + "step": 3230 + }, + { + "epoch": 5.217665138308057, + "grad_norm": 1.0769531726837158, + "learning_rate": 0.0001, + "loss": 68.8167, + "step": 3240 + }, + { + "epoch": 5.233788481886431, + "grad_norm": 1.1438790559768677, + "learning_rate": 0.0001, + "loss": 68.9288, + "step": 3250 + }, + { + "epoch": 5.249911825464806, + "grad_norm": 0.9534218311309814, + "learning_rate": 0.0001, + "loss": 70.3277, + "step": 3260 + }, + { + "epoch": 5.26603516904318, + "grad_norm": 0.88532954454422, + "learning_rate": 0.0001, + "loss": 67.5781, + "step": 3270 + }, + { + "epoch": 5.282158512621555, + "grad_norm": 0.7986133694648743, + "learning_rate": 0.0001, + "loss": 69.759, + "step": 3280 + }, + { + "epoch": 5.29828185619993, + "grad_norm": 0.9100566506385803, + "learning_rate": 0.0001, + "loss": 68.6368, + "step": 3290 + }, + { + "epoch": 5.314405199778304, + "grad_norm": 0.8861745595932007, + "learning_rate": 0.0001, + "loss": 70.5756, + "step": 3300 + }, + { + "epoch": 5.330528543356679, + "grad_norm": 1.0729680061340332, + "learning_rate": 0.0001, + "loss": 70.3178, + "step": 3310 + }, + { + "epoch": 5.346651886935053, + "grad_norm": 1.0288280248641968, + "learning_rate": 0.0001, + "loss": 69.422, + "step": 3320 + }, + { + "epoch": 5.362775230513428, + "grad_norm": 0.8335140347480774, + "learning_rate": 0.0001, + "loss": 69.8281, + "step": 3330 + }, + { + "epoch": 5.378898574091802, + "grad_norm": 0.9789717197418213, + "learning_rate": 0.0001, + "loss": 71.1532, + "step": 3340 + }, + { + "epoch": 5.395021917670177, + "grad_norm": 0.9884677529335022, + "learning_rate": 0.0001, + "loss": 69.5673, + "step": 3350 + }, + { + "epoch": 5.411145261248551, + "grad_norm": 1.050154447555542, + "learning_rate": 0.0001, + "loss": 70.8015, + "step": 3360 + }, + { + "epoch": 5.427268604826926, + "grad_norm": 0.9645342230796814, + "learning_rate": 0.0001, + "loss": 69.1615, + "step": 3370 + }, + { + "epoch": 5.4433919484053, + "grad_norm": 0.8383159041404724, + "learning_rate": 0.0001, + "loss": 70.6299, + "step": 3380 + }, + { + "epoch": 5.459515291983675, + "grad_norm": 0.9250556826591492, + "learning_rate": 0.0001, + "loss": 69.231, + "step": 3390 + }, + { + "epoch": 5.475638635562049, + "grad_norm": 0.8108687400817871, + "learning_rate": 0.0001, + "loss": 70.5858, + "step": 3400 + }, + { + "epoch": 5.491761979140424, + "grad_norm": 0.8748818039894104, + "learning_rate": 0.0001, + "loss": 69.9001, + "step": 3410 + }, + { + "epoch": 5.507885322718799, + "grad_norm": 0.7411381006240845, + "learning_rate": 0.0001, + "loss": 68.3248, + "step": 3420 + }, + { + "epoch": 5.524008666297173, + "grad_norm": 1.0753326416015625, + "learning_rate": 0.0001, + "loss": 69.5831, + "step": 3430 + }, + { + "epoch": 5.540132009875548, + "grad_norm": 0.875254213809967, + "learning_rate": 0.0001, + "loss": 69.7516, + "step": 3440 + }, + { + "epoch": 5.556255353453922, + "grad_norm": 0.7637733817100525, + "learning_rate": 0.0001, + "loss": 68.0078, + "step": 3450 + }, + { + "epoch": 5.572378697032297, + "grad_norm": 0.968203604221344, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 3460 + }, + { + "epoch": 5.588502040610671, + "grad_norm": 0.8903141617774963, + "learning_rate": 0.0001, + "loss": 68.5835, + "step": 3470 + }, + { + "epoch": 5.604625384189046, + "grad_norm": 0.9536195397377014, + "learning_rate": 0.0001, + "loss": 71.0259, + "step": 3480 + }, + { + "epoch": 5.62074872776742, + "grad_norm": 0.8951714038848877, + "learning_rate": 0.0001, + "loss": 69.4422, + "step": 3490 + }, + { + "epoch": 5.636872071345795, + "grad_norm": 0.9639522433280945, + "learning_rate": 0.0001, + "loss": 66.5173, + "step": 3500 + }, + { + "epoch": 5.65299541492417, + "grad_norm": 1.0022730827331543, + "learning_rate": 0.0001, + "loss": 68.3443, + "step": 3510 + }, + { + "epoch": 5.669118758502544, + "grad_norm": 1.0654487609863281, + "learning_rate": 0.0001, + "loss": 69.0835, + "step": 3520 + }, + { + "epoch": 5.685242102080919, + "grad_norm": 0.9934629797935486, + "learning_rate": 0.0001, + "loss": 69.2278, + "step": 3530 + }, + { + "epoch": 5.7013654456592935, + "grad_norm": 0.8773566484451294, + "learning_rate": 0.0001, + "loss": 68.9339, + "step": 3540 + }, + { + "epoch": 5.7174887892376685, + "grad_norm": 0.7756998538970947, + "learning_rate": 0.0001, + "loss": 69.2683, + "step": 3550 + }, + { + "epoch": 5.7336121328160425, + "grad_norm": 0.8819359540939331, + "learning_rate": 0.0001, + "loss": 68.5071, + "step": 3560 + }, + { + "epoch": 5.7497354763944175, + "grad_norm": 1.1069825887680054, + "learning_rate": 0.0001, + "loss": 68.5236, + "step": 3570 + }, + { + "epoch": 5.765858819972792, + "grad_norm": 0.8006312847137451, + "learning_rate": 0.0001, + "loss": 68.7192, + "step": 3580 + }, + { + "epoch": 5.781982163551167, + "grad_norm": 1.0278719663619995, + "learning_rate": 0.0001, + "loss": 69.2545, + "step": 3590 + }, + { + "epoch": 5.798105507129541, + "grad_norm": 0.9205041527748108, + "learning_rate": 0.0001, + "loss": 69.7698, + "step": 3600 + }, + { + "epoch": 5.814228850707916, + "grad_norm": 1.0769038200378418, + "learning_rate": 0.0001, + "loss": 67.6568, + "step": 3610 + }, + { + "epoch": 5.83035219428629, + "grad_norm": 1.0565663576126099, + "learning_rate": 0.0001, + "loss": 68.9166, + "step": 3620 + }, + { + "epoch": 5.846475537864665, + "grad_norm": 0.9008663296699524, + "learning_rate": 0.0001, + "loss": 69.5837, + "step": 3630 + }, + { + "epoch": 5.86259888144304, + "grad_norm": 0.8369411826133728, + "learning_rate": 0.0001, + "loss": 68.5661, + "step": 3640 + }, + { + "epoch": 5.878722225021414, + "grad_norm": 1.0472379922866821, + "learning_rate": 0.0001, + "loss": 68.3122, + "step": 3650 + }, + { + "epoch": 5.894845568599789, + "grad_norm": 0.965277373790741, + "learning_rate": 0.0001, + "loss": 69.8062, + "step": 3660 + }, + { + "epoch": 5.910968912178163, + "grad_norm": 0.8936495780944824, + "learning_rate": 0.0001, + "loss": 67.1016, + "step": 3670 + }, + { + "epoch": 5.927092255756538, + "grad_norm": 0.8559162020683289, + "learning_rate": 0.0001, + "loss": 69.9986, + "step": 3680 + }, + { + "epoch": 5.943215599334912, + "grad_norm": 1.4815773963928223, + "learning_rate": 0.0001, + "loss": 66.7648, + "step": 3690 + }, + { + "epoch": 5.959338942913287, + "grad_norm": 0.9034469723701477, + "learning_rate": 0.0001, + "loss": 68.0767, + "step": 3700 + }, + { + "epoch": 5.975462286491661, + "grad_norm": 1.838799238204956, + "learning_rate": 0.0001, + "loss": 66.5848, + "step": 3710 + }, + { + "epoch": 5.991585630070036, + "grad_norm": 0.9274787902832031, + "learning_rate": 0.0001, + "loss": 70.0396, + "step": 3720 + }, + { + "epoch": 6.0064493374313495, + "grad_norm": 1.016974687576294, + "learning_rate": 0.0001, + "loss": 62.2268, + "step": 3730 + }, + { + "epoch": 6.022572681009724, + "grad_norm": 1.0522642135620117, + "learning_rate": 0.0001, + "loss": 69.3381, + "step": 3740 + }, + { + "epoch": 6.0386960245880985, + "grad_norm": 1.1157186031341553, + "learning_rate": 0.0001, + "loss": 68.8969, + "step": 3750 + }, + { + "epoch": 6.0548193681664735, + "grad_norm": 0.9913288950920105, + "learning_rate": 0.0001, + "loss": 67.3634, + "step": 3760 + }, + { + "epoch": 6.0709427117448485, + "grad_norm": 0.8823490142822266, + "learning_rate": 0.0001, + "loss": 70.978, + "step": 3770 + }, + { + "epoch": 6.0870660553232225, + "grad_norm": 0.853837788105011, + "learning_rate": 0.0001, + "loss": 68.9924, + "step": 3780 + }, + { + "epoch": 6.1031893989015975, + "grad_norm": 0.7937766909599304, + "learning_rate": 0.0001, + "loss": 68.1994, + "step": 3790 + }, + { + "epoch": 6.119312742479972, + "grad_norm": 1.7638057470321655, + "learning_rate": 0.0001, + "loss": 68.5078, + "step": 3800 + }, + { + "epoch": 6.135436086058347, + "grad_norm": 1.0542887449264526, + "learning_rate": 0.0001, + "loss": 68.9362, + "step": 3810 + }, + { + "epoch": 6.151559429636721, + "grad_norm": 1.06019926071167, + "learning_rate": 0.0001, + "loss": 67.2493, + "step": 3820 + }, + { + "epoch": 6.167682773215096, + "grad_norm": 0.9765111804008484, + "learning_rate": 0.0001, + "loss": 70.4797, + "step": 3830 + }, + { + "epoch": 6.18380611679347, + "grad_norm": 1.006372332572937, + "learning_rate": 0.0001, + "loss": 69.4233, + "step": 3840 + }, + { + "epoch": 6.199929460371845, + "grad_norm": 0.8274241089820862, + "learning_rate": 0.0001, + "loss": 67.9907, + "step": 3850 + }, + { + "epoch": 6.216052803950219, + "grad_norm": 0.9905765056610107, + "learning_rate": 0.0001, + "loss": 69.6988, + "step": 3860 + }, + { + "epoch": 6.232176147528594, + "grad_norm": 1.0065921545028687, + "learning_rate": 0.0001, + "loss": 68.7678, + "step": 3870 + }, + { + "epoch": 6.248299491106969, + "grad_norm": 0.8845255970954895, + "learning_rate": 0.0001, + "loss": 68.3039, + "step": 3880 + }, + { + "epoch": 6.264422834685343, + "grad_norm": 0.9030968546867371, + "learning_rate": 0.0001, + "loss": 68.671, + "step": 3890 + }, + { + "epoch": 6.280546178263718, + "grad_norm": 0.8650258183479309, + "learning_rate": 0.0001, + "loss": 68.5309, + "step": 3900 + }, + { + "epoch": 6.296669521842092, + "grad_norm": 0.8961272835731506, + "learning_rate": 0.0001, + "loss": 67.5611, + "step": 3910 + }, + { + "epoch": 6.312792865420467, + "grad_norm": 1.1702407598495483, + "learning_rate": 0.0001, + "loss": 68.2768, + "step": 3920 + }, + { + "epoch": 6.328916208998841, + "grad_norm": 0.8598962426185608, + "learning_rate": 0.0001, + "loss": 69.1961, + "step": 3930 + }, + { + "epoch": 6.345039552577216, + "grad_norm": 0.8714122176170349, + "learning_rate": 0.0001, + "loss": 66.8628, + "step": 3940 + }, + { + "epoch": 6.36116289615559, + "grad_norm": 0.7757895588874817, + "learning_rate": 0.0001, + "loss": 68.6908, + "step": 3950 + }, + { + "epoch": 6.377286239733965, + "grad_norm": 0.9189490675926208, + "learning_rate": 0.0001, + "loss": 67.8685, + "step": 3960 + }, + { + "epoch": 6.393409583312339, + "grad_norm": 0.8161299228668213, + "learning_rate": 0.0001, + "loss": 66.816, + "step": 3970 + }, + { + "epoch": 6.409532926890714, + "grad_norm": 0.8559791445732117, + "learning_rate": 0.0001, + "loss": 68.4302, + "step": 3980 + }, + { + "epoch": 6.425656270469089, + "grad_norm": 0.8087984919548035, + "learning_rate": 0.0001, + "loss": 66.2646, + "step": 3990 + }, + { + "epoch": 6.441779614047463, + "grad_norm": 0.7733110785484314, + "learning_rate": 0.0001, + "loss": 69.2769, + "step": 4000 + }, + { + "epoch": 6.457902957625838, + "grad_norm": 0.9335984587669373, + "learning_rate": 0.0001, + "loss": 67.1199, + "step": 4010 + }, + { + "epoch": 6.474026301204212, + "grad_norm": 0.9546418786048889, + "learning_rate": 0.0001, + "loss": 68.8706, + "step": 4020 + }, + { + "epoch": 6.490149644782587, + "grad_norm": 0.9670593738555908, + "learning_rate": 0.0001, + "loss": 68.2329, + "step": 4030 + }, + { + "epoch": 6.506272988360961, + "grad_norm": 0.8428588509559631, + "learning_rate": 0.0001, + "loss": 68.0105, + "step": 4040 + }, + { + "epoch": 6.522396331939336, + "grad_norm": 0.9755622744560242, + "learning_rate": 0.0001, + "loss": 68.656, + "step": 4050 + }, + { + "epoch": 6.53851967551771, + "grad_norm": 0.8972346186637878, + "learning_rate": 0.0001, + "loss": 69.3665, + "step": 4060 + }, + { + "epoch": 6.554643019096085, + "grad_norm": 0.9716668128967285, + "learning_rate": 0.0001, + "loss": 68.1053, + "step": 4070 + }, + { + "epoch": 6.570766362674459, + "grad_norm": 0.8147910833358765, + "learning_rate": 0.0001, + "loss": 67.9778, + "step": 4080 + }, + { + "epoch": 6.586889706252834, + "grad_norm": 0.8304234147071838, + "learning_rate": 0.0001, + "loss": 67.7069, + "step": 4090 + }, + { + "epoch": 6.603013049831208, + "grad_norm": 0.9356856346130371, + "learning_rate": 0.0001, + "loss": 68.1942, + "step": 4100 + }, + { + "epoch": 6.619136393409583, + "grad_norm": 1.017134189605713, + "learning_rate": 0.0001, + "loss": 67.4211, + "step": 4110 + }, + { + "epoch": 6.635259736987958, + "grad_norm": 1.142558217048645, + "learning_rate": 0.0001, + "loss": 68.4413, + "step": 4120 + }, + { + "epoch": 6.651383080566332, + "grad_norm": 0.9071153998374939, + "learning_rate": 0.0001, + "loss": 69.3666, + "step": 4130 + }, + { + "epoch": 6.667506424144707, + "grad_norm": 0.9620397686958313, + "learning_rate": 0.0001, + "loss": 68.208, + "step": 4140 + }, + { + "epoch": 6.683629767723081, + "grad_norm": 0.8282128572463989, + "learning_rate": 0.0001, + "loss": 67.1702, + "step": 4150 + }, + { + "epoch": 6.699753111301456, + "grad_norm": 0.8838199973106384, + "learning_rate": 0.0001, + "loss": 69.0074, + "step": 4160 + }, + { + "epoch": 6.71587645487983, + "grad_norm": 0.8864356279373169, + "learning_rate": 0.0001, + "loss": 67.8052, + "step": 4170 + }, + { + "epoch": 6.731999798458205, + "grad_norm": 0.9373385906219482, + "learning_rate": 0.0001, + "loss": 67.446, + "step": 4180 + }, + { + "epoch": 6.7481231420365795, + "grad_norm": 0.8308956027030945, + "learning_rate": 0.0001, + "loss": 68.716, + "step": 4190 + }, + { + "epoch": 6.7642464856149545, + "grad_norm": 0.8031372427940369, + "learning_rate": 0.0001, + "loss": 68.0482, + "step": 4200 + }, + { + "epoch": 6.780369829193329, + "grad_norm": 0.9258586764335632, + "learning_rate": 0.0001, + "loss": 68.5607, + "step": 4210 + }, + { + "epoch": 6.7964931727717035, + "grad_norm": 0.8147327899932861, + "learning_rate": 0.0001, + "loss": 67.7265, + "step": 4220 + }, + { + "epoch": 6.8126165163500785, + "grad_norm": 0.8717535734176636, + "learning_rate": 0.0001, + "loss": 67.8097, + "step": 4230 + }, + { + "epoch": 6.828739859928453, + "grad_norm": 0.9876209497451782, + "learning_rate": 0.0001, + "loss": 67.9068, + "step": 4240 + }, + { + "epoch": 6.8448632035068275, + "grad_norm": 0.718376874923706, + "learning_rate": 0.0001, + "loss": 68.1762, + "step": 4250 + }, + { + "epoch": 6.860986547085202, + "grad_norm": 0.8264320492744446, + "learning_rate": 0.0001, + "loss": 67.8391, + "step": 4260 + }, + { + "epoch": 6.877109890663577, + "grad_norm": 0.7478179335594177, + "learning_rate": 0.0001, + "loss": 67.6635, + "step": 4270 + }, + { + "epoch": 6.893233234241951, + "grad_norm": 0.8394957184791565, + "learning_rate": 0.0001, + "loss": 68.6073, + "step": 4280 + }, + { + "epoch": 6.909356577820326, + "grad_norm": 0.8534071445465088, + "learning_rate": 0.0001, + "loss": 69.6206, + "step": 4290 + }, + { + "epoch": 6.9254799213987, + "grad_norm": 0.9137582778930664, + "learning_rate": 0.0001, + "loss": 67.4362, + "step": 4300 + }, + { + "epoch": 6.941603264977075, + "grad_norm": 0.8869971036911011, + "learning_rate": 0.0001, + "loss": 67.6811, + "step": 4310 + }, + { + "epoch": 6.957726608555449, + "grad_norm": 0.889025866985321, + "learning_rate": 0.0001, + "loss": 65.3354, + "step": 4320 + }, + { + "epoch": 6.973849952133824, + "grad_norm": 0.7968549728393555, + "learning_rate": 0.0001, + "loss": 66.192, + "step": 4330 + }, + { + "epoch": 6.989973295712199, + "grad_norm": 1.0077433586120605, + "learning_rate": 0.0001, + "loss": 67.5216, + "step": 4340 + }, + { + "epoch": 7.004837003073512, + "grad_norm": 0.7900649309158325, + "learning_rate": 0.0001, + "loss": 62.2421, + "step": 4350 + }, + { + "epoch": 7.020960346651887, + "grad_norm": 1.0160410404205322, + "learning_rate": 0.0001, + "loss": 68.0978, + "step": 4360 + }, + { + "epoch": 7.037083690230261, + "grad_norm": 0.841387927532196, + "learning_rate": 0.0001, + "loss": 68.1008, + "step": 4370 + }, + { + "epoch": 7.053207033808636, + "grad_norm": 0.87884122133255, + "learning_rate": 0.0001, + "loss": 66.4667, + "step": 4380 + }, + { + "epoch": 7.06933037738701, + "grad_norm": 0.8580564856529236, + "learning_rate": 0.0001, + "loss": 67.2002, + "step": 4390 + }, + { + "epoch": 7.085453720965385, + "grad_norm": 0.7560063600540161, + "learning_rate": 0.0001, + "loss": 67.6743, + "step": 4400 + }, + { + "epoch": 7.1015770645437595, + "grad_norm": 0.855478048324585, + "learning_rate": 0.0001, + "loss": 67.822, + "step": 4410 + }, + { + "epoch": 7.1177004081221344, + "grad_norm": 0.7994911670684814, + "learning_rate": 0.0001, + "loss": 66.0094, + "step": 4420 + }, + { + "epoch": 7.1338237517005085, + "grad_norm": 0.8728616833686829, + "learning_rate": 0.0001, + "loss": 66.405, + "step": 4430 + }, + { + "epoch": 7.1499470952788835, + "grad_norm": 0.8434081673622131, + "learning_rate": 0.0001, + "loss": 67.0535, + "step": 4440 + }, + { + "epoch": 7.166070438857258, + "grad_norm": 0.8543094396591187, + "learning_rate": 0.0001, + "loss": 66.2417, + "step": 4450 + }, + { + "epoch": 7.182193782435633, + "grad_norm": 0.7209319472312927, + "learning_rate": 0.0001, + "loss": 67.3958, + "step": 4460 + }, + { + "epoch": 7.1983171260140075, + "grad_norm": 0.6735154390335083, + "learning_rate": 0.0001, + "loss": 67.4086, + "step": 4470 + }, + { + "epoch": 7.214440469592382, + "grad_norm": 0.9126104116439819, + "learning_rate": 0.0001, + "loss": 68.4191, + "step": 4480 + }, + { + "epoch": 7.230563813170757, + "grad_norm": 0.9334300756454468, + "learning_rate": 0.0001, + "loss": 68.3247, + "step": 4490 + }, + { + "epoch": 7.246687156749131, + "grad_norm": 1.5956038236618042, + "learning_rate": 0.0001, + "loss": 68.2779, + "step": 4500 + }, + { + "epoch": 7.262810500327506, + "grad_norm": 1.5012775659561157, + "learning_rate": 0.0001, + "loss": 68.3533, + "step": 4510 + }, + { + "epoch": 7.27893384390588, + "grad_norm": 1.1376632452011108, + "learning_rate": 0.0001, + "loss": 65.3627, + "step": 4520 + }, + { + "epoch": 7.295057187484255, + "grad_norm": 0.8597291111946106, + "learning_rate": 0.0001, + "loss": 67.8653, + "step": 4530 + }, + { + "epoch": 7.311180531062629, + "grad_norm": 0.7645048499107361, + "learning_rate": 0.0001, + "loss": 65.4592, + "step": 4540 + }, + { + "epoch": 7.327303874641004, + "grad_norm": 0.7717318534851074, + "learning_rate": 0.0001, + "loss": 66.902, + "step": 4550 + }, + { + "epoch": 7.343427218219378, + "grad_norm": 0.8884950876235962, + "learning_rate": 0.0001, + "loss": 67.1858, + "step": 4560 + }, + { + "epoch": 7.359550561797753, + "grad_norm": 0.7826898694038391, + "learning_rate": 0.0001, + "loss": 68.4673, + "step": 4570 + }, + { + "epoch": 7.375673905376128, + "grad_norm": 0.7484182119369507, + "learning_rate": 0.0001, + "loss": 67.4859, + "step": 4580 + }, + { + "epoch": 7.391797248954502, + "grad_norm": 0.918194591999054, + "learning_rate": 0.0001, + "loss": 67.7393, + "step": 4590 + }, + { + "epoch": 7.407920592532877, + "grad_norm": 1.0696896314620972, + "learning_rate": 0.0001, + "loss": 67.0101, + "step": 4600 + }, + { + "epoch": 7.424043936111251, + "grad_norm": 0.8190692067146301, + "learning_rate": 0.0001, + "loss": 68.5358, + "step": 4610 + }, + { + "epoch": 7.440167279689626, + "grad_norm": 0.9063783884048462, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 4620 + }, + { + "epoch": 7.456290623268, + "grad_norm": 0.7588100433349609, + "learning_rate": 0.0001, + "loss": 68.0066, + "step": 4630 + }, + { + "epoch": 7.472413966846375, + "grad_norm": 0.897281289100647, + "learning_rate": 0.0001, + "loss": 67.2499, + "step": 4640 + }, + { + "epoch": 7.488537310424749, + "grad_norm": 0.9505569338798523, + "learning_rate": 0.0001, + "loss": 67.5452, + "step": 4650 + }, + { + "epoch": 7.504660654003124, + "grad_norm": 0.8409159183502197, + "learning_rate": 0.0001, + "loss": 66.6293, + "step": 4660 + }, + { + "epoch": 7.520783997581498, + "grad_norm": 0.9878574013710022, + "learning_rate": 0.0001, + "loss": 65.3379, + "step": 4670 + }, + { + "epoch": 7.536907341159873, + "grad_norm": 1.0410685539245605, + "learning_rate": 0.0001, + "loss": 65.8028, + "step": 4680 + }, + { + "epoch": 7.553030684738248, + "grad_norm": 0.8417725563049316, + "learning_rate": 0.0001, + "loss": 65.4283, + "step": 4690 + }, + { + "epoch": 7.569154028316622, + "grad_norm": 0.7660863399505615, + "learning_rate": 0.0001, + "loss": 66.6765, + "step": 4700 + }, + { + "epoch": 7.585277371894997, + "grad_norm": 0.8074233531951904, + "learning_rate": 0.0001, + "loss": 67.1082, + "step": 4710 + }, + { + "epoch": 7.601400715473371, + "grad_norm": 0.7653326988220215, + "learning_rate": 0.0001, + "loss": 66.028, + "step": 4720 + }, + { + "epoch": 7.617524059051746, + "grad_norm": 1.1764041185379028, + "learning_rate": 0.0001, + "loss": 65.2587, + "step": 4730 + }, + { + "epoch": 7.63364740263012, + "grad_norm": 1.7604950666427612, + "learning_rate": 0.0001, + "loss": 66.0543, + "step": 4740 + }, + { + "epoch": 7.649770746208495, + "grad_norm": 0.941438615322113, + "learning_rate": 0.0001, + "loss": 65.6954, + "step": 4750 + }, + { + "epoch": 7.665894089786869, + "grad_norm": 0.740069568157196, + "learning_rate": 0.0001, + "loss": 67.9701, + "step": 4760 + }, + { + "epoch": 7.682017433365244, + "grad_norm": 0.9867332577705383, + "learning_rate": 0.0001, + "loss": 66.1445, + "step": 4770 + }, + { + "epoch": 7.698140776943618, + "grad_norm": 0.8156166672706604, + "learning_rate": 0.0001, + "loss": 67.2647, + "step": 4780 + }, + { + "epoch": 7.714264120521993, + "grad_norm": 0.7689579129219055, + "learning_rate": 0.0001, + "loss": 67.0961, + "step": 4790 + }, + { + "epoch": 7.730387464100367, + "grad_norm": 0.8159700036048889, + "learning_rate": 0.0001, + "loss": 66.4974, + "step": 4800 + }, + { + "epoch": 7.746510807678742, + "grad_norm": 0.8554222583770752, + "learning_rate": 0.0001, + "loss": 64.5057, + "step": 4810 + }, + { + "epoch": 7.762634151257117, + "grad_norm": 0.856629490852356, + "learning_rate": 0.0001, + "loss": 67.7819, + "step": 4820 + }, + { + "epoch": 7.778757494835491, + "grad_norm": 1.157179832458496, + "learning_rate": 0.0001, + "loss": 66.0122, + "step": 4830 + }, + { + "epoch": 7.794880838413866, + "grad_norm": 0.9597711563110352, + "learning_rate": 0.0001, + "loss": 66.8248, + "step": 4840 + }, + { + "epoch": 7.8110041819922404, + "grad_norm": 0.9175744652748108, + "learning_rate": 0.0001, + "loss": 65.7998, + "step": 4850 + }, + { + "epoch": 7.827127525570615, + "grad_norm": 0.8467298150062561, + "learning_rate": 0.0001, + "loss": 67.5363, + "step": 4860 + }, + { + "epoch": 7.8432508691489895, + "grad_norm": 0.863922655582428, + "learning_rate": 0.0001, + "loss": 67.7219, + "step": 4870 + }, + { + "epoch": 7.8593742127273645, + "grad_norm": 0.9896290302276611, + "learning_rate": 0.0001, + "loss": 66.7098, + "step": 4880 + }, + { + "epoch": 7.8754975563057386, + "grad_norm": 0.8828520774841309, + "learning_rate": 0.0001, + "loss": 66.0192, + "step": 4890 + }, + { + "epoch": 7.8916208998841135, + "grad_norm": 0.9093656539916992, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 4900 + }, + { + "epoch": 7.9077442434624885, + "grad_norm": 0.8665091395378113, + "learning_rate": 0.0001, + "loss": 66.5851, + "step": 4910 + }, + { + "epoch": 7.923867587040863, + "grad_norm": 0.8027398586273193, + "learning_rate": 0.0001, + "loss": 64.8607, + "step": 4920 + }, + { + "epoch": 7.9399909306192376, + "grad_norm": 0.7808436751365662, + "learning_rate": 0.0001, + "loss": 65.052, + "step": 4930 + }, + { + "epoch": 7.956114274197612, + "grad_norm": 0.7246070504188538, + "learning_rate": 0.0001, + "loss": 64.789, + "step": 4940 + }, + { + "epoch": 7.972237617775987, + "grad_norm": 1.0405027866363525, + "learning_rate": 0.0001, + "loss": 66.7735, + "step": 4950 + }, + { + "epoch": 7.988360961354361, + "grad_norm": 0.812921404838562, + "learning_rate": 0.0001, + "loss": 64.1286, + "step": 4960 + }, + { + "epoch": 8.003224668715674, + "grad_norm": 0.884215772151947, + "learning_rate": 0.0001, + "loss": 61.0812, + "step": 4970 + }, + { + "epoch": 8.01934801229405, + "grad_norm": 0.8519641757011414, + "learning_rate": 0.0001, + "loss": 66.2784, + "step": 4980 + }, + { + "epoch": 8.035471355872424, + "grad_norm": 0.8331199884414673, + "learning_rate": 0.0001, + "loss": 67.1405, + "step": 4990 + }, + { + "epoch": 8.0515946994508, + "grad_norm": 0.8568746447563171, + "learning_rate": 0.0001, + "loss": 66.7835, + "step": 5000 + }, + { + "epoch": 8.067718043029172, + "grad_norm": 1.170482873916626, + "learning_rate": 0.0001, + "loss": 64.2269, + "step": 5010 + }, + { + "epoch": 8.083841386607547, + "grad_norm": 0.9171113967895508, + "learning_rate": 0.0001, + "loss": 66.5567, + "step": 5020 + }, + { + "epoch": 8.099964730185922, + "grad_norm": 0.8988714814186096, + "learning_rate": 0.0001, + "loss": 66.0471, + "step": 5030 + }, + { + "epoch": 8.116088073764297, + "grad_norm": 0.7651671171188354, + "learning_rate": 0.0001, + "loss": 65.7185, + "step": 5040 + }, + { + "epoch": 8.132211417342672, + "grad_norm": 0.6890843510627747, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 5050 + }, + { + "epoch": 8.148334760921045, + "grad_norm": 0.7683818340301514, + "learning_rate": 0.0001, + "loss": 65.895, + "step": 5060 + }, + { + "epoch": 8.16445810449942, + "grad_norm": 0.9398573637008667, + "learning_rate": 0.0001, + "loss": 64.674, + "step": 5070 + }, + { + "epoch": 8.180581448077795, + "grad_norm": 0.7708531022071838, + "learning_rate": 0.0001, + "loss": 65.0637, + "step": 5080 + }, + { + "epoch": 8.19670479165617, + "grad_norm": 0.8840665221214294, + "learning_rate": 0.0001, + "loss": 65.6449, + "step": 5090 + }, + { + "epoch": 8.212828135234544, + "grad_norm": 0.8390524983406067, + "learning_rate": 0.0001, + "loss": 65.4738, + "step": 5100 + }, + { + "epoch": 8.228951478812919, + "grad_norm": 1.7559128999710083, + "learning_rate": 0.0001, + "loss": 68.911, + "step": 5110 + }, + { + "epoch": 8.245074822391294, + "grad_norm": 0.8293817043304443, + "learning_rate": 0.0001, + "loss": 65.2601, + "step": 5120 + }, + { + "epoch": 8.261198165969668, + "grad_norm": 0.8947123289108276, + "learning_rate": 0.0001, + "loss": 65.8611, + "step": 5130 + }, + { + "epoch": 8.277321509548042, + "grad_norm": 0.8526618480682373, + "learning_rate": 0.0001, + "loss": 65.559, + "step": 5140 + }, + { + "epoch": 8.293444853126417, + "grad_norm": 1.7423408031463623, + "learning_rate": 0.0001, + "loss": 64.5007, + "step": 5150 + }, + { + "epoch": 8.309568196704792, + "grad_norm": 1.0512502193450928, + "learning_rate": 0.0001, + "loss": 66.2593, + "step": 5160 + }, + { + "epoch": 8.325691540283167, + "grad_norm": 0.8129869103431702, + "learning_rate": 0.0001, + "loss": 64.7989, + "step": 5170 + }, + { + "epoch": 8.341814883861542, + "grad_norm": 0.852199912071228, + "learning_rate": 0.0001, + "loss": 65.9627, + "step": 5180 + }, + { + "epoch": 8.357938227439915, + "grad_norm": 0.8932474851608276, + "learning_rate": 0.0001, + "loss": 64.1035, + "step": 5190 + }, + { + "epoch": 8.37406157101829, + "grad_norm": 1.2631597518920898, + "learning_rate": 0.0001, + "loss": 65.3406, + "step": 5200 + }, + { + "epoch": 8.390184914596665, + "grad_norm": 0.9062619209289551, + "learning_rate": 0.0001, + "loss": 65.1841, + "step": 5210 + }, + { + "epoch": 8.40630825817504, + "grad_norm": 0.9153940081596375, + "learning_rate": 0.0001, + "loss": 66.3852, + "step": 5220 + }, + { + "epoch": 8.422431601753413, + "grad_norm": 0.8056164979934692, + "learning_rate": 0.0001, + "loss": 66.511, + "step": 5230 + }, + { + "epoch": 8.438554945331788, + "grad_norm": 0.7506036162376404, + "learning_rate": 0.0001, + "loss": 65.275, + "step": 5240 + }, + { + "epoch": 8.454678288910163, + "grad_norm": 0.7994081377983093, + "learning_rate": 0.0001, + "loss": 64.9946, + "step": 5250 + }, + { + "epoch": 8.470801632488538, + "grad_norm": 0.9887873530387878, + "learning_rate": 0.0001, + "loss": 67.2057, + "step": 5260 + }, + { + "epoch": 8.486924976066913, + "grad_norm": 0.8495044112205505, + "learning_rate": 0.0001, + "loss": 66.2481, + "step": 5270 + }, + { + "epoch": 8.503048319645286, + "grad_norm": 0.7353820204734802, + "learning_rate": 0.0001, + "loss": 65.6163, + "step": 5280 + }, + { + "epoch": 8.519171663223661, + "grad_norm": 0.795216977596283, + "learning_rate": 0.0001, + "loss": 64.808, + "step": 5290 + }, + { + "epoch": 8.535295006802036, + "grad_norm": 0.8121851682662964, + "learning_rate": 0.0001, + "loss": 65.9755, + "step": 5300 + }, + { + "epoch": 8.55141835038041, + "grad_norm": 0.8164243698120117, + "learning_rate": 0.0001, + "loss": 66.0583, + "step": 5310 + }, + { + "epoch": 8.567541693958784, + "grad_norm": 0.9893137216567993, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 5320 + }, + { + "epoch": 8.583665037537159, + "grad_norm": 1.3616434335708618, + "learning_rate": 0.0001, + "loss": 64.1444, + "step": 5330 + }, + { + "epoch": 8.599788381115534, + "grad_norm": 0.7002740502357483, + "learning_rate": 0.0001, + "loss": 65.0906, + "step": 5340 + }, + { + "epoch": 8.615911724693909, + "grad_norm": 0.838424563407898, + "learning_rate": 0.0001, + "loss": 65.4578, + "step": 5350 + }, + { + "epoch": 8.632035068272282, + "grad_norm": 0.988902747631073, + "learning_rate": 0.0001, + "loss": 64.3405, + "step": 5360 + }, + { + "epoch": 8.648158411850657, + "grad_norm": 0.7941252589225769, + "learning_rate": 0.0001, + "loss": 67.0146, + "step": 5370 + }, + { + "epoch": 8.664281755429032, + "grad_norm": 0.7727184295654297, + "learning_rate": 0.0001, + "loss": 64.8011, + "step": 5380 + }, + { + "epoch": 8.680405099007407, + "grad_norm": 0.9514367580413818, + "learning_rate": 0.0001, + "loss": 65.8165, + "step": 5390 + }, + { + "epoch": 8.69652844258578, + "grad_norm": 0.870544970035553, + "learning_rate": 0.0001, + "loss": 65.803, + "step": 5400 + }, + { + "epoch": 8.712651786164155, + "grad_norm": 0.7348082065582275, + "learning_rate": 0.0001, + "loss": 65.8409, + "step": 5410 + }, + { + "epoch": 8.72877512974253, + "grad_norm": 0.7543814778327942, + "learning_rate": 0.0001, + "loss": 65.5958, + "step": 5420 + }, + { + "epoch": 8.744898473320905, + "grad_norm": 0.7520368099212646, + "learning_rate": 0.0001, + "loss": 65.3323, + "step": 5430 + }, + { + "epoch": 8.76102181689928, + "grad_norm": 0.7610031366348267, + "learning_rate": 0.0001, + "loss": 65.4386, + "step": 5440 + }, + { + "epoch": 8.777145160477653, + "grad_norm": 0.8854930400848389, + "learning_rate": 0.0001, + "loss": 66.181, + "step": 5450 + }, + { + "epoch": 8.793268504056028, + "grad_norm": 0.7659342885017395, + "learning_rate": 0.0001, + "loss": 65.2586, + "step": 5460 + }, + { + "epoch": 8.809391847634403, + "grad_norm": 0.8592572212219238, + "learning_rate": 0.0001, + "loss": 66.3306, + "step": 5470 + }, + { + "epoch": 8.825515191212778, + "grad_norm": 0.8169172406196594, + "learning_rate": 0.0001, + "loss": 65.8283, + "step": 5480 + }, + { + "epoch": 8.841638534791151, + "grad_norm": 0.7380614876747131, + "learning_rate": 0.0001, + "loss": 65.8711, + "step": 5490 + }, + { + "epoch": 8.857761878369526, + "grad_norm": 0.9993331432342529, + "learning_rate": 0.0001, + "loss": 64.9378, + "step": 5500 + }, + { + "epoch": 8.873885221947901, + "grad_norm": 0.6252862811088562, + "learning_rate": 0.0001, + "loss": 64.1655, + "step": 5510 + }, + { + "epoch": 8.890008565526276, + "grad_norm": 0.8441852331161499, + "learning_rate": 0.0001, + "loss": 64.9795, + "step": 5520 + }, + { + "epoch": 8.906131909104651, + "grad_norm": 0.7075499892234802, + "learning_rate": 0.0001, + "loss": 65.5843, + "step": 5530 + }, + { + "epoch": 8.922255252683025, + "grad_norm": 0.7374144196510315, + "learning_rate": 0.0001, + "loss": 67.5064, + "step": 5540 + }, + { + "epoch": 8.9383785962614, + "grad_norm": 0.8073632121086121, + "learning_rate": 0.0001, + "loss": 67.0182, + "step": 5550 + }, + { + "epoch": 8.954501939839774, + "grad_norm": 0.7576195001602173, + "learning_rate": 0.0001, + "loss": 65.9139, + "step": 5560 + }, + { + "epoch": 8.97062528341815, + "grad_norm": 1.6479626893997192, + "learning_rate": 0.0001, + "loss": 66.3517, + "step": 5570 + }, + { + "epoch": 8.986748626996523, + "grad_norm": 0.846513569355011, + "learning_rate": 0.0001, + "loss": 64.2908, + "step": 5580 + }, + { + "epoch": 9.001612334357837, + "grad_norm": 0.9551037549972534, + "learning_rate": 0.0001, + "loss": 61.7312, + "step": 5590 + }, + { + "epoch": 9.017735677936212, + "grad_norm": 0.8250304460525513, + "learning_rate": 0.0001, + "loss": 66.3351, + "step": 5600 + }, + { + "epoch": 9.033859021514587, + "grad_norm": 0.7932960391044617, + "learning_rate": 0.0001, + "loss": 66.7753, + "step": 5610 + }, + { + "epoch": 9.04998236509296, + "grad_norm": 0.9839881658554077, + "learning_rate": 0.0001, + "loss": 64.5146, + "step": 5620 + }, + { + "epoch": 9.066105708671335, + "grad_norm": 0.7337850332260132, + "learning_rate": 0.0001, + "loss": 65.0544, + "step": 5630 + }, + { + "epoch": 9.08222905224971, + "grad_norm": 0.7496634721755981, + "learning_rate": 0.0001, + "loss": 64.1451, + "step": 5640 + }, + { + "epoch": 9.098352395828085, + "grad_norm": 0.9457297921180725, + "learning_rate": 0.0001, + "loss": 65.3036, + "step": 5650 + }, + { + "epoch": 9.11447573940646, + "grad_norm": 0.8315348625183105, + "learning_rate": 0.0001, + "loss": 66.0585, + "step": 5660 + }, + { + "epoch": 9.130599082984833, + "grad_norm": 0.9666195511817932, + "learning_rate": 0.0001, + "loss": 64.6802, + "step": 5670 + }, + { + "epoch": 9.146722426563208, + "grad_norm": 0.7416576147079468, + "learning_rate": 0.0001, + "loss": 65.1557, + "step": 5680 + }, + { + "epoch": 9.162845770141583, + "grad_norm": 0.7723799347877502, + "learning_rate": 0.0001, + "loss": 66.2871, + "step": 5690 + }, + { + "epoch": 9.178969113719958, + "grad_norm": 0.7461305856704712, + "learning_rate": 0.0001, + "loss": 65.5795, + "step": 5700 + }, + { + "epoch": 9.195092457298331, + "grad_norm": 0.6663411259651184, + "learning_rate": 0.0001, + "loss": 66.4094, + "step": 5710 + }, + { + "epoch": 9.211215800876706, + "grad_norm": 0.9340134859085083, + "learning_rate": 0.0001, + "loss": 63.5753, + "step": 5720 + }, + { + "epoch": 9.227339144455081, + "grad_norm": 0.7643963694572449, + "learning_rate": 0.0001, + "loss": 65.3359, + "step": 5730 + }, + { + "epoch": 9.243462488033456, + "grad_norm": 0.6931570768356323, + "learning_rate": 0.0001, + "loss": 65.0755, + "step": 5740 + }, + { + "epoch": 9.259585831611831, + "grad_norm": 0.9096988439559937, + "learning_rate": 0.0001, + "loss": 64.7983, + "step": 5750 + }, + { + "epoch": 9.275709175190205, + "grad_norm": 0.8360128402709961, + "learning_rate": 0.0001, + "loss": 64.3604, + "step": 5760 + }, + { + "epoch": 9.29183251876858, + "grad_norm": 0.9302591681480408, + "learning_rate": 0.0001, + "loss": 66.2754, + "step": 5770 + }, + { + "epoch": 9.307955862346954, + "grad_norm": 0.7000735998153687, + "learning_rate": 0.0001, + "loss": 65.2241, + "step": 5780 + }, + { + "epoch": 9.32407920592533, + "grad_norm": 0.7321175932884216, + "learning_rate": 0.0001, + "loss": 65.3558, + "step": 5790 + }, + { + "epoch": 9.340202549503703, + "grad_norm": 0.7948715090751648, + "learning_rate": 0.0001, + "loss": 64.2793, + "step": 5800 + }, + { + "epoch": 9.356325893082078, + "grad_norm": 0.7676129341125488, + "learning_rate": 0.0001, + "loss": 66.4778, + "step": 5810 + }, + { + "epoch": 9.372449236660453, + "grad_norm": 0.977236270904541, + "learning_rate": 0.0001, + "loss": 64.5587, + "step": 5820 + }, + { + "epoch": 9.388572580238828, + "grad_norm": 0.86808842420578, + "learning_rate": 0.0001, + "loss": 65.8026, + "step": 5830 + }, + { + "epoch": 9.4046959238172, + "grad_norm": 0.7455324530601501, + "learning_rate": 0.0001, + "loss": 65.066, + "step": 5840 + }, + { + "epoch": 9.420819267395576, + "grad_norm": 0.6784212589263916, + "learning_rate": 0.0001, + "loss": 65.4167, + "step": 5850 + }, + { + "epoch": 9.43694261097395, + "grad_norm": 1.033731460571289, + "learning_rate": 0.0001, + "loss": 66.4502, + "step": 5860 + }, + { + "epoch": 9.453065954552326, + "grad_norm": 0.7946107983589172, + "learning_rate": 0.0001, + "loss": 63.7567, + "step": 5870 + }, + { + "epoch": 9.4691892981307, + "grad_norm": 0.7826364636421204, + "learning_rate": 0.0001, + "loss": 65.4484, + "step": 5880 + }, + { + "epoch": 9.485312641709074, + "grad_norm": 0.7605016827583313, + "learning_rate": 0.0001, + "loss": 66.8459, + "step": 5890 + }, + { + "epoch": 9.501435985287449, + "grad_norm": 0.7383126020431519, + "learning_rate": 0.0001, + "loss": 64.6873, + "step": 5900 + }, + { + "epoch": 9.517559328865824, + "grad_norm": 0.8355686664581299, + "learning_rate": 0.0001, + "loss": 63.5145, + "step": 5910 + }, + { + "epoch": 9.533682672444199, + "grad_norm": 0.8827091455459595, + "learning_rate": 0.0001, + "loss": 67.2207, + "step": 5920 + }, + { + "epoch": 9.549806016022572, + "grad_norm": 0.6988142728805542, + "learning_rate": 0.0001, + "loss": 64.2289, + "step": 5930 + }, + { + "epoch": 9.565929359600947, + "grad_norm": 0.7609000205993652, + "learning_rate": 0.0001, + "loss": 63.9324, + "step": 5940 + }, + { + "epoch": 9.582052703179322, + "grad_norm": 0.8053853511810303, + "learning_rate": 0.0001, + "loss": 62.5564, + "step": 5950 + }, + { + "epoch": 9.598176046757697, + "grad_norm": 0.7684671878814697, + "learning_rate": 0.0001, + "loss": 66.2669, + "step": 5960 + }, + { + "epoch": 9.614299390336072, + "grad_norm": 0.8411065340042114, + "learning_rate": 0.0001, + "loss": 64.5087, + "step": 5970 + }, + { + "epoch": 9.630422733914445, + "grad_norm": 0.963498592376709, + "learning_rate": 0.0001, + "loss": 68.1736, + "step": 5980 + }, + { + "epoch": 9.64654607749282, + "grad_norm": 0.8715981841087341, + "learning_rate": 0.0001, + "loss": 64.1686, + "step": 5990 + }, + { + "epoch": 9.662669421071195, + "grad_norm": 0.8217800855636597, + "learning_rate": 0.0001, + "loss": 64.2211, + "step": 6000 + }, + { + "epoch": 9.67879276464957, + "grad_norm": 0.7824742197990417, + "learning_rate": 0.0001, + "loss": 65.9052, + "step": 6010 + }, + { + "epoch": 9.694916108227943, + "grad_norm": 0.7440009117126465, + "learning_rate": 0.0001, + "loss": 63.3749, + "step": 6020 + }, + { + "epoch": 9.711039451806318, + "grad_norm": 0.8759479522705078, + "learning_rate": 0.0001, + "loss": 63.4488, + "step": 6030 + }, + { + "epoch": 9.727162795384693, + "grad_norm": 3.662144184112549, + "learning_rate": 0.0001, + "loss": 62.8787, + "step": 6040 + }, + { + "epoch": 9.743286138963068, + "grad_norm": 1.007149577140808, + "learning_rate": 0.0001, + "loss": 66.0053, + "step": 6050 + }, + { + "epoch": 9.759409482541441, + "grad_norm": 0.8369242548942566, + "learning_rate": 0.0001, + "loss": 67.233, + "step": 6060 + }, + { + "epoch": 9.775532826119816, + "grad_norm": 0.8160434365272522, + "learning_rate": 0.0001, + "loss": 65.2089, + "step": 6070 + }, + { + "epoch": 9.791656169698191, + "grad_norm": 0.8482319712638855, + "learning_rate": 0.0001, + "loss": 65.0718, + "step": 6080 + }, + { + "epoch": 9.807779513276566, + "grad_norm": 0.8800129890441895, + "learning_rate": 0.0001, + "loss": 65.145, + "step": 6090 + }, + { + "epoch": 9.82390285685494, + "grad_norm": 0.8911405801773071, + "learning_rate": 0.0001, + "loss": 65.7952, + "step": 6100 + }, + { + "epoch": 9.840026200433314, + "grad_norm": 0.8108358979225159, + "learning_rate": 0.0001, + "loss": 65.1928, + "step": 6110 + }, + { + "epoch": 9.85614954401169, + "grad_norm": 0.8919743895530701, + "learning_rate": 0.0001, + "loss": 65.8129, + "step": 6120 + }, + { + "epoch": 9.872272887590064, + "grad_norm": 0.73152756690979, + "learning_rate": 0.0001, + "loss": 65.7141, + "step": 6130 + }, + { + "epoch": 9.88839623116844, + "grad_norm": 0.8583903312683105, + "learning_rate": 0.0001, + "loss": 64.8231, + "step": 6140 + }, + { + "epoch": 9.904519574746812, + "grad_norm": 0.798543393611908, + "learning_rate": 0.0001, + "loss": 64.5073, + "step": 6150 + }, + { + "epoch": 9.920642918325187, + "grad_norm": 0.8657358288764954, + "learning_rate": 0.0001, + "loss": 64.1207, + "step": 6160 + }, + { + "epoch": 9.936766261903562, + "grad_norm": 0.9324677586555481, + "learning_rate": 0.0001, + "loss": 64.7129, + "step": 6170 + }, + { + "epoch": 9.952889605481937, + "grad_norm": 0.7371154427528381, + "learning_rate": 0.0001, + "loss": 64.0251, + "step": 6180 + }, + { + "epoch": 9.96901294906031, + "grad_norm": 0.8273237347602844, + "learning_rate": 0.0001, + "loss": 65.8633, + "step": 6190 + }, + { + "epoch": 9.985136292638686, + "grad_norm": 0.7219365239143372, + "learning_rate": 0.0001, + "loss": 63.4029, + "step": 6200 + }, + { + "epoch": 10.0, + "grad_norm": 0.76242995262146, + "learning_rate": 0.0001, + "loss": 60.9994, + "step": 6210 + }, + { + "epoch": 10.016123343578375, + "grad_norm": 0.8677287697792053, + "learning_rate": 0.0001, + "loss": 65.9687, + "step": 6220 + }, + { + "epoch": 10.03224668715675, + "grad_norm": 0.7073521018028259, + "learning_rate": 0.0001, + "loss": 63.0083, + "step": 6230 + }, + { + "epoch": 10.048370030735123, + "grad_norm": 0.794535756111145, + "learning_rate": 0.0001, + "loss": 63.0616, + "step": 6240 + }, + { + "epoch": 10.064493374313498, + "grad_norm": 0.6974631547927856, + "learning_rate": 0.0001, + "loss": 64.6579, + "step": 6250 + }, + { + "epoch": 10.080616717891873, + "grad_norm": 0.8279150128364563, + "learning_rate": 0.0001, + "loss": 65.1069, + "step": 6260 + }, + { + "epoch": 10.096740061470248, + "grad_norm": 0.8682932257652283, + "learning_rate": 0.0001, + "loss": 64.5019, + "step": 6270 + }, + { + "epoch": 10.112863405048621, + "grad_norm": 1.0158190727233887, + "learning_rate": 0.0001, + "loss": 65.8854, + "step": 6280 + }, + { + "epoch": 10.128986748626996, + "grad_norm": 0.6911011934280396, + "learning_rate": 0.0001, + "loss": 62.4879, + "step": 6290 + }, + { + "epoch": 10.145110092205371, + "grad_norm": 0.7394022941589355, + "learning_rate": 0.0001, + "loss": 64.2604, + "step": 6300 + }, + { + "epoch": 10.161233435783746, + "grad_norm": 0.7941621541976929, + "learning_rate": 0.0001, + "loss": 64.5133, + "step": 6310 + }, + { + "epoch": 10.17735677936212, + "grad_norm": 1.0238994359970093, + "learning_rate": 0.0001, + "loss": 63.8594, + "step": 6320 + }, + { + "epoch": 10.193480122940494, + "grad_norm": 0.7332146763801575, + "learning_rate": 0.0001, + "loss": 66.5434, + "step": 6330 + }, + { + "epoch": 10.20960346651887, + "grad_norm": 0.9375078082084656, + "learning_rate": 0.0001, + "loss": 63.4793, + "step": 6340 + }, + { + "epoch": 10.225726810097244, + "grad_norm": 0.715931236743927, + "learning_rate": 0.0001, + "loss": 65.8812, + "step": 6350 + }, + { + "epoch": 10.24185015367562, + "grad_norm": 1.291536808013916, + "learning_rate": 0.0001, + "loss": 64.9125, + "step": 6360 + }, + { + "epoch": 10.257973497253992, + "grad_norm": 0.8942781686782837, + "learning_rate": 0.0001, + "loss": 67.1706, + "step": 6370 + }, + { + "epoch": 10.274096840832367, + "grad_norm": 0.8272124528884888, + "learning_rate": 0.0001, + "loss": 62.1721, + "step": 6380 + }, + { + "epoch": 10.290220184410742, + "grad_norm": 0.8191152215003967, + "learning_rate": 0.0001, + "loss": 64.8931, + "step": 6390 + }, + { + "epoch": 10.306343527989117, + "grad_norm": 1.7500241994857788, + "learning_rate": 0.0001, + "loss": 62.9693, + "step": 6400 + }, + { + "epoch": 10.32246687156749, + "grad_norm": 1.0159430503845215, + "learning_rate": 0.0001, + "loss": 63.0889, + "step": 6410 + }, + { + "epoch": 10.338590215145866, + "grad_norm": 0.7300348281860352, + "learning_rate": 0.0001, + "loss": 65.1742, + "step": 6420 + }, + { + "epoch": 10.35471355872424, + "grad_norm": 1.3132387399673462, + "learning_rate": 0.0001, + "loss": 64.8867, + "step": 6430 + }, + { + "epoch": 10.370836902302615, + "grad_norm": 0.7502451539039612, + "learning_rate": 0.0001, + "loss": 66.5094, + "step": 6440 + }, + { + "epoch": 10.38696024588099, + "grad_norm": 0.9408686757087708, + "learning_rate": 0.0001, + "loss": 65.1515, + "step": 6450 + }, + { + "epoch": 10.403083589459364, + "grad_norm": 0.769598126411438, + "learning_rate": 0.0001, + "loss": 65.1478, + "step": 6460 + }, + { + "epoch": 10.419206933037739, + "grad_norm": 0.9086162447929382, + "learning_rate": 0.0001, + "loss": 63.5263, + "step": 6470 + }, + { + "epoch": 10.435330276616114, + "grad_norm": 0.7899322509765625, + "learning_rate": 0.0001, + "loss": 63.3939, + "step": 6480 + }, + { + "epoch": 10.451453620194489, + "grad_norm": 0.9135434031486511, + "learning_rate": 0.0001, + "loss": 63.867, + "step": 6490 + }, + { + "epoch": 10.467576963772862, + "grad_norm": 0.9312737584114075, + "learning_rate": 0.0001, + "loss": 63.6939, + "step": 6500 + }, + { + "epoch": 10.483700307351237, + "grad_norm": 0.861764132976532, + "learning_rate": 0.0001, + "loss": 63.4523, + "step": 6510 + }, + { + "epoch": 10.499823650929612, + "grad_norm": 0.7056366801261902, + "learning_rate": 0.0001, + "loss": 63.2889, + "step": 6520 + }, + { + "epoch": 10.515946994507987, + "grad_norm": 0.7007710337638855, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 6530 + }, + { + "epoch": 10.53207033808636, + "grad_norm": 0.7091413736343384, + "learning_rate": 0.0001, + "loss": 63.4185, + "step": 6540 + }, + { + "epoch": 10.548193681664735, + "grad_norm": 0.7147234678268433, + "learning_rate": 0.0001, + "loss": 65.0726, + "step": 6550 + }, + { + "epoch": 10.56431702524311, + "grad_norm": 0.7954673767089844, + "learning_rate": 0.0001, + "loss": 65.0334, + "step": 6560 + }, + { + "epoch": 10.580440368821485, + "grad_norm": 0.725352942943573, + "learning_rate": 0.0001, + "loss": 62.6663, + "step": 6570 + }, + { + "epoch": 10.59656371239986, + "grad_norm": 0.9563508629798889, + "learning_rate": 0.0001, + "loss": 64.0705, + "step": 6580 + }, + { + "epoch": 10.612687055978233, + "grad_norm": 0.7778734564781189, + "learning_rate": 0.0001, + "loss": 63.7619, + "step": 6590 + }, + { + "epoch": 10.628810399556608, + "grad_norm": 0.8414369821548462, + "learning_rate": 0.0001, + "loss": 63.9238, + "step": 6600 + }, + { + "epoch": 10.644933743134983, + "grad_norm": 0.8070562481880188, + "learning_rate": 0.0001, + "loss": 63.9591, + "step": 6610 + }, + { + "epoch": 10.661057086713358, + "grad_norm": 0.6493734121322632, + "learning_rate": 0.0001, + "loss": 63.7237, + "step": 6620 + }, + { + "epoch": 10.677180430291731, + "grad_norm": 0.8263372778892517, + "learning_rate": 0.0001, + "loss": 64.2262, + "step": 6630 + }, + { + "epoch": 10.693303773870106, + "grad_norm": 1.0090227127075195, + "learning_rate": 0.0001, + "loss": 63.5469, + "step": 6640 + }, + { + "epoch": 10.709427117448481, + "grad_norm": 0.998319685459137, + "learning_rate": 0.0001, + "loss": 62.9113, + "step": 6650 + }, + { + "epoch": 10.725550461026856, + "grad_norm": 1.006489634513855, + "learning_rate": 0.0001, + "loss": 66.1458, + "step": 6660 + }, + { + "epoch": 10.741673804605231, + "grad_norm": 0.9575757384300232, + "learning_rate": 0.0001, + "loss": 63.0913, + "step": 6670 + }, + { + "epoch": 10.757797148183604, + "grad_norm": 0.6832238435745239, + "learning_rate": 0.0001, + "loss": 66.7501, + "step": 6680 + }, + { + "epoch": 10.773920491761979, + "grad_norm": 0.8075717687606812, + "learning_rate": 0.0001, + "loss": 64.2839, + "step": 6690 + }, + { + "epoch": 10.790043835340354, + "grad_norm": 0.9402624368667603, + "learning_rate": 0.0001, + "loss": 64.9296, + "step": 6700 + }, + { + "epoch": 10.806167178918729, + "grad_norm": 0.7681158185005188, + "learning_rate": 0.0001, + "loss": 63.5686, + "step": 6710 + }, + { + "epoch": 10.822290522497102, + "grad_norm": 0.8451285362243652, + "learning_rate": 0.0001, + "loss": 64.5404, + "step": 6720 + }, + { + "epoch": 10.838413866075477, + "grad_norm": 0.7278769016265869, + "learning_rate": 0.0001, + "loss": 64.1341, + "step": 6730 + }, + { + "epoch": 10.854537209653852, + "grad_norm": 0.8524075746536255, + "learning_rate": 0.0001, + "loss": 64.0483, + "step": 6740 + }, + { + "epoch": 10.870660553232227, + "grad_norm": 0.703598141670227, + "learning_rate": 0.0001, + "loss": 64.0072, + "step": 6750 + }, + { + "epoch": 10.8867838968106, + "grad_norm": 0.7740384936332703, + "learning_rate": 0.0001, + "loss": 63.4171, + "step": 6760 + }, + { + "epoch": 10.902907240388975, + "grad_norm": 0.8302356004714966, + "learning_rate": 0.0001, + "loss": 65.5159, + "step": 6770 + }, + { + "epoch": 10.91903058396735, + "grad_norm": 1.0289733409881592, + "learning_rate": 0.0001, + "loss": 64.9785, + "step": 6780 + }, + { + "epoch": 10.935153927545725, + "grad_norm": 0.6721987128257751, + "learning_rate": 0.0001, + "loss": 63.965, + "step": 6790 + }, + { + "epoch": 10.951277271124098, + "grad_norm": 0.7614948749542236, + "learning_rate": 0.0001, + "loss": 64.1375, + "step": 6800 + }, + { + "epoch": 10.967400614702473, + "grad_norm": 0.7342216372489929, + "learning_rate": 0.0001, + "loss": 64.4342, + "step": 6810 + }, + { + "epoch": 10.983523958280848, + "grad_norm": 0.8808079957962036, + "learning_rate": 0.0001, + "loss": 65.2864, + "step": 6820 + }, + { + "epoch": 10.999647301859223, + "grad_norm": 0.818477988243103, + "learning_rate": 0.0001, + "loss": 63.9931, + "step": 6830 + }, + { + "epoch": 11.014511009220538, + "grad_norm": 0.7867474555969238, + "learning_rate": 0.0001, + "loss": 59.2426, + "step": 6840 + }, + { + "epoch": 11.030634352798911, + "grad_norm": 0.6163678765296936, + "learning_rate": 0.0001, + "loss": 65.0149, + "step": 6850 + }, + { + "epoch": 11.046757696377286, + "grad_norm": 0.8334609866142273, + "learning_rate": 0.0001, + "loss": 63.5017, + "step": 6860 + }, + { + "epoch": 11.062881039955661, + "grad_norm": 0.7335264086723328, + "learning_rate": 0.0001, + "loss": 64.6307, + "step": 6870 + }, + { + "epoch": 11.079004383534036, + "grad_norm": 0.7723846435546875, + "learning_rate": 0.0001, + "loss": 64.316, + "step": 6880 + }, + { + "epoch": 11.09512772711241, + "grad_norm": 0.7972145080566406, + "learning_rate": 0.0001, + "loss": 63.1469, + "step": 6890 + }, + { + "epoch": 11.111251070690784, + "grad_norm": 0.7696133852005005, + "learning_rate": 0.0001, + "loss": 64.0338, + "step": 6900 + }, + { + "epoch": 11.127374414269159, + "grad_norm": 0.8972133994102478, + "learning_rate": 0.0001, + "loss": 65.5727, + "step": 6910 + }, + { + "epoch": 11.143497757847534, + "grad_norm": 0.6776295900344849, + "learning_rate": 0.0001, + "loss": 66.176, + "step": 6920 + }, + { + "epoch": 11.159621101425909, + "grad_norm": 0.7345874905586243, + "learning_rate": 0.0001, + "loss": 64.0949, + "step": 6930 + }, + { + "epoch": 11.175744445004282, + "grad_norm": 0.7988374829292297, + "learning_rate": 0.0001, + "loss": 64.7721, + "step": 6940 + }, + { + "epoch": 11.191867788582657, + "grad_norm": 0.7552387714385986, + "learning_rate": 0.0001, + "loss": 63.1053, + "step": 6950 + }, + { + "epoch": 11.207991132161032, + "grad_norm": 0.6388214230537415, + "learning_rate": 0.0001, + "loss": 63.7871, + "step": 6960 + }, + { + "epoch": 11.224114475739407, + "grad_norm": 0.6837960481643677, + "learning_rate": 0.0001, + "loss": 63.906, + "step": 6970 + }, + { + "epoch": 11.24023781931778, + "grad_norm": 0.7050679326057434, + "learning_rate": 0.0001, + "loss": 64.2803, + "step": 6980 + }, + { + "epoch": 11.256361162896155, + "grad_norm": 0.8619478940963745, + "learning_rate": 0.0001, + "loss": 62.8786, + "step": 6990 + }, + { + "epoch": 11.27248450647453, + "grad_norm": 0.9759813547134399, + "learning_rate": 0.0001, + "loss": 65.2299, + "step": 7000 + }, + { + "epoch": 11.288607850052905, + "grad_norm": 0.7584514021873474, + "learning_rate": 0.0001, + "loss": 61.4178, + "step": 7010 + }, + { + "epoch": 11.304731193631278, + "grad_norm": 0.8467874526977539, + "learning_rate": 0.0001, + "loss": 65.6229, + "step": 7020 + }, + { + "epoch": 11.320854537209653, + "grad_norm": 0.7121654152870178, + "learning_rate": 0.0001, + "loss": 63.9061, + "step": 7030 + }, + { + "epoch": 11.336977880788028, + "grad_norm": 0.8885557055473328, + "learning_rate": 0.0001, + "loss": 65.1436, + "step": 7040 + }, + { + "epoch": 11.353101224366403, + "grad_norm": 0.6320177912712097, + "learning_rate": 0.0001, + "loss": 62.8535, + "step": 7050 + }, + { + "epoch": 11.369224567944778, + "grad_norm": 0.9287629723548889, + "learning_rate": 0.0001, + "loss": 62.7407, + "step": 7060 + }, + { + "epoch": 11.385347911523152, + "grad_norm": 0.7705608606338501, + "learning_rate": 0.0001, + "loss": 63.469, + "step": 7070 + }, + { + "epoch": 11.401471255101526, + "grad_norm": 0.6687246561050415, + "learning_rate": 0.0001, + "loss": 62.722, + "step": 7080 + }, + { + "epoch": 11.417594598679901, + "grad_norm": 0.7774648666381836, + "learning_rate": 0.0001, + "loss": 63.1145, + "step": 7090 + }, + { + "epoch": 11.433717942258276, + "grad_norm": 0.7808363437652588, + "learning_rate": 0.0001, + "loss": 63.9342, + "step": 7100 + }, + { + "epoch": 11.44984128583665, + "grad_norm": 0.8033562302589417, + "learning_rate": 0.0001, + "loss": 65.7876, + "step": 7110 + }, + { + "epoch": 11.465964629415025, + "grad_norm": 0.8143486976623535, + "learning_rate": 0.0001, + "loss": 64.8782, + "step": 7120 + }, + { + "epoch": 11.4820879729934, + "grad_norm": 0.7855561971664429, + "learning_rate": 0.0001, + "loss": 63.4944, + "step": 7130 + }, + { + "epoch": 11.498211316571775, + "grad_norm": 0.7525351047515869, + "learning_rate": 0.0001, + "loss": 62.4088, + "step": 7140 + }, + { + "epoch": 11.514334660150148, + "grad_norm": 0.7193494439125061, + "learning_rate": 0.0001, + "loss": 64.2386, + "step": 7150 + }, + { + "epoch": 11.530458003728523, + "grad_norm": 0.892281174659729, + "learning_rate": 0.0001, + "loss": 63.8075, + "step": 7160 + }, + { + "epoch": 11.546581347306898, + "grad_norm": 0.720287561416626, + "learning_rate": 0.0001, + "loss": 63.3259, + "step": 7170 + }, + { + "epoch": 11.562704690885273, + "grad_norm": 1.4404689073562622, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 7180 + }, + { + "epoch": 11.578828034463648, + "grad_norm": 0.9082726240158081, + "learning_rate": 0.0001, + "loss": 64.6877, + "step": 7190 + }, + { + "epoch": 11.59495137804202, + "grad_norm": 0.7137670516967773, + "learning_rate": 0.0001, + "loss": 63.4832, + "step": 7200 + }, + { + "epoch": 11.611074721620396, + "grad_norm": 0.6848503947257996, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 7210 + }, + { + "epoch": 11.62719806519877, + "grad_norm": 0.7483687996864319, + "learning_rate": 0.0001, + "loss": 63.294, + "step": 7220 + }, + { + "epoch": 11.643321408777146, + "grad_norm": 0.935321569442749, + "learning_rate": 0.0001, + "loss": 62.8669, + "step": 7230 + }, + { + "epoch": 11.659444752355519, + "grad_norm": 0.8359326124191284, + "learning_rate": 0.0001, + "loss": 64.4482, + "step": 7240 + }, + { + "epoch": 11.675568095933894, + "grad_norm": 0.7838515043258667, + "learning_rate": 0.0001, + "loss": 64.1975, + "step": 7250 + }, + { + "epoch": 11.691691439512269, + "grad_norm": 0.9128443598747253, + "learning_rate": 0.0001, + "loss": 63.1822, + "step": 7260 + }, + { + "epoch": 11.707814783090644, + "grad_norm": 1.0545419454574585, + "learning_rate": 0.0001, + "loss": 63.3112, + "step": 7270 + }, + { + "epoch": 11.723938126669019, + "grad_norm": 0.8572486639022827, + "learning_rate": 0.0001, + "loss": 64.9605, + "step": 7280 + }, + { + "epoch": 11.740061470247392, + "grad_norm": 1.0894116163253784, + "learning_rate": 0.0001, + "loss": 64.3671, + "step": 7290 + }, + { + "epoch": 11.756184813825767, + "grad_norm": 0.7500355839729309, + "learning_rate": 0.0001, + "loss": 62.6381, + "step": 7300 + }, + { + "epoch": 11.772308157404142, + "grad_norm": 0.7315185070037842, + "learning_rate": 0.0001, + "loss": 66.1438, + "step": 7310 + }, + { + "epoch": 11.788431500982517, + "grad_norm": 0.8123629093170166, + "learning_rate": 0.0001, + "loss": 62.6012, + "step": 7320 + }, + { + "epoch": 11.80455484456089, + "grad_norm": 0.8072763085365295, + "learning_rate": 0.0001, + "loss": 63.4424, + "step": 7330 + }, + { + "epoch": 11.820678188139265, + "grad_norm": 1.1077874898910522, + "learning_rate": 0.0001, + "loss": 63.2812, + "step": 7340 + }, + { + "epoch": 11.83680153171764, + "grad_norm": 0.7116726636886597, + "learning_rate": 0.0001, + "loss": 63.6077, + "step": 7350 + }, + { + "epoch": 11.852924875296015, + "grad_norm": 0.8817790746688843, + "learning_rate": 0.0001, + "loss": 63.1938, + "step": 7360 + }, + { + "epoch": 11.86904821887439, + "grad_norm": 1.021730661392212, + "learning_rate": 0.0001, + "loss": 65.2215, + "step": 7370 + }, + { + "epoch": 11.885171562452763, + "grad_norm": 0.9871250987052917, + "learning_rate": 0.0001, + "loss": 64.6468, + "step": 7380 + }, + { + "epoch": 11.901294906031138, + "grad_norm": 1.3591430187225342, + "learning_rate": 0.0001, + "loss": 61.5546, + "step": 7390 + }, + { + "epoch": 11.917418249609513, + "grad_norm": 0.8114227056503296, + "learning_rate": 0.0001, + "loss": 61.8215, + "step": 7400 + }, + { + "epoch": 11.933541593187888, + "grad_norm": 0.7151288986206055, + "learning_rate": 0.0001, + "loss": 61.4856, + "step": 7410 + }, + { + "epoch": 11.949664936766261, + "grad_norm": 0.7494287490844727, + "learning_rate": 0.0001, + "loss": 63.6451, + "step": 7420 + }, + { + "epoch": 11.965788280344636, + "grad_norm": 0.7377745509147644, + "learning_rate": 0.0001, + "loss": 63.002, + "step": 7430 + }, + { + "epoch": 11.981911623923011, + "grad_norm": 0.6534659266471863, + "learning_rate": 0.0001, + "loss": 63.6404, + "step": 7440 + }, + { + "epoch": 11.998034967501386, + "grad_norm": 0.7548736333847046, + "learning_rate": 0.0001, + "loss": 64.1986, + "step": 7450 + }, + { + "epoch": 12.012898674862699, + "grad_norm": 0.7227553129196167, + "learning_rate": 0.0001, + "loss": 56.8327, + "step": 7460 + }, + { + "epoch": 12.029022018441074, + "grad_norm": 0.6348695158958435, + "learning_rate": 0.0001, + "loss": 62.4472, + "step": 7470 + }, + { + "epoch": 12.045145362019449, + "grad_norm": 0.8249921202659607, + "learning_rate": 0.0001, + "loss": 65.1877, + "step": 7480 + }, + { + "epoch": 12.061268705597824, + "grad_norm": 0.7152777314186096, + "learning_rate": 0.0001, + "loss": 64.2609, + "step": 7490 + }, + { + "epoch": 12.077392049176197, + "grad_norm": 0.7399184703826904, + "learning_rate": 0.0001, + "loss": 63.3997, + "step": 7500 + }, + { + "epoch": 12.093515392754572, + "grad_norm": 0.6972354650497437, + "learning_rate": 0.0001, + "loss": 61.3573, + "step": 7510 + }, + { + "epoch": 12.109638736332947, + "grad_norm": 0.7666720151901245, + "learning_rate": 0.0001, + "loss": 63.8438, + "step": 7520 + }, + { + "epoch": 12.125762079911322, + "grad_norm": 0.7309302687644958, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 7530 + }, + { + "epoch": 12.141885423489697, + "grad_norm": 1.3805177211761475, + "learning_rate": 0.0001, + "loss": 64.68, + "step": 7540 + }, + { + "epoch": 12.15800876706807, + "grad_norm": 1.0315922498703003, + "learning_rate": 0.0001, + "loss": 63.6372, + "step": 7550 + }, + { + "epoch": 12.174132110646445, + "grad_norm": 1.2012155055999756, + "learning_rate": 0.0001, + "loss": 62.5765, + "step": 7560 + }, + { + "epoch": 12.19025545422482, + "grad_norm": 0.7769846320152283, + "learning_rate": 0.0001, + "loss": 63.9737, + "step": 7570 + }, + { + "epoch": 12.206378797803195, + "grad_norm": 0.6587379574775696, + "learning_rate": 0.0001, + "loss": 63.382, + "step": 7580 + }, + { + "epoch": 12.222502141381568, + "grad_norm": 0.8309435248374939, + "learning_rate": 0.0001, + "loss": 64.4027, + "step": 7590 + }, + { + "epoch": 12.238625484959943, + "grad_norm": 0.8221393823623657, + "learning_rate": 0.0001, + "loss": 62.8561, + "step": 7600 + }, + { + "epoch": 12.254748828538318, + "grad_norm": 0.8629311323165894, + "learning_rate": 0.0001, + "loss": 64.3673, + "step": 7610 + }, + { + "epoch": 12.270872172116693, + "grad_norm": 0.698113739490509, + "learning_rate": 0.0001, + "loss": 63.6153, + "step": 7620 + }, + { + "epoch": 12.286995515695068, + "grad_norm": 0.72906094789505, + "learning_rate": 0.0001, + "loss": 64.8439, + "step": 7630 + }, + { + "epoch": 12.303118859273441, + "grad_norm": 0.7402384877204895, + "learning_rate": 0.0001, + "loss": 64.2103, + "step": 7640 + }, + { + "epoch": 12.319242202851816, + "grad_norm": 0.9972009062767029, + "learning_rate": 0.0001, + "loss": 63.803, + "step": 7650 + }, + { + "epoch": 12.335365546430191, + "grad_norm": 0.7852274775505066, + "learning_rate": 0.0001, + "loss": 64.8144, + "step": 7660 + }, + { + "epoch": 12.351488890008566, + "grad_norm": 0.7433444857597351, + "learning_rate": 0.0001, + "loss": 63.0123, + "step": 7670 + }, + { + "epoch": 12.36761223358694, + "grad_norm": 0.8548759818077087, + "learning_rate": 0.0001, + "loss": 63.2215, + "step": 7680 + }, + { + "epoch": 12.383735577165314, + "grad_norm": 0.6759565472602844, + "learning_rate": 0.0001, + "loss": 62.1246, + "step": 7690 + }, + { + "epoch": 12.39985892074369, + "grad_norm": 0.9079536199569702, + "learning_rate": 0.0001, + "loss": 64.1692, + "step": 7700 + }, + { + "epoch": 12.415982264322064, + "grad_norm": 0.6890071034431458, + "learning_rate": 0.0001, + "loss": 62.5602, + "step": 7710 + }, + { + "epoch": 12.432105607900438, + "grad_norm": 0.8096348643302917, + "learning_rate": 0.0001, + "loss": 61.0103, + "step": 7720 + }, + { + "epoch": 12.448228951478812, + "grad_norm": 0.7359873056411743, + "learning_rate": 0.0001, + "loss": 62.1914, + "step": 7730 + }, + { + "epoch": 12.464352295057187, + "grad_norm": 1.0701091289520264, + "learning_rate": 0.0001, + "loss": 61.7295, + "step": 7740 + }, + { + "epoch": 12.480475638635562, + "grad_norm": 0.791928768157959, + "learning_rate": 0.0001, + "loss": 62.6812, + "step": 7750 + }, + { + "epoch": 12.496598982213937, + "grad_norm": 0.7719446420669556, + "learning_rate": 0.0001, + "loss": 61.9114, + "step": 7760 + }, + { + "epoch": 12.51272232579231, + "grad_norm": 0.8720338344573975, + "learning_rate": 0.0001, + "loss": 64.8107, + "step": 7770 + }, + { + "epoch": 12.528845669370686, + "grad_norm": 0.9999920129776001, + "learning_rate": 0.0001, + "loss": 61.198, + "step": 7780 + }, + { + "epoch": 12.54496901294906, + "grad_norm": 0.710754930973053, + "learning_rate": 0.0001, + "loss": 64.1462, + "step": 7790 + }, + { + "epoch": 12.561092356527435, + "grad_norm": 0.7401363849639893, + "learning_rate": 0.0001, + "loss": 61.8988, + "step": 7800 + }, + { + "epoch": 12.577215700105809, + "grad_norm": 0.9183729290962219, + "learning_rate": 0.0001, + "loss": 62.6392, + "step": 7810 + }, + { + "epoch": 12.593339043684184, + "grad_norm": 0.7341399788856506, + "learning_rate": 0.0001, + "loss": 62.7068, + "step": 7820 + }, + { + "epoch": 12.609462387262559, + "grad_norm": 0.8066722750663757, + "learning_rate": 0.0001, + "loss": 64.0625, + "step": 7830 + }, + { + "epoch": 12.625585730840934, + "grad_norm": 0.6088835597038269, + "learning_rate": 0.0001, + "loss": 64.3208, + "step": 7840 + }, + { + "epoch": 12.641709074419307, + "grad_norm": 0.6969302892684937, + "learning_rate": 0.0001, + "loss": 62.5817, + "step": 7850 + }, + { + "epoch": 12.657832417997682, + "grad_norm": 0.82635098695755, + "learning_rate": 0.0001, + "loss": 62.0061, + "step": 7860 + }, + { + "epoch": 12.673955761576057, + "grad_norm": 0.7016922235488892, + "learning_rate": 0.0001, + "loss": 62.5166, + "step": 7870 + }, + { + "epoch": 12.690079105154432, + "grad_norm": 0.8035777807235718, + "learning_rate": 0.0001, + "loss": 63.6543, + "step": 7880 + }, + { + "epoch": 12.706202448732807, + "grad_norm": 0.8573809862136841, + "learning_rate": 0.0001, + "loss": 62.5258, + "step": 7890 + }, + { + "epoch": 12.72232579231118, + "grad_norm": 0.9497472643852234, + "learning_rate": 0.0001, + "loss": 63.1133, + "step": 7900 + }, + { + "epoch": 12.738449135889555, + "grad_norm": 0.7494457960128784, + "learning_rate": 0.0001, + "loss": 63.8157, + "step": 7910 + }, + { + "epoch": 12.75457247946793, + "grad_norm": 0.7657570242881775, + "learning_rate": 0.0001, + "loss": 62.6255, + "step": 7920 + }, + { + "epoch": 12.770695823046305, + "grad_norm": 0.7848380208015442, + "learning_rate": 0.0001, + "loss": 62.9095, + "step": 7930 + }, + { + "epoch": 12.786819166624678, + "grad_norm": 0.8386877775192261, + "learning_rate": 0.0001, + "loss": 63.2254, + "step": 7940 + }, + { + "epoch": 12.802942510203053, + "grad_norm": 0.6774776577949524, + "learning_rate": 0.0001, + "loss": 64.3929, + "step": 7950 + }, + { + "epoch": 12.819065853781428, + "grad_norm": 0.798547625541687, + "learning_rate": 0.0001, + "loss": 62.9884, + "step": 7960 + }, + { + "epoch": 12.835189197359803, + "grad_norm": 0.9340888261795044, + "learning_rate": 0.0001, + "loss": 62.6197, + "step": 7970 + }, + { + "epoch": 12.851312540938178, + "grad_norm": 0.8211432695388794, + "learning_rate": 0.0001, + "loss": 62.7756, + "step": 7980 + }, + { + "epoch": 12.867435884516551, + "grad_norm": 0.7612001299858093, + "learning_rate": 0.0001, + "loss": 62.0792, + "step": 7990 + }, + { + "epoch": 12.883559228094926, + "grad_norm": 0.7743386626243591, + "learning_rate": 0.0001, + "loss": 63.9203, + "step": 8000 + }, + { + "epoch": 12.899682571673301, + "grad_norm": 0.7419672608375549, + "learning_rate": 0.0001, + "loss": 62.1705, + "step": 8010 + }, + { + "epoch": 12.915805915251676, + "grad_norm": 0.6556256413459778, + "learning_rate": 0.0001, + "loss": 64.5863, + "step": 8020 + }, + { + "epoch": 12.93192925883005, + "grad_norm": 1.136175274848938, + "learning_rate": 0.0001, + "loss": 63.2619, + "step": 8030 + }, + { + "epoch": 12.948052602408424, + "grad_norm": 0.8631499409675598, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8040 + }, + { + "epoch": 12.9641759459868, + "grad_norm": 0.8386172652244568, + "learning_rate": 0.0001, + "loss": 63.0598, + "step": 8050 + }, + { + "epoch": 12.980299289565174, + "grad_norm": 0.8078306913375854, + "learning_rate": 0.0001, + "loss": 62.6877, + "step": 8060 + }, + { + "epoch": 12.996422633143549, + "grad_norm": 0.8486189842224121, + "learning_rate": 0.0001, + "loss": 62.8643, + "step": 8070 + }, + { + "epoch": 13.011286340504862, + "grad_norm": 0.8717893958091736, + "learning_rate": 0.0001, + "loss": 57.6947, + "step": 8080 + }, + { + "epoch": 13.027409684083237, + "grad_norm": 1.2023789882659912, + "learning_rate": 0.0001, + "loss": 60.4728, + "step": 8090 + }, + { + "epoch": 13.043533027661612, + "grad_norm": 0.6879557967185974, + "learning_rate": 0.0001, + "loss": 61.7643, + "step": 8100 + }, + { + "epoch": 13.059656371239987, + "grad_norm": 1.0918916463851929, + "learning_rate": 0.0001, + "loss": 65.3682, + "step": 8110 + }, + { + "epoch": 13.07577971481836, + "grad_norm": 0.85753333568573, + "learning_rate": 0.0001, + "loss": 63.0815, + "step": 8120 + }, + { + "epoch": 13.091903058396735, + "grad_norm": 0.6568583846092224, + "learning_rate": 0.0001, + "loss": 64.7167, + "step": 8130 + }, + { + "epoch": 13.10802640197511, + "grad_norm": 0.6178144812583923, + "learning_rate": 0.0001, + "loss": 62.4294, + "step": 8140 + }, + { + "epoch": 13.124149745553485, + "grad_norm": 0.7930784821510315, + "learning_rate": 0.0001, + "loss": 64.542, + "step": 8150 + }, + { + "epoch": 13.140273089131858, + "grad_norm": 0.7675037980079651, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8160 + }, + { + "epoch": 13.156396432710233, + "grad_norm": 0.786997377872467, + "learning_rate": 0.0001, + "loss": 64.1495, + "step": 8170 + }, + { + "epoch": 13.172519776288608, + "grad_norm": 0.7752560377120972, + "learning_rate": 0.0001, + "loss": 61.6483, + "step": 8180 + }, + { + "epoch": 13.188643119866983, + "grad_norm": 0.8455203771591187, + "learning_rate": 0.0001, + "loss": 63.3551, + "step": 8190 + }, + { + "epoch": 13.204766463445356, + "grad_norm": 0.7223097681999207, + "learning_rate": 0.0001, + "loss": 63.1473, + "step": 8200 + }, + { + "epoch": 13.220889807023731, + "grad_norm": 0.7688442468643188, + "learning_rate": 0.0001, + "loss": 62.6723, + "step": 8210 + }, + { + "epoch": 13.237013150602106, + "grad_norm": 0.7730395793914795, + "learning_rate": 0.0001, + "loss": 64.3424, + "step": 8220 + }, + { + "epoch": 13.253136494180481, + "grad_norm": 0.8479727506637573, + "learning_rate": 0.0001, + "loss": 61.4336, + "step": 8230 + }, + { + "epoch": 13.269259837758856, + "grad_norm": 0.9070560932159424, + "learning_rate": 0.0001, + "loss": 63.1221, + "step": 8240 + }, + { + "epoch": 13.28538318133723, + "grad_norm": 0.8924600481987, + "learning_rate": 0.0001, + "loss": 63.9172, + "step": 8250 + }, + { + "epoch": 13.301506524915604, + "grad_norm": 1.0688443183898926, + "learning_rate": 0.0001, + "loss": 63.7944, + "step": 8260 + }, + { + "epoch": 13.31762986849398, + "grad_norm": 0.6866253018379211, + "learning_rate": 0.0001, + "loss": 65.5847, + "step": 8270 + }, + { + "epoch": 13.333753212072354, + "grad_norm": 0.711171567440033, + "learning_rate": 0.0001, + "loss": 62.698, + "step": 8280 + }, + { + "epoch": 13.349876555650727, + "grad_norm": 0.8128448724746704, + "learning_rate": 0.0001, + "loss": 61.5992, + "step": 8290 + }, + { + "epoch": 13.365999899229102, + "grad_norm": 0.7483755350112915, + "learning_rate": 0.0001, + "loss": 63.782, + "step": 8300 + }, + { + "epoch": 13.382123242807477, + "grad_norm": 0.7872599363327026, + "learning_rate": 0.0001, + "loss": 65.4922, + "step": 8310 + }, + { + "epoch": 13.398246586385852, + "grad_norm": 0.6861543655395508, + "learning_rate": 0.0001, + "loss": 62.6946, + "step": 8320 + }, + { + "epoch": 13.414369929964227, + "grad_norm": 0.7192599773406982, + "learning_rate": 0.0001, + "loss": 62.894, + "step": 8330 + }, + { + "epoch": 13.4304932735426, + "grad_norm": 0.7665286064147949, + "learning_rate": 0.0001, + "loss": 62.6253, + "step": 8340 + }, + { + "epoch": 13.446616617120975, + "grad_norm": 0.7909309267997742, + "learning_rate": 0.0001, + "loss": 62.5355, + "step": 8350 + }, + { + "epoch": 13.46273996069935, + "grad_norm": 0.7339541912078857, + "learning_rate": 0.0001, + "loss": 62.8514, + "step": 8360 + }, + { + "epoch": 13.478863304277725, + "grad_norm": 0.9893004298210144, + "learning_rate": 0.0001, + "loss": 63.7751, + "step": 8370 + }, + { + "epoch": 13.494986647856098, + "grad_norm": 0.9109547734260559, + "learning_rate": 0.0001, + "loss": 62.6697, + "step": 8380 + }, + { + "epoch": 13.511109991434473, + "grad_norm": 0.6807034015655518, + "learning_rate": 0.0001, + "loss": 62.1418, + "step": 8390 + }, + { + "epoch": 13.527233335012848, + "grad_norm": 0.915533185005188, + "learning_rate": 0.0001, + "loss": 61.7645, + "step": 8400 + }, + { + "epoch": 13.543356678591223, + "grad_norm": 0.8662852048873901, + "learning_rate": 0.0001, + "loss": 62.5546, + "step": 8410 + }, + { + "epoch": 13.559480022169598, + "grad_norm": 0.8678365349769592, + "learning_rate": 0.0001, + "loss": 63.8795, + "step": 8420 + }, + { + "epoch": 13.575603365747972, + "grad_norm": 0.6808590888977051, + "learning_rate": 0.0001, + "loss": 64.158, + "step": 8430 + }, + { + "epoch": 13.591726709326347, + "grad_norm": 0.7896480560302734, + "learning_rate": 0.0001, + "loss": 62.1052, + "step": 8440 + }, + { + "epoch": 13.607850052904721, + "grad_norm": 0.8973979949951172, + "learning_rate": 0.0001, + "loss": 64.478, + "step": 8450 + }, + { + "epoch": 13.623973396483096, + "grad_norm": 0.8228474259376526, + "learning_rate": 0.0001, + "loss": 61.2416, + "step": 8460 + }, + { + "epoch": 13.64009674006147, + "grad_norm": 0.9414492845535278, + "learning_rate": 0.0001, + "loss": 62.816, + "step": 8470 + }, + { + "epoch": 13.656220083639845, + "grad_norm": 0.9369436502456665, + "learning_rate": 0.0001, + "loss": 63.4951, + "step": 8480 + }, + { + "epoch": 13.67234342721822, + "grad_norm": 0.7152880430221558, + "learning_rate": 0.0001, + "loss": 62.4737, + "step": 8490 + }, + { + "epoch": 13.688466770796595, + "grad_norm": 0.7364594340324402, + "learning_rate": 0.0001, + "loss": 63.8688, + "step": 8500 + }, + { + "epoch": 13.704590114374968, + "grad_norm": 0.8499143123626709, + "learning_rate": 0.0001, + "loss": 64.5654, + "step": 8510 + }, + { + "epoch": 13.720713457953343, + "grad_norm": 0.7699156999588013, + "learning_rate": 0.0001, + "loss": 63.3726, + "step": 8520 + }, + { + "epoch": 13.736836801531718, + "grad_norm": 0.9146804809570312, + "learning_rate": 0.0001, + "loss": 63.6565, + "step": 8530 + }, + { + "epoch": 13.752960145110093, + "grad_norm": 0.773318350315094, + "learning_rate": 0.0001, + "loss": 65.0719, + "step": 8540 + }, + { + "epoch": 13.769083488688466, + "grad_norm": 0.6158450245857239, + "learning_rate": 0.0001, + "loss": 60.7291, + "step": 8550 + }, + { + "epoch": 13.78520683226684, + "grad_norm": 0.8053523302078247, + "learning_rate": 0.0001, + "loss": 62.5622, + "step": 8560 + }, + { + "epoch": 13.801330175845216, + "grad_norm": 0.8983457684516907, + "learning_rate": 0.0001, + "loss": 64.2921, + "step": 8570 + }, + { + "epoch": 13.81745351942359, + "grad_norm": 0.8214695453643799, + "learning_rate": 0.0001, + "loss": 62.2649, + "step": 8580 + }, + { + "epoch": 13.833576863001966, + "grad_norm": 0.8079172372817993, + "learning_rate": 0.0001, + "loss": 64.0828, + "step": 8590 + }, + { + "epoch": 13.849700206580339, + "grad_norm": 1.126568078994751, + "learning_rate": 0.0001, + "loss": 62.3927, + "step": 8600 + }, + { + "epoch": 13.865823550158714, + "grad_norm": 0.7597793936729431, + "learning_rate": 0.0001, + "loss": 62.5922, + "step": 8610 + }, + { + "epoch": 13.881946893737089, + "grad_norm": 0.671531617641449, + "learning_rate": 0.0001, + "loss": 61.9885, + "step": 8620 + }, + { + "epoch": 13.898070237315464, + "grad_norm": 0.823423445224762, + "learning_rate": 0.0001, + "loss": 62.0632, + "step": 8630 + }, + { + "epoch": 13.914193580893837, + "grad_norm": 0.7694658637046814, + "learning_rate": 0.0001, + "loss": 64.5485, + "step": 8640 + }, + { + "epoch": 13.930316924472212, + "grad_norm": 0.7394612431526184, + "learning_rate": 0.0001, + "loss": 61.9626, + "step": 8650 + }, + { + "epoch": 13.946440268050587, + "grad_norm": 0.724054217338562, + "learning_rate": 0.0001, + "loss": 62.3761, + "step": 8660 + }, + { + "epoch": 13.962563611628962, + "grad_norm": 0.7160128951072693, + "learning_rate": 0.0001, + "loss": 60.5142, + "step": 8670 + }, + { + "epoch": 13.978686955207337, + "grad_norm": 1.0045746564865112, + "learning_rate": 0.0001, + "loss": 62.7059, + "step": 8680 + }, + { + "epoch": 13.99481029878571, + "grad_norm": 0.6980776190757751, + "learning_rate": 0.0001, + "loss": 62.9427, + "step": 8690 + }, + { + "epoch": 14.009674006147025, + "grad_norm": 0.810961127281189, + "learning_rate": 0.0001, + "loss": 58.7552, + "step": 8700 + }, + { + "epoch": 14.0257973497254, + "grad_norm": 0.9210551381111145, + "learning_rate": 0.0001, + "loss": 63.7412, + "step": 8710 + }, + { + "epoch": 14.041920693303775, + "grad_norm": 0.7805785536766052, + "learning_rate": 0.0001, + "loss": 61.477, + "step": 8720 + }, + { + "epoch": 14.058044036882148, + "grad_norm": 0.8528208136558533, + "learning_rate": 0.0001, + "loss": 62.6746, + "step": 8730 + }, + { + "epoch": 14.074167380460523, + "grad_norm": 0.7467619776725769, + "learning_rate": 0.0001, + "loss": 63.8744, + "step": 8740 + }, + { + "epoch": 14.090290724038898, + "grad_norm": 0.6434402465820312, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 8750 + }, + { + "epoch": 14.106414067617273, + "grad_norm": 0.7217757701873779, + "learning_rate": 0.0001, + "loss": 64.0428, + "step": 8760 + }, + { + "epoch": 14.122537411195646, + "grad_norm": 0.8665125966072083, + "learning_rate": 0.0001, + "loss": 62.2772, + "step": 8770 + }, + { + "epoch": 14.13866075477402, + "grad_norm": 0.7868321537971497, + "learning_rate": 0.0001, + "loss": 62.0509, + "step": 8780 + }, + { + "epoch": 14.154784098352396, + "grad_norm": 0.8716760873794556, + "learning_rate": 0.0001, + "loss": 61.6021, + "step": 8790 + }, + { + "epoch": 14.17090744193077, + "grad_norm": 1.0922521352767944, + "learning_rate": 0.0001, + "loss": 64.4197, + "step": 8800 + }, + { + "epoch": 14.187030785509146, + "grad_norm": 0.6545577049255371, + "learning_rate": 0.0001, + "loss": 62.9259, + "step": 8810 + }, + { + "epoch": 14.203154129087519, + "grad_norm": 0.7634063959121704, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 8820 + }, + { + "epoch": 14.219277472665894, + "grad_norm": 0.9075300097465515, + "learning_rate": 0.0001, + "loss": 62.9128, + "step": 8830 + }, + { + "epoch": 14.235400816244269, + "grad_norm": 3.475320816040039, + "learning_rate": 0.0001, + "loss": 62.4841, + "step": 8840 + }, + { + "epoch": 14.251524159822644, + "grad_norm": 0.6839455962181091, + "learning_rate": 0.0001, + "loss": 61.5898, + "step": 8850 + }, + { + "epoch": 14.267647503401017, + "grad_norm": 0.8024231791496277, + "learning_rate": 0.0001, + "loss": 61.1875, + "step": 8860 + }, + { + "epoch": 14.283770846979392, + "grad_norm": 0.8948174118995667, + "learning_rate": 0.0001, + "loss": 62.3386, + "step": 8870 + }, + { + "epoch": 14.299894190557767, + "grad_norm": 0.7571985721588135, + "learning_rate": 0.0001, + "loss": 60.579, + "step": 8880 + }, + { + "epoch": 14.316017534136142, + "grad_norm": 1.432371973991394, + "learning_rate": 0.0001, + "loss": 62.4405, + "step": 8890 + }, + { + "epoch": 14.332140877714515, + "grad_norm": 0.6696875691413879, + "learning_rate": 0.0001, + "loss": 62.9984, + "step": 8900 + }, + { + "epoch": 14.34826422129289, + "grad_norm": 0.749396026134491, + "learning_rate": 0.0001, + "loss": 61.492, + "step": 8910 + }, + { + "epoch": 14.364387564871265, + "grad_norm": 0.664193332195282, + "learning_rate": 0.0001, + "loss": 62.9893, + "step": 8920 + }, + { + "epoch": 14.38051090844964, + "grad_norm": 0.7032697796821594, + "learning_rate": 0.0001, + "loss": 61.1682, + "step": 8930 + }, + { + "epoch": 14.396634252028015, + "grad_norm": 0.813119649887085, + "learning_rate": 0.0001, + "loss": 62.906, + "step": 8940 + }, + { + "epoch": 14.412757595606388, + "grad_norm": 0.7699596881866455, + "learning_rate": 0.0001, + "loss": 64.8087, + "step": 8950 + }, + { + "epoch": 14.428880939184763, + "grad_norm": 0.6424354910850525, + "learning_rate": 0.0001, + "loss": 62.2418, + "step": 8960 + }, + { + "epoch": 14.445004282763138, + "grad_norm": 0.8741731643676758, + "learning_rate": 0.0001, + "loss": 62.656, + "step": 8970 + }, + { + "epoch": 14.461127626341513, + "grad_norm": 0.710617184638977, + "learning_rate": 0.0001, + "loss": 60.8979, + "step": 8980 + }, + { + "epoch": 14.477250969919886, + "grad_norm": 0.6493995785713196, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 8990 + }, + { + "epoch": 14.493374313498261, + "grad_norm": 0.6655835509300232, + "learning_rate": 0.0001, + "loss": 64.151, + "step": 9000 + }, + { + "epoch": 14.509497657076636, + "grad_norm": 0.8140881657600403, + "learning_rate": 0.0001, + "loss": 62.4447, + "step": 9010 + }, + { + "epoch": 14.525621000655011, + "grad_norm": 0.7288498878479004, + "learning_rate": 0.0001, + "loss": 62.5189, + "step": 9020 + }, + { + "epoch": 14.541744344233386, + "grad_norm": 0.6972017884254456, + "learning_rate": 0.0001, + "loss": 61.5441, + "step": 9030 + }, + { + "epoch": 14.55786768781176, + "grad_norm": 0.7444366216659546, + "learning_rate": 0.0001, + "loss": 62.3233, + "step": 9040 + }, + { + "epoch": 14.573991031390134, + "grad_norm": 0.6602845191955566, + "learning_rate": 0.0001, + "loss": 61.4176, + "step": 9050 + }, + { + "epoch": 14.59011437496851, + "grad_norm": 0.7149770855903625, + "learning_rate": 0.0001, + "loss": 62.159, + "step": 9060 + }, + { + "epoch": 14.606237718546884, + "grad_norm": 0.6975772976875305, + "learning_rate": 0.0001, + "loss": 63.1857, + "step": 9070 + }, + { + "epoch": 14.622361062125258, + "grad_norm": 0.7835262417793274, + "learning_rate": 0.0001, + "loss": 61.6795, + "step": 9080 + }, + { + "epoch": 14.638484405703633, + "grad_norm": 5.790656566619873, + "learning_rate": 0.0001, + "loss": 62.7087, + "step": 9090 + }, + { + "epoch": 14.654607749282007, + "grad_norm": 1.0082288980484009, + "learning_rate": 0.0001, + "loss": 61.6808, + "step": 9100 + }, + { + "epoch": 14.670731092860382, + "grad_norm": 0.7758098840713501, + "learning_rate": 0.0001, + "loss": 61.0734, + "step": 9110 + }, + { + "epoch": 14.686854436438756, + "grad_norm": 0.7841722965240479, + "learning_rate": 0.0001, + "loss": 62.8882, + "step": 9120 + }, + { + "epoch": 14.70297778001713, + "grad_norm": 0.7695890069007874, + "learning_rate": 0.0001, + "loss": 60.7811, + "step": 9130 + }, + { + "epoch": 14.719101123595506, + "grad_norm": 0.7444761395454407, + "learning_rate": 0.0001, + "loss": 61.692, + "step": 9140 + }, + { + "epoch": 14.73522446717388, + "grad_norm": 0.7036821246147156, + "learning_rate": 0.0001, + "loss": 63.1698, + "step": 9150 + }, + { + "epoch": 14.751347810752256, + "grad_norm": 0.7891719937324524, + "learning_rate": 0.0001, + "loss": 63.8575, + "step": 9160 + }, + { + "epoch": 14.767471154330629, + "grad_norm": 0.6371109485626221, + "learning_rate": 0.0001, + "loss": 61.3358, + "step": 9170 + }, + { + "epoch": 14.783594497909004, + "grad_norm": 0.6551769375801086, + "learning_rate": 0.0001, + "loss": 62.2705, + "step": 9180 + }, + { + "epoch": 14.799717841487379, + "grad_norm": 0.6922310590744019, + "learning_rate": 0.0001, + "loss": 61.6442, + "step": 9190 + }, + { + "epoch": 14.815841185065754, + "grad_norm": 0.9615464210510254, + "learning_rate": 0.0001, + "loss": 60.932, + "step": 9200 + }, + { + "epoch": 14.831964528644127, + "grad_norm": 0.6367932558059692, + "learning_rate": 0.0001, + "loss": 61.3475, + "step": 9210 + }, + { + "epoch": 14.848087872222502, + "grad_norm": 0.5634505748748779, + "learning_rate": 0.0001, + "loss": 61.7562, + "step": 9220 + }, + { + "epoch": 14.864211215800877, + "grad_norm": 0.7021980285644531, + "learning_rate": 0.0001, + "loss": 62.0672, + "step": 9230 + }, + { + "epoch": 14.880334559379252, + "grad_norm": 0.8832064867019653, + "learning_rate": 0.0001, + "loss": 60.9882, + "step": 9240 + }, + { + "epoch": 14.896457902957625, + "grad_norm": 0.9043561220169067, + "learning_rate": 0.0001, + "loss": 63.0968, + "step": 9250 + }, + { + "epoch": 14.912581246536, + "grad_norm": 0.8958032131195068, + "learning_rate": 0.0001, + "loss": 61.5431, + "step": 9260 + }, + { + "epoch": 14.928704590114375, + "grad_norm": 0.7181615829467773, + "learning_rate": 0.0001, + "loss": 62.3626, + "step": 9270 + }, + { + "epoch": 14.94482793369275, + "grad_norm": 0.7541075348854065, + "learning_rate": 0.0001, + "loss": 61.7186, + "step": 9280 + }, + { + "epoch": 14.960951277271125, + "grad_norm": 1.5627169609069824, + "learning_rate": 0.0001, + "loss": 61.654, + "step": 9290 + }, + { + "epoch": 14.977074620849498, + "grad_norm": 0.7126879096031189, + "learning_rate": 0.0001, + "loss": 61.6806, + "step": 9300 + }, + { + "epoch": 14.993197964427873, + "grad_norm": 0.7397864460945129, + "learning_rate": 0.0001, + "loss": 61.2669, + "step": 9310 + }, + { + "epoch": 15.008061671789187, + "grad_norm": 0.7282766699790955, + "learning_rate": 0.0001, + "loss": 56.2289, + "step": 9320 + }, + { + "epoch": 15.024185015367562, + "grad_norm": 0.8346580862998962, + "learning_rate": 0.0001, + "loss": 61.4335, + "step": 9330 + }, + { + "epoch": 15.040308358945936, + "grad_norm": 0.9117271304130554, + "learning_rate": 0.0001, + "loss": 60.4636, + "step": 9340 + }, + { + "epoch": 15.05643170252431, + "grad_norm": 0.8483272194862366, + "learning_rate": 0.0001, + "loss": 61.3003, + "step": 9350 + }, + { + "epoch": 15.072555046102686, + "grad_norm": 1.1898932456970215, + "learning_rate": 0.0001, + "loss": 62.7169, + "step": 9360 + }, + { + "epoch": 15.08867838968106, + "grad_norm": 0.7258167862892151, + "learning_rate": 0.0001, + "loss": 63.5285, + "step": 9370 + }, + { + "epoch": 15.104801733259436, + "grad_norm": 0.7810865044593811, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 9380 + }, + { + "epoch": 15.120925076837809, + "grad_norm": 1.0359688997268677, + "learning_rate": 0.0001, + "loss": 62.5435, + "step": 9390 + }, + { + "epoch": 15.137048420416184, + "grad_norm": 0.7371546030044556, + "learning_rate": 0.0001, + "loss": 60.5583, + "step": 9400 + }, + { + "epoch": 15.153171763994559, + "grad_norm": 0.7578926086425781, + "learning_rate": 0.0001, + "loss": 61.2853, + "step": 9410 + }, + { + "epoch": 15.169295107572934, + "grad_norm": 0.6161713004112244, + "learning_rate": 0.0001, + "loss": 62.9725, + "step": 9420 + }, + { + "epoch": 15.185418451151307, + "grad_norm": 0.8325598835945129, + "learning_rate": 0.0001, + "loss": 62.857, + "step": 9430 + }, + { + "epoch": 15.201541794729682, + "grad_norm": 1.1438454389572144, + "learning_rate": 0.0001, + "loss": 61.8167, + "step": 9440 + }, + { + "epoch": 15.217665138308057, + "grad_norm": 1.1820837259292603, + "learning_rate": 0.0001, + "loss": 60.9899, + "step": 9450 + }, + { + "epoch": 15.233788481886432, + "grad_norm": 0.6057641506195068, + "learning_rate": 0.0001, + "loss": 62.5144, + "step": 9460 + }, + { + "epoch": 15.249911825464805, + "grad_norm": 0.6661948561668396, + "learning_rate": 0.0001, + "loss": 62.0835, + "step": 9470 + }, + { + "epoch": 15.26603516904318, + "grad_norm": 0.7623168230056763, + "learning_rate": 0.0001, + "loss": 61.9491, + "step": 9480 + }, + { + "epoch": 15.282158512621555, + "grad_norm": 0.6784029006958008, + "learning_rate": 0.0001, + "loss": 63.6006, + "step": 9490 + }, + { + "epoch": 15.29828185619993, + "grad_norm": 0.9433781504631042, + "learning_rate": 0.0001, + "loss": 61.6677, + "step": 9500 + }, + { + "epoch": 15.314405199778305, + "grad_norm": 0.6544079184532166, + "learning_rate": 0.0001, + "loss": 63.1439, + "step": 9510 + }, + { + "epoch": 15.330528543356678, + "grad_norm": 0.7979472279548645, + "learning_rate": 0.0001, + "loss": 62.719, + "step": 9520 + }, + { + "epoch": 15.346651886935053, + "grad_norm": 0.6641252040863037, + "learning_rate": 0.0001, + "loss": 63.2543, + "step": 9530 + }, + { + "epoch": 15.362775230513428, + "grad_norm": 0.7780525088310242, + "learning_rate": 0.0001, + "loss": 62.5659, + "step": 9540 + }, + { + "epoch": 15.378898574091803, + "grad_norm": 0.842349648475647, + "learning_rate": 0.0001, + "loss": 60.041, + "step": 9550 + }, + { + "epoch": 15.395021917670176, + "grad_norm": 0.8991017937660217, + "learning_rate": 0.0001, + "loss": 62.9075, + "step": 9560 + }, + { + "epoch": 15.411145261248551, + "grad_norm": 0.6734052300453186, + "learning_rate": 0.0001, + "loss": 60.8647, + "step": 9570 + }, + { + "epoch": 15.427268604826926, + "grad_norm": 0.7585113644599915, + "learning_rate": 0.0001, + "loss": 62.105, + "step": 9580 + }, + { + "epoch": 15.443391948405301, + "grad_norm": 0.7915840148925781, + "learning_rate": 0.0001, + "loss": 62.2741, + "step": 9590 + }, + { + "epoch": 15.459515291983674, + "grad_norm": 0.8676218390464783, + "learning_rate": 0.0001, + "loss": 62.4548, + "step": 9600 + }, + { + "epoch": 15.47563863556205, + "grad_norm": 0.7698650360107422, + "learning_rate": 0.0001, + "loss": 61.3412, + "step": 9610 + }, + { + "epoch": 15.491761979140424, + "grad_norm": 0.6600908637046814, + "learning_rate": 0.0001, + "loss": 61.1561, + "step": 9620 + }, + { + "epoch": 15.5078853227188, + "grad_norm": 0.5995001792907715, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 9630 + }, + { + "epoch": 15.524008666297174, + "grad_norm": 0.7258557677268982, + "learning_rate": 0.0001, + "loss": 62.4659, + "step": 9640 + }, + { + "epoch": 15.540132009875547, + "grad_norm": 0.6562618613243103, + "learning_rate": 0.0001, + "loss": 63.288, + "step": 9650 + }, + { + "epoch": 15.556255353453922, + "grad_norm": 0.9224995374679565, + "learning_rate": 0.0001, + "loss": 61.4861, + "step": 9660 + }, + { + "epoch": 15.572378697032297, + "grad_norm": 0.7053484916687012, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 9670 + }, + { + "epoch": 15.588502040610672, + "grad_norm": 1.095229148864746, + "learning_rate": 0.0001, + "loss": 62.1542, + "step": 9680 + }, + { + "epoch": 15.604625384189045, + "grad_norm": 0.7563759088516235, + "learning_rate": 0.0001, + "loss": 61.1644, + "step": 9690 + }, + { + "epoch": 15.62074872776742, + "grad_norm": 0.6508835554122925, + "learning_rate": 0.0001, + "loss": 60.1374, + "step": 9700 + }, + { + "epoch": 15.636872071345795, + "grad_norm": 0.664799153804779, + "learning_rate": 0.0001, + "loss": 60.2331, + "step": 9710 + }, + { + "epoch": 15.65299541492417, + "grad_norm": 0.6525771617889404, + "learning_rate": 0.0001, + "loss": 60.6142, + "step": 9720 + }, + { + "epoch": 15.669118758502545, + "grad_norm": 0.7395381331443787, + "learning_rate": 0.0001, + "loss": 63.5623, + "step": 9730 + }, + { + "epoch": 15.685242102080919, + "grad_norm": 0.7063724398612976, + "learning_rate": 0.0001, + "loss": 63.7133, + "step": 9740 + }, + { + "epoch": 15.701365445659293, + "grad_norm": 0.7918921113014221, + "learning_rate": 0.0001, + "loss": 62.5957, + "step": 9750 + }, + { + "epoch": 15.717488789237668, + "grad_norm": 0.8135486245155334, + "learning_rate": 0.0001, + "loss": 62.9146, + "step": 9760 + }, + { + "epoch": 15.733612132816043, + "grad_norm": 0.8422091007232666, + "learning_rate": 0.0001, + "loss": 61.1466, + "step": 9770 + }, + { + "epoch": 15.749735476394417, + "grad_norm": 0.6329447627067566, + "learning_rate": 0.0001, + "loss": 61.7247, + "step": 9780 + }, + { + "epoch": 15.765858819972792, + "grad_norm": 1.0627039670944214, + "learning_rate": 0.0001, + "loss": 60.6789, + "step": 9790 + }, + { + "epoch": 15.781982163551167, + "grad_norm": 0.7316949367523193, + "learning_rate": 0.0001, + "loss": 61.9896, + "step": 9800 + }, + { + "epoch": 15.798105507129542, + "grad_norm": 0.6968954205513, + "learning_rate": 0.0001, + "loss": 60.8571, + "step": 9810 + }, + { + "epoch": 15.814228850707915, + "grad_norm": 0.7226986289024353, + "learning_rate": 0.0001, + "loss": 62.2664, + "step": 9820 + }, + { + "epoch": 15.83035219428629, + "grad_norm": 0.6542951464653015, + "learning_rate": 0.0001, + "loss": 61.3316, + "step": 9830 + }, + { + "epoch": 15.846475537864665, + "grad_norm": 0.7662289142608643, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 9840 + }, + { + "epoch": 15.86259888144304, + "grad_norm": 0.7060043215751648, + "learning_rate": 0.0001, + "loss": 62.2647, + "step": 9850 + }, + { + "epoch": 15.878722225021415, + "grad_norm": 0.6948843002319336, + "learning_rate": 0.0001, + "loss": 62.8678, + "step": 9860 + }, + { + "epoch": 15.894845568599788, + "grad_norm": 0.7787327766418457, + "learning_rate": 0.0001, + "loss": 62.9154, + "step": 9870 + }, + { + "epoch": 15.910968912178163, + "grad_norm": 0.7077026963233948, + "learning_rate": 0.0001, + "loss": 62.9537, + "step": 9880 + }, + { + "epoch": 15.927092255756538, + "grad_norm": 0.9957109093666077, + "learning_rate": 0.0001, + "loss": 60.9461, + "step": 9890 + }, + { + "epoch": 15.943215599334913, + "grad_norm": 0.9767161011695862, + "learning_rate": 0.0001, + "loss": 62.5944, + "step": 9900 + }, + { + "epoch": 15.959338942913286, + "grad_norm": 0.6016744375228882, + "learning_rate": 0.0001, + "loss": 62.5889, + "step": 9910 + }, + { + "epoch": 15.975462286491661, + "grad_norm": 0.5464991331100464, + "learning_rate": 0.0001, + "loss": 61.3138, + "step": 9920 + }, + { + "epoch": 15.991585630070036, + "grad_norm": 0.8242359757423401, + "learning_rate": 0.0001, + "loss": 61.8091, + "step": 9930 + }, + { + "epoch": 16.00644933743135, + "grad_norm": 0.7031534314155579, + "learning_rate": 0.0001, + "loss": 57.4847, + "step": 9940 + }, + { + "epoch": 16.022572681009724, + "grad_norm": 0.8429487943649292, + "learning_rate": 0.0001, + "loss": 61.7286, + "step": 9950 + }, + { + "epoch": 16.0386960245881, + "grad_norm": 0.7200850248336792, + "learning_rate": 0.0001, + "loss": 62.6968, + "step": 9960 + }, + { + "epoch": 16.054819368166473, + "grad_norm": 0.7412559390068054, + "learning_rate": 0.0001, + "loss": 62.413, + "step": 9970 + }, + { + "epoch": 16.07094271174485, + "grad_norm": 0.8179170489311218, + "learning_rate": 0.0001, + "loss": 62.7044, + "step": 9980 + }, + { + "epoch": 16.087066055323223, + "grad_norm": 0.6939452886581421, + "learning_rate": 0.0001, + "loss": 61.9062, + "step": 9990 + }, + { + "epoch": 16.1031893989016, + "grad_norm": 0.7145045399665833, + "learning_rate": 0.0001, + "loss": 61.6201, + "step": 10000 + }, + { + "epoch": 16.119312742479973, + "grad_norm": 0.7874398231506348, + "learning_rate": 0.0001, + "loss": 60.4299, + "step": 10010 + }, + { + "epoch": 16.135436086058345, + "grad_norm": 0.6640581488609314, + "learning_rate": 0.0001, + "loss": 62.0195, + "step": 10020 + }, + { + "epoch": 16.15155942963672, + "grad_norm": 0.7903703451156616, + "learning_rate": 0.0001, + "loss": 62.501, + "step": 10030 + }, + { + "epoch": 16.167682773215095, + "grad_norm": 0.6669818162918091, + "learning_rate": 0.0001, + "loss": 59.1489, + "step": 10040 + }, + { + "epoch": 16.18380611679347, + "grad_norm": 0.8380771279335022, + "learning_rate": 0.0001, + "loss": 61.0985, + "step": 10050 + }, + { + "epoch": 16.199929460371845, + "grad_norm": 0.7925304174423218, + "learning_rate": 0.0001, + "loss": 62.9251, + "step": 10060 + }, + { + "epoch": 16.21605280395022, + "grad_norm": 1.2176182270050049, + "learning_rate": 0.0001, + "loss": 61.6772, + "step": 10070 + }, + { + "epoch": 16.232176147528595, + "grad_norm": 0.828171968460083, + "learning_rate": 0.0001, + "loss": 61.3296, + "step": 10080 + }, + { + "epoch": 16.24829949110697, + "grad_norm": 0.6884972453117371, + "learning_rate": 0.0001, + "loss": 61.5371, + "step": 10090 + }, + { + "epoch": 16.264422834685345, + "grad_norm": 0.6455051302909851, + "learning_rate": 0.0001, + "loss": 60.6548, + "step": 10100 + }, + { + "epoch": 16.280546178263716, + "grad_norm": 0.687982976436615, + "learning_rate": 0.0001, + "loss": 62.0357, + "step": 10110 + }, + { + "epoch": 16.29666952184209, + "grad_norm": 0.811744213104248, + "learning_rate": 0.0001, + "loss": 60.5664, + "step": 10120 + }, + { + "epoch": 16.312792865420466, + "grad_norm": 0.7140675187110901, + "learning_rate": 0.0001, + "loss": 62.9491, + "step": 10130 + }, + { + "epoch": 16.32891620899884, + "grad_norm": 0.9068045020103455, + "learning_rate": 0.0001, + "loss": 61.118, + "step": 10140 + }, + { + "epoch": 16.345039552577216, + "grad_norm": 0.9028809666633606, + "learning_rate": 0.0001, + "loss": 61.4308, + "step": 10150 + }, + { + "epoch": 16.36116289615559, + "grad_norm": 0.7502166628837585, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10160 + }, + { + "epoch": 16.377286239733966, + "grad_norm": 0.807391345500946, + "learning_rate": 0.0001, + "loss": 61.6825, + "step": 10170 + }, + { + "epoch": 16.39340958331234, + "grad_norm": 0.762725830078125, + "learning_rate": 0.0001, + "loss": 62.7879, + "step": 10180 + }, + { + "epoch": 16.409532926890712, + "grad_norm": 0.7973660230636597, + "learning_rate": 0.0001, + "loss": 61.0727, + "step": 10190 + }, + { + "epoch": 16.425656270469087, + "grad_norm": 1.004622459411621, + "learning_rate": 0.0001, + "loss": 59.8048, + "step": 10200 + }, + { + "epoch": 16.441779614047462, + "grad_norm": 0.8243395686149597, + "learning_rate": 0.0001, + "loss": 63.9312, + "step": 10210 + }, + { + "epoch": 16.457902957625837, + "grad_norm": 0.7006281614303589, + "learning_rate": 0.0001, + "loss": 61.3766, + "step": 10220 + }, + { + "epoch": 16.474026301204212, + "grad_norm": 0.8672197461128235, + "learning_rate": 0.0001, + "loss": 61.6659, + "step": 10230 + }, + { + "epoch": 16.490149644782587, + "grad_norm": 0.7836477160453796, + "learning_rate": 0.0001, + "loss": 61.3686, + "step": 10240 + }, + { + "epoch": 16.506272988360962, + "grad_norm": 0.5842607617378235, + "learning_rate": 0.0001, + "loss": 62.616, + "step": 10250 + }, + { + "epoch": 16.522396331939337, + "grad_norm": 0.6777102947235107, + "learning_rate": 0.0001, + "loss": 62.1135, + "step": 10260 + }, + { + "epoch": 16.538519675517712, + "grad_norm": 0.856697678565979, + "learning_rate": 0.0001, + "loss": 61.849, + "step": 10270 + }, + { + "epoch": 16.554643019096083, + "grad_norm": 0.7300705313682556, + "learning_rate": 0.0001, + "loss": 60.7678, + "step": 10280 + }, + { + "epoch": 16.57076636267446, + "grad_norm": 0.799802839756012, + "learning_rate": 0.0001, + "loss": 62.4888, + "step": 10290 + }, + { + "epoch": 16.586889706252833, + "grad_norm": 0.791251540184021, + "learning_rate": 0.0001, + "loss": 61.7589, + "step": 10300 + }, + { + "epoch": 16.60301304983121, + "grad_norm": 0.6498433947563171, + "learning_rate": 0.0001, + "loss": 61.087, + "step": 10310 + }, + { + "epoch": 16.619136393409583, + "grad_norm": 0.702925443649292, + "learning_rate": 0.0001, + "loss": 59.3493, + "step": 10320 + }, + { + "epoch": 16.63525973698796, + "grad_norm": 0.8508023619651794, + "learning_rate": 0.0001, + "loss": 61.2188, + "step": 10330 + }, + { + "epoch": 16.651383080566333, + "grad_norm": 0.8325870633125305, + "learning_rate": 0.0001, + "loss": 62.9164, + "step": 10340 + }, + { + "epoch": 16.667506424144708, + "grad_norm": 0.7140557765960693, + "learning_rate": 0.0001, + "loss": 62.5838, + "step": 10350 + }, + { + "epoch": 16.683629767723083, + "grad_norm": 0.5574952363967896, + "learning_rate": 0.0001, + "loss": 59.767, + "step": 10360 + }, + { + "epoch": 16.699753111301455, + "grad_norm": 0.8022965788841248, + "learning_rate": 0.0001, + "loss": 60.2198, + "step": 10370 + }, + { + "epoch": 16.71587645487983, + "grad_norm": 1.1291698217391968, + "learning_rate": 0.0001, + "loss": 61.4908, + "step": 10380 + }, + { + "epoch": 16.731999798458205, + "grad_norm": 0.8651410937309265, + "learning_rate": 0.0001, + "loss": 60.5228, + "step": 10390 + }, + { + "epoch": 16.74812314203658, + "grad_norm": 0.8426535129547119, + "learning_rate": 0.0001, + "loss": 62.3786, + "step": 10400 + }, + { + "epoch": 16.764246485614954, + "grad_norm": 0.6329453587532043, + "learning_rate": 0.0001, + "loss": 60.9621, + "step": 10410 + }, + { + "epoch": 16.78036982919333, + "grad_norm": 0.6745129227638245, + "learning_rate": 0.0001, + "loss": 59.3405, + "step": 10420 + }, + { + "epoch": 16.796493172771704, + "grad_norm": 0.8483593463897705, + "learning_rate": 0.0001, + "loss": 61.1527, + "step": 10430 + }, + { + "epoch": 16.81261651635008, + "grad_norm": 0.8649617433547974, + "learning_rate": 0.0001, + "loss": 61.051, + "step": 10440 + }, + { + "epoch": 16.828739859928454, + "grad_norm": 0.7309728264808655, + "learning_rate": 0.0001, + "loss": 60.3626, + "step": 10450 + }, + { + "epoch": 16.844863203506826, + "grad_norm": 0.6467884182929993, + "learning_rate": 0.0001, + "loss": 63.8254, + "step": 10460 + }, + { + "epoch": 16.8609865470852, + "grad_norm": 0.7628102898597717, + "learning_rate": 0.0001, + "loss": 59.5604, + "step": 10470 + }, + { + "epoch": 16.877109890663576, + "grad_norm": 0.78266841173172, + "learning_rate": 0.0001, + "loss": 60.9387, + "step": 10480 + }, + { + "epoch": 16.89323323424195, + "grad_norm": 0.6936920285224915, + "learning_rate": 0.0001, + "loss": 62.5492, + "step": 10490 + }, + { + "epoch": 16.909356577820326, + "grad_norm": 0.8343939185142517, + "learning_rate": 0.0001, + "loss": 60.9975, + "step": 10500 + }, + { + "epoch": 16.9254799213987, + "grad_norm": 0.798984169960022, + "learning_rate": 0.0001, + "loss": 59.5835, + "step": 10510 + }, + { + "epoch": 16.941603264977076, + "grad_norm": 1.369275450706482, + "learning_rate": 0.0001, + "loss": 61.749, + "step": 10520 + }, + { + "epoch": 16.95772660855545, + "grad_norm": 0.5993408560752869, + "learning_rate": 0.0001, + "loss": 62.1093, + "step": 10530 + }, + { + "epoch": 16.973849952133826, + "grad_norm": 0.8153544664382935, + "learning_rate": 0.0001, + "loss": 61.857, + "step": 10540 + }, + { + "epoch": 16.989973295712197, + "grad_norm": 0.7042689919471741, + "learning_rate": 0.0001, + "loss": 60.5178, + "step": 10550 + }, + { + "epoch": 17.00483700307351, + "grad_norm": 1.1931812763214111, + "learning_rate": 0.0001, + "loss": 56.4815, + "step": 10560 + }, + { + "epoch": 17.020960346651886, + "grad_norm": 0.9051854610443115, + "learning_rate": 0.0001, + "loss": 60.7729, + "step": 10570 + }, + { + "epoch": 17.03708369023026, + "grad_norm": 1.093518614768982, + "learning_rate": 0.0001, + "loss": 60.904, + "step": 10580 + }, + { + "epoch": 17.053207033808636, + "grad_norm": 0.7545284032821655, + "learning_rate": 0.0001, + "loss": 59.7165, + "step": 10590 + }, + { + "epoch": 17.06933037738701, + "grad_norm": 0.8542641401290894, + "learning_rate": 0.0001, + "loss": 60.8947, + "step": 10600 + }, + { + "epoch": 17.085453720965386, + "grad_norm": 0.7393799424171448, + "learning_rate": 0.0001, + "loss": 62.8384, + "step": 10610 + }, + { + "epoch": 17.10157706454376, + "grad_norm": 0.6756435632705688, + "learning_rate": 0.0001, + "loss": 61.164, + "step": 10620 + }, + { + "epoch": 17.117700408122133, + "grad_norm": 0.725150465965271, + "learning_rate": 0.0001, + "loss": 60.3789, + "step": 10630 + }, + { + "epoch": 17.133823751700508, + "grad_norm": 1.1529417037963867, + "learning_rate": 0.0001, + "loss": 62.2465, + "step": 10640 + }, + { + "epoch": 17.149947095278883, + "grad_norm": 0.8266693353652954, + "learning_rate": 0.0001, + "loss": 61.6755, + "step": 10650 + }, + { + "epoch": 17.166070438857258, + "grad_norm": 0.7444040775299072, + "learning_rate": 0.0001, + "loss": 62.7084, + "step": 10660 + }, + { + "epoch": 17.182193782435633, + "grad_norm": 0.8164665102958679, + "learning_rate": 0.0001, + "loss": 61.6032, + "step": 10670 + }, + { + "epoch": 17.198317126014008, + "grad_norm": 0.8215296268463135, + "learning_rate": 0.0001, + "loss": 61.033, + "step": 10680 + }, + { + "epoch": 17.214440469592382, + "grad_norm": 0.7939956188201904, + "learning_rate": 0.0001, + "loss": 60.5016, + "step": 10690 + }, + { + "epoch": 17.230563813170757, + "grad_norm": 0.9533108472824097, + "learning_rate": 0.0001, + "loss": 62.6995, + "step": 10700 + }, + { + "epoch": 17.246687156749132, + "grad_norm": 0.6602820754051208, + "learning_rate": 0.0001, + "loss": 61.2709, + "step": 10710 + }, + { + "epoch": 17.262810500327504, + "grad_norm": 0.8369201421737671, + "learning_rate": 0.0001, + "loss": 63.1138, + "step": 10720 + }, + { + "epoch": 17.27893384390588, + "grad_norm": 0.7431164979934692, + "learning_rate": 0.0001, + "loss": 62.8588, + "step": 10730 + }, + { + "epoch": 17.295057187484254, + "grad_norm": 0.7957271337509155, + "learning_rate": 0.0001, + "loss": 60.5119, + "step": 10740 + }, + { + "epoch": 17.31118053106263, + "grad_norm": 0.9407948851585388, + "learning_rate": 0.0001, + "loss": 61.1877, + "step": 10750 + }, + { + "epoch": 17.327303874641004, + "grad_norm": 0.6641581654548645, + "learning_rate": 0.0001, + "loss": 60.542, + "step": 10760 + }, + { + "epoch": 17.34342721821938, + "grad_norm": 0.7786923050880432, + "learning_rate": 0.0001, + "loss": 62.1582, + "step": 10770 + }, + { + "epoch": 17.359550561797754, + "grad_norm": 0.8150411248207092, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10780 + }, + { + "epoch": 17.37567390537613, + "grad_norm": 0.8315487504005432, + "learning_rate": 0.0001, + "loss": 60.4382, + "step": 10790 + }, + { + "epoch": 17.391797248954504, + "grad_norm": 0.8305006623268127, + "learning_rate": 0.0001, + "loss": 61.3262, + "step": 10800 + }, + { + "epoch": 17.407920592532875, + "grad_norm": 0.897731363773346, + "learning_rate": 0.0001, + "loss": 60.7809, + "step": 10810 + }, + { + "epoch": 17.42404393611125, + "grad_norm": 0.7461619973182678, + "learning_rate": 0.0001, + "loss": 61.3858, + "step": 10820 + }, + { + "epoch": 17.440167279689625, + "grad_norm": 0.6750472187995911, + "learning_rate": 0.0001, + "loss": 60.2692, + "step": 10830 + }, + { + "epoch": 17.456290623268, + "grad_norm": 0.5412933230400085, + "learning_rate": 0.0001, + "loss": 61.7872, + "step": 10840 + }, + { + "epoch": 17.472413966846375, + "grad_norm": 0.6485120058059692, + "learning_rate": 0.0001, + "loss": 59.5407, + "step": 10850 + }, + { + "epoch": 17.48853731042475, + "grad_norm": 0.818708598613739, + "learning_rate": 0.0001, + "loss": 61.8066, + "step": 10860 + }, + { + "epoch": 17.504660654003125, + "grad_norm": 0.7121092081069946, + "learning_rate": 0.0001, + "loss": 61.8086, + "step": 10870 + }, + { + "epoch": 17.5207839975815, + "grad_norm": 0.7031684517860413, + "learning_rate": 0.0001, + "loss": 60.0706, + "step": 10880 + }, + { + "epoch": 17.536907341159875, + "grad_norm": 0.6259323358535767, + "learning_rate": 0.0001, + "loss": 61.9625, + "step": 10890 + }, + { + "epoch": 17.553030684738246, + "grad_norm": 0.8526529669761658, + "learning_rate": 0.0001, + "loss": 61.3299, + "step": 10900 + }, + { + "epoch": 17.56915402831662, + "grad_norm": 0.6860859990119934, + "learning_rate": 0.0001, + "loss": 61.3193, + "step": 10910 + }, + { + "epoch": 17.585277371894996, + "grad_norm": 0.7715994119644165, + "learning_rate": 0.0001, + "loss": 61.9111, + "step": 10920 + }, + { + "epoch": 17.60140071547337, + "grad_norm": 0.7171186804771423, + "learning_rate": 0.0001, + "loss": 60.3638, + "step": 10930 + }, + { + "epoch": 17.617524059051746, + "grad_norm": 0.7796649932861328, + "learning_rate": 0.0001, + "loss": 61.9437, + "step": 10940 + }, + { + "epoch": 17.63364740263012, + "grad_norm": 0.7930101752281189, + "learning_rate": 0.0001, + "loss": 60.1085, + "step": 10950 + }, + { + "epoch": 17.649770746208496, + "grad_norm": 0.6370466351509094, + "learning_rate": 0.0001, + "loss": 61.8433, + "step": 10960 + }, + { + "epoch": 17.66589408978687, + "grad_norm": 0.7830466628074646, + "learning_rate": 0.0001, + "loss": 61.3403, + "step": 10970 + }, + { + "epoch": 17.682017433365242, + "grad_norm": 0.873456597328186, + "learning_rate": 0.0001, + "loss": 62.4567, + "step": 10980 + }, + { + "epoch": 17.698140776943617, + "grad_norm": 0.7560754418373108, + "learning_rate": 0.0001, + "loss": 60.9808, + "step": 10990 + }, + { + "epoch": 17.714264120521992, + "grad_norm": 0.6832230091094971, + "learning_rate": 0.0001, + "loss": 61.5059, + "step": 11000 + }, + { + "epoch": 17.730387464100367, + "grad_norm": 0.7701742649078369, + "learning_rate": 0.0001, + "loss": 62.4963, + "step": 11010 + }, + { + "epoch": 17.746510807678742, + "grad_norm": 0.7088269591331482, + "learning_rate": 0.0001, + "loss": 62.6467, + "step": 11020 + }, + { + "epoch": 17.762634151257117, + "grad_norm": 0.8424612879753113, + "learning_rate": 0.0001, + "loss": 60.1213, + "step": 11030 + }, + { + "epoch": 17.778757494835492, + "grad_norm": 0.9974421858787537, + "learning_rate": 0.0001, + "loss": 61.6051, + "step": 11040 + }, + { + "epoch": 17.794880838413867, + "grad_norm": 0.701970636844635, + "learning_rate": 0.0001, + "loss": 62.4155, + "step": 11050 + }, + { + "epoch": 17.811004181992242, + "grad_norm": 0.6826842427253723, + "learning_rate": 0.0001, + "loss": 60.2127, + "step": 11060 + }, + { + "epoch": 17.827127525570614, + "grad_norm": 0.7019745707511902, + "learning_rate": 0.0001, + "loss": 60.9281, + "step": 11070 + }, + { + "epoch": 17.84325086914899, + "grad_norm": 0.836280345916748, + "learning_rate": 0.0001, + "loss": 61.1217, + "step": 11080 + }, + { + "epoch": 17.859374212727364, + "grad_norm": 0.6405923366546631, + "learning_rate": 0.0001, + "loss": 61.6393, + "step": 11090 + }, + { + "epoch": 17.87549755630574, + "grad_norm": 0.6809528470039368, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 11100 + }, + { + "epoch": 17.891620899884114, + "grad_norm": 0.6983410120010376, + "learning_rate": 0.0001, + "loss": 59.933, + "step": 11110 + }, + { + "epoch": 17.90774424346249, + "grad_norm": 1.0240110158920288, + "learning_rate": 0.0001, + "loss": 61.1964, + "step": 11120 + }, + { + "epoch": 17.923867587040863, + "grad_norm": 0.860657811164856, + "learning_rate": 0.0001, + "loss": 61.1613, + "step": 11130 + }, + { + "epoch": 17.93999093061924, + "grad_norm": 0.6434107422828674, + "learning_rate": 0.0001, + "loss": 60.2435, + "step": 11140 + }, + { + "epoch": 17.956114274197613, + "grad_norm": 0.7400848269462585, + "learning_rate": 0.0001, + "loss": 62.452, + "step": 11150 + }, + { + "epoch": 17.972237617775985, + "grad_norm": 0.6526095867156982, + "learning_rate": 0.0001, + "loss": 62.3344, + "step": 11160 + }, + { + "epoch": 17.98836096135436, + "grad_norm": 0.6971097588539124, + "learning_rate": 0.0001, + "loss": 60.2415, + "step": 11170 + }, + { + "epoch": 18.003224668715674, + "grad_norm": 0.724881649017334, + "learning_rate": 0.0001, + "loss": 56.0513, + "step": 11180 + }, + { + "epoch": 18.01934801229405, + "grad_norm": 0.653710663318634, + "learning_rate": 0.0001, + "loss": 60.8364, + "step": 11190 + }, + { + "epoch": 18.035471355872424, + "grad_norm": 0.7356674075126648, + "learning_rate": 0.0001, + "loss": 61.6681, + "step": 11200 + }, + { + "epoch": 18.0515946994508, + "grad_norm": 0.6726879477500916, + "learning_rate": 0.0001, + "loss": 60.6682, + "step": 11210 + }, + { + "epoch": 18.067718043029174, + "grad_norm": 0.7367722392082214, + "learning_rate": 0.0001, + "loss": 63.0462, + "step": 11220 + }, + { + "epoch": 18.08384138660755, + "grad_norm": 0.6687121987342834, + "learning_rate": 0.0001, + "loss": 60.8707, + "step": 11230 + }, + { + "epoch": 18.09996473018592, + "grad_norm": 1.0027060508728027, + "learning_rate": 0.0001, + "loss": 62.4703, + "step": 11240 + }, + { + "epoch": 18.116088073764296, + "grad_norm": 1.0447720289230347, + "learning_rate": 0.0001, + "loss": 62.1746, + "step": 11250 + }, + { + "epoch": 18.13221141734267, + "grad_norm": 0.7687053084373474, + "learning_rate": 0.0001, + "loss": 60.1413, + "step": 11260 + }, + { + "epoch": 18.148334760921045, + "grad_norm": 0.7250781059265137, + "learning_rate": 0.0001, + "loss": 60.1143, + "step": 11270 + }, + { + "epoch": 18.16445810449942, + "grad_norm": 0.7356208562850952, + "learning_rate": 0.0001, + "loss": 62.0788, + "step": 11280 + }, + { + "epoch": 18.180581448077795, + "grad_norm": 0.7838225364685059, + "learning_rate": 0.0001, + "loss": 61.7046, + "step": 11290 + }, + { + "epoch": 18.19670479165617, + "grad_norm": 0.7600042819976807, + "learning_rate": 0.0001, + "loss": 61.4646, + "step": 11300 + }, + { + "epoch": 18.212828135234545, + "grad_norm": 0.7339571714401245, + "learning_rate": 0.0001, + "loss": 61.0983, + "step": 11310 + }, + { + "epoch": 18.22895147881292, + "grad_norm": 0.6982674598693848, + "learning_rate": 0.0001, + "loss": 60.2853, + "step": 11320 + }, + { + "epoch": 18.24507482239129, + "grad_norm": 0.7572301030158997, + "learning_rate": 0.0001, + "loss": 61.8967, + "step": 11330 + }, + { + "epoch": 18.261198165969667, + "grad_norm": 0.6206921935081482, + "learning_rate": 0.0001, + "loss": 62.1467, + "step": 11340 + }, + { + "epoch": 18.27732150954804, + "grad_norm": 0.658437967300415, + "learning_rate": 0.0001, + "loss": 61.2117, + "step": 11350 + }, + { + "epoch": 18.293444853126417, + "grad_norm": 0.7880764603614807, + "learning_rate": 0.0001, + "loss": 61.6246, + "step": 11360 + }, + { + "epoch": 18.30956819670479, + "grad_norm": 0.6601800918579102, + "learning_rate": 0.0001, + "loss": 61.4818, + "step": 11370 + }, + { + "epoch": 18.325691540283167, + "grad_norm": 0.6816290616989136, + "learning_rate": 0.0001, + "loss": 60.4834, + "step": 11380 + }, + { + "epoch": 18.34181488386154, + "grad_norm": 0.7667511105537415, + "learning_rate": 0.0001, + "loss": 60.9106, + "step": 11390 + }, + { + "epoch": 18.357938227439917, + "grad_norm": 0.6490039825439453, + "learning_rate": 0.0001, + "loss": 59.6968, + "step": 11400 + }, + { + "epoch": 18.37406157101829, + "grad_norm": 0.6865658760070801, + "learning_rate": 0.0001, + "loss": 61.7898, + "step": 11410 + }, + { + "epoch": 18.390184914596663, + "grad_norm": 0.8605737090110779, + "learning_rate": 0.0001, + "loss": 60.924, + "step": 11420 + }, + { + "epoch": 18.406308258175038, + "grad_norm": 1.0474110841751099, + "learning_rate": 0.0001, + "loss": 61.0381, + "step": 11430 + }, + { + "epoch": 18.422431601753413, + "grad_norm": 0.9241340160369873, + "learning_rate": 0.0001, + "loss": 61.2609, + "step": 11440 + }, + { + "epoch": 18.438554945331788, + "grad_norm": 1.1441621780395508, + "learning_rate": 0.0001, + "loss": 60.5495, + "step": 11450 + }, + { + "epoch": 18.454678288910163, + "grad_norm": 0.7277412414550781, + "learning_rate": 0.0001, + "loss": 60.4654, + "step": 11460 + }, + { + "epoch": 18.470801632488538, + "grad_norm": 0.6668717861175537, + "learning_rate": 0.0001, + "loss": 60.057, + "step": 11470 + }, + { + "epoch": 18.486924976066913, + "grad_norm": 0.7335819005966187, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 11480 + }, + { + "epoch": 18.503048319645288, + "grad_norm": 0.7261807918548584, + "learning_rate": 0.0001, + "loss": 61.1508, + "step": 11490 + }, + { + "epoch": 18.519171663223663, + "grad_norm": 0.7715283036231995, + "learning_rate": 0.0001, + "loss": 61.5492, + "step": 11500 + }, + { + "epoch": 18.535295006802034, + "grad_norm": 0.9603212475776672, + "learning_rate": 0.0001, + "loss": 60.4321, + "step": 11510 + }, + { + "epoch": 18.55141835038041, + "grad_norm": 0.6130046844482422, + "learning_rate": 0.0001, + "loss": 61.2942, + "step": 11520 + }, + { + "epoch": 18.567541693958784, + "grad_norm": 0.8147348165512085, + "learning_rate": 0.0001, + "loss": 61.4313, + "step": 11530 + }, + { + "epoch": 18.58366503753716, + "grad_norm": 0.7909948825836182, + "learning_rate": 0.0001, + "loss": 60.3547, + "step": 11540 + }, + { + "epoch": 18.599788381115534, + "grad_norm": 0.6470443606376648, + "learning_rate": 0.0001, + "loss": 61.4144, + "step": 11550 + }, + { + "epoch": 18.61591172469391, + "grad_norm": 0.7132186889648438, + "learning_rate": 0.0001, + "loss": 61.8102, + "step": 11560 + }, + { + "epoch": 18.632035068272284, + "grad_norm": 0.7054323554039001, + "learning_rate": 0.0001, + "loss": 62.1431, + "step": 11570 + }, + { + "epoch": 18.64815841185066, + "grad_norm": 0.8784761428833008, + "learning_rate": 0.0001, + "loss": 61.6653, + "step": 11580 + }, + { + "epoch": 18.66428175542903, + "grad_norm": 0.7143248319625854, + "learning_rate": 0.0001, + "loss": 60.1275, + "step": 11590 + }, + { + "epoch": 18.680405099007405, + "grad_norm": 7.87012243270874, + "learning_rate": 0.0001, + "loss": 61.3484, + "step": 11600 + }, + { + "epoch": 18.69652844258578, + "grad_norm": 0.7263020873069763, + "learning_rate": 0.0001, + "loss": 59.906, + "step": 11610 + }, + { + "epoch": 18.712651786164155, + "grad_norm": 0.6468373537063599, + "learning_rate": 0.0001, + "loss": 61.8781, + "step": 11620 + }, + { + "epoch": 18.72877512974253, + "grad_norm": 3.34307861328125, + "learning_rate": 0.0001, + "loss": 61.0841, + "step": 11630 + }, + { + "epoch": 18.744898473320905, + "grad_norm": 0.7348340153694153, + "learning_rate": 0.0001, + "loss": 61.3201, + "step": 11640 + }, + { + "epoch": 18.76102181689928, + "grad_norm": 0.7646427154541016, + "learning_rate": 0.0001, + "loss": 60.1858, + "step": 11650 + }, + { + "epoch": 18.777145160477655, + "grad_norm": 0.6659495234489441, + "learning_rate": 0.0001, + "loss": 60.8999, + "step": 11660 + }, + { + "epoch": 18.79326850405603, + "grad_norm": 0.9913583993911743, + "learning_rate": 0.0001, + "loss": 59.0732, + "step": 11670 + }, + { + "epoch": 18.8093918476344, + "grad_norm": 0.7480027079582214, + "learning_rate": 0.0001, + "loss": 58.6846, + "step": 11680 + }, + { + "epoch": 18.825515191212777, + "grad_norm": 0.8000630736351013, + "learning_rate": 0.0001, + "loss": 62.8883, + "step": 11690 + }, + { + "epoch": 18.84163853479115, + "grad_norm": 0.8450509309768677, + "learning_rate": 0.0001, + "loss": 63.4962, + "step": 11700 + }, + { + "epoch": 18.857761878369526, + "grad_norm": 0.8365609645843506, + "learning_rate": 0.0001, + "loss": 62.2701, + "step": 11710 + }, + { + "epoch": 18.8738852219479, + "grad_norm": 0.7197144031524658, + "learning_rate": 0.0001, + "loss": 61.3158, + "step": 11720 + }, + { + "epoch": 18.890008565526276, + "grad_norm": 0.6460379958152771, + "learning_rate": 0.0001, + "loss": 62.6645, + "step": 11730 + }, + { + "epoch": 18.90613190910465, + "grad_norm": 0.8203967213630676, + "learning_rate": 0.0001, + "loss": 62.3257, + "step": 11740 + }, + { + "epoch": 18.922255252683026, + "grad_norm": 0.682134211063385, + "learning_rate": 0.0001, + "loss": 59.9144, + "step": 11750 + }, + { + "epoch": 18.9383785962614, + "grad_norm": 0.7080821394920349, + "learning_rate": 0.0001, + "loss": 61.4325, + "step": 11760 + }, + { + "epoch": 18.954501939839773, + "grad_norm": 0.7963712811470032, + "learning_rate": 0.0001, + "loss": 58.6136, + "step": 11770 + }, + { + "epoch": 18.970625283418148, + "grad_norm": 0.7317046523094177, + "learning_rate": 0.0001, + "loss": 61.3507, + "step": 11780 + }, + { + "epoch": 18.986748626996523, + "grad_norm": 0.6586548686027527, + "learning_rate": 0.0001, + "loss": 61.4898, + "step": 11790 + }, + { + "epoch": 19.001612334357837, + "grad_norm": 0.9010424613952637, + "learning_rate": 0.0001, + "loss": 56.8915, + "step": 11800 + }, + { + "epoch": 19.017735677936212, + "grad_norm": 0.7102377414703369, + "learning_rate": 0.0001, + "loss": 62.2097, + "step": 11810 + }, + { + "epoch": 19.033859021514587, + "grad_norm": 0.7959222793579102, + "learning_rate": 0.0001, + "loss": 61.4484, + "step": 11820 + }, + { + "epoch": 19.049982365092962, + "grad_norm": 0.6483829617500305, + "learning_rate": 0.0001, + "loss": 60.3563, + "step": 11830 + }, + { + "epoch": 19.066105708671337, + "grad_norm": 1.1301988363265991, + "learning_rate": 0.0001, + "loss": 61.3698, + "step": 11840 + }, + { + "epoch": 19.082229052249712, + "grad_norm": 0.7304167151451111, + "learning_rate": 0.0001, + "loss": 59.2096, + "step": 11850 + }, + { + "epoch": 19.098352395828083, + "grad_norm": 0.7757825255393982, + "learning_rate": 0.0001, + "loss": 59.914, + "step": 11860 + }, + { + "epoch": 19.11447573940646, + "grad_norm": 0.7009251713752747, + "learning_rate": 0.0001, + "loss": 61.0768, + "step": 11870 + }, + { + "epoch": 19.130599082984833, + "grad_norm": 0.6015574336051941, + "learning_rate": 0.0001, + "loss": 61.2698, + "step": 11880 + }, + { + "epoch": 19.14672242656321, + "grad_norm": 0.8028033971786499, + "learning_rate": 0.0001, + "loss": 61.422, + "step": 11890 + }, + { + "epoch": 19.162845770141583, + "grad_norm": 0.5753505825996399, + "learning_rate": 0.0001, + "loss": 61.5066, + "step": 11900 + }, + { + "epoch": 19.17896911371996, + "grad_norm": 0.7702978849411011, + "learning_rate": 0.0001, + "loss": 60.0737, + "step": 11910 + }, + { + "epoch": 19.195092457298333, + "grad_norm": 0.6469365358352661, + "learning_rate": 0.0001, + "loss": 59.5091, + "step": 11920 + }, + { + "epoch": 19.211215800876708, + "grad_norm": 0.8400216698646545, + "learning_rate": 0.0001, + "loss": 62.3739, + "step": 11930 + }, + { + "epoch": 19.22733914445508, + "grad_norm": 1.9919443130493164, + "learning_rate": 0.0001, + "loss": 59.7271, + "step": 11940 + }, + { + "epoch": 19.243462488033455, + "grad_norm": 1.0247725248336792, + "learning_rate": 0.0001, + "loss": 59.3804, + "step": 11950 + }, + { + "epoch": 19.25958583161183, + "grad_norm": 1.0717105865478516, + "learning_rate": 0.0001, + "loss": 61.3617, + "step": 11960 + }, + { + "epoch": 19.275709175190205, + "grad_norm": 0.6324111223220825, + "learning_rate": 0.0001, + "loss": 59.6094, + "step": 11970 + }, + { + "epoch": 19.29183251876858, + "grad_norm": 0.6665195226669312, + "learning_rate": 0.0001, + "loss": 61.5879, + "step": 11980 + }, + { + "epoch": 19.307955862346954, + "grad_norm": 0.7933892011642456, + "learning_rate": 0.0001, + "loss": 60.4011, + "step": 11990 + }, + { + "epoch": 19.32407920592533, + "grad_norm": 0.8135427236557007, + "learning_rate": 0.0001, + "loss": 61.2936, + "step": 12000 + }, + { + "epoch": 19.340202549503704, + "grad_norm": 0.7154449820518494, + "learning_rate": 0.0001, + "loss": 62.3996, + "step": 12010 + }, + { + "epoch": 19.35632589308208, + "grad_norm": 0.9926077127456665, + "learning_rate": 0.0001, + "loss": 60.5219, + "step": 12020 + }, + { + "epoch": 19.37244923666045, + "grad_norm": 1.013094425201416, + "learning_rate": 0.0001, + "loss": 59.9494, + "step": 12030 + }, + { + "epoch": 19.388572580238826, + "grad_norm": 0.984582245349884, + "learning_rate": 0.0001, + "loss": 59.7877, + "step": 12040 + }, + { + "epoch": 19.4046959238172, + "grad_norm": 1.256519079208374, + "learning_rate": 0.0001, + "loss": 60.2608, + "step": 12050 + }, + { + "epoch": 19.420819267395576, + "grad_norm": 0.7013972997665405, + "learning_rate": 0.0001, + "loss": 63.4573, + "step": 12060 + }, + { + "epoch": 19.43694261097395, + "grad_norm": 0.9027557373046875, + "learning_rate": 0.0001, + "loss": 61.3252, + "step": 12070 + }, + { + "epoch": 19.453065954552326, + "grad_norm": 0.9057058691978455, + "learning_rate": 0.0001, + "loss": 60.4992, + "step": 12080 + }, + { + "epoch": 19.4691892981307, + "grad_norm": 0.6828131079673767, + "learning_rate": 0.0001, + "loss": 61.5369, + "step": 12090 + }, + { + "epoch": 19.485312641709076, + "grad_norm": 0.617858350276947, + "learning_rate": 0.0001, + "loss": 61.4837, + "step": 12100 + }, + { + "epoch": 19.50143598528745, + "grad_norm": 0.7722935676574707, + "learning_rate": 0.0001, + "loss": 61.0886, + "step": 12110 + }, + { + "epoch": 19.517559328865822, + "grad_norm": 0.7039631009101868, + "learning_rate": 0.0001, + "loss": 59.9055, + "step": 12120 + }, + { + "epoch": 19.533682672444197, + "grad_norm": 0.8353239297866821, + "learning_rate": 0.0001, + "loss": 62.0864, + "step": 12130 + }, + { + "epoch": 19.549806016022572, + "grad_norm": 0.764098048210144, + "learning_rate": 0.0001, + "loss": 58.9591, + "step": 12140 + }, + { + "epoch": 19.565929359600947, + "grad_norm": 0.7842984795570374, + "learning_rate": 0.0001, + "loss": 59.0494, + "step": 12150 + }, + { + "epoch": 19.582052703179322, + "grad_norm": 0.7140147686004639, + "learning_rate": 0.0001, + "loss": 60.8067, + "step": 12160 + }, + { + "epoch": 19.598176046757697, + "grad_norm": 0.8182031512260437, + "learning_rate": 0.0001, + "loss": 59.6013, + "step": 12170 + }, + { + "epoch": 19.614299390336072, + "grad_norm": 0.7428567409515381, + "learning_rate": 0.0001, + "loss": 61.8644, + "step": 12180 + }, + { + "epoch": 19.630422733914447, + "grad_norm": 0.6174675822257996, + "learning_rate": 0.0001, + "loss": 60.2086, + "step": 12190 + }, + { + "epoch": 19.646546077492822, + "grad_norm": 0.6416913866996765, + "learning_rate": 0.0001, + "loss": 61.3452, + "step": 12200 + }, + { + "epoch": 19.662669421071193, + "grad_norm": 1.3036755323410034, + "learning_rate": 0.0001, + "loss": 60.5332, + "step": 12210 + }, + { + "epoch": 19.678792764649568, + "grad_norm": 1.230710506439209, + "learning_rate": 0.0001, + "loss": 62.5671, + "step": 12220 + }, + { + "epoch": 19.694916108227943, + "grad_norm": 0.6914290189743042, + "learning_rate": 0.0001, + "loss": 60.7302, + "step": 12230 + }, + { + "epoch": 19.711039451806318, + "grad_norm": 0.6492108106613159, + "learning_rate": 0.0001, + "loss": 60.6465, + "step": 12240 + }, + { + "epoch": 19.727162795384693, + "grad_norm": 0.7104936838150024, + "learning_rate": 0.0001, + "loss": 61.14, + "step": 12250 + }, + { + "epoch": 19.743286138963068, + "grad_norm": 1.0740787982940674, + "learning_rate": 0.0001, + "loss": 60.1977, + "step": 12260 + }, + { + "epoch": 19.759409482541443, + "grad_norm": 0.5764663815498352, + "learning_rate": 0.0001, + "loss": 59.994, + "step": 12270 + }, + { + "epoch": 19.775532826119818, + "grad_norm": 0.6204820871353149, + "learning_rate": 0.0001, + "loss": 60.3091, + "step": 12280 + }, + { + "epoch": 19.791656169698193, + "grad_norm": 0.702944815158844, + "learning_rate": 0.0001, + "loss": 60.8637, + "step": 12290 + }, + { + "epoch": 19.807779513276564, + "grad_norm": 0.7591354250907898, + "learning_rate": 0.0001, + "loss": 59.54, + "step": 12300 + }, + { + "epoch": 19.82390285685494, + "grad_norm": 0.755224883556366, + "learning_rate": 0.0001, + "loss": 60.0285, + "step": 12310 + }, + { + "epoch": 19.840026200433314, + "grad_norm": 0.7563093900680542, + "learning_rate": 0.0001, + "loss": 60.2669, + "step": 12320 + }, + { + "epoch": 19.85614954401169, + "grad_norm": 0.8339024782180786, + "learning_rate": 0.0001, + "loss": 60.606, + "step": 12330 + }, + { + "epoch": 19.872272887590064, + "grad_norm": 0.7017751932144165, + "learning_rate": 0.0001, + "loss": 59.579, + "step": 12340 + }, + { + "epoch": 19.88839623116844, + "grad_norm": 1.3561909198760986, + "learning_rate": 0.0001, + "loss": 59.7727, + "step": 12350 + }, + { + "epoch": 19.904519574746814, + "grad_norm": 0.727020263671875, + "learning_rate": 0.0001, + "loss": 60.6587, + "step": 12360 + }, + { + "epoch": 19.92064291832519, + "grad_norm": 0.6882230043411255, + "learning_rate": 0.0001, + "loss": 60.3916, + "step": 12370 + }, + { + "epoch": 19.93676626190356, + "grad_norm": 0.6410917639732361, + "learning_rate": 0.0001, + "loss": 61.6649, + "step": 12380 + }, + { + "epoch": 19.952889605481936, + "grad_norm": 0.9747995138168335, + "learning_rate": 0.0001, + "loss": 60.4622, + "step": 12390 + }, + { + "epoch": 19.96901294906031, + "grad_norm": 1.260621190071106, + "learning_rate": 0.0001, + "loss": 60.4228, + "step": 12400 + }, + { + "epoch": 19.985136292638686, + "grad_norm": 0.6525571942329407, + "learning_rate": 0.0001, + "loss": 59.0612, + "step": 12410 + }, + { + "epoch": 20.0, + "grad_norm": 0.7864645719528198, + "learning_rate": 0.0001, + "loss": 57.2347, + "step": 12420 + }, + { + "epoch": 20.016123343578375, + "grad_norm": 0.7479318380355835, + "learning_rate": 0.0001, + "loss": 60.9263, + "step": 12430 + }, + { + "epoch": 20.03224668715675, + "grad_norm": 0.6433089375495911, + "learning_rate": 0.0001, + "loss": 59.8846, + "step": 12440 + }, + { + "epoch": 20.048370030735125, + "grad_norm": 0.7594776153564453, + "learning_rate": 0.0001, + "loss": 61.7793, + "step": 12450 + }, + { + "epoch": 20.0644933743135, + "grad_norm": 0.8710535764694214, + "learning_rate": 0.0001, + "loss": 60.5331, + "step": 12460 + }, + { + "epoch": 20.08061671789187, + "grad_norm": 0.7302114963531494, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 12470 + }, + { + "epoch": 20.096740061470246, + "grad_norm": 0.6039112210273743, + "learning_rate": 0.0001, + "loss": 60.8272, + "step": 12480 + }, + { + "epoch": 20.11286340504862, + "grad_norm": 0.7339144945144653, + "learning_rate": 0.0001, + "loss": 61.4149, + "step": 12490 + }, + { + "epoch": 20.128986748626996, + "grad_norm": 0.7278878092765808, + "learning_rate": 0.0001, + "loss": 62.631, + "step": 12500 + }, + { + "epoch": 20.14511009220537, + "grad_norm": 0.7241142988204956, + "learning_rate": 0.0001, + "loss": 59.9328, + "step": 12510 + }, + { + "epoch": 20.161233435783746, + "grad_norm": 0.6967542767524719, + "learning_rate": 0.0001, + "loss": 58.7463, + "step": 12520 + }, + { + "epoch": 20.17735677936212, + "grad_norm": 0.7326555848121643, + "learning_rate": 0.0001, + "loss": 59.7082, + "step": 12530 + }, + { + "epoch": 20.193480122940496, + "grad_norm": 0.6706270575523376, + "learning_rate": 0.0001, + "loss": 60.5978, + "step": 12540 + }, + { + "epoch": 20.20960346651887, + "grad_norm": 0.8541543483734131, + "learning_rate": 0.0001, + "loss": 58.9632, + "step": 12550 + }, + { + "epoch": 20.225726810097242, + "grad_norm": 0.6599560379981995, + "learning_rate": 0.0001, + "loss": 58.4572, + "step": 12560 + }, + { + "epoch": 20.241850153675617, + "grad_norm": 0.6668654680252075, + "learning_rate": 0.0001, + "loss": 61.8741, + "step": 12570 + }, + { + "epoch": 20.257973497253992, + "grad_norm": 0.6453808546066284, + "learning_rate": 0.0001, + "loss": 60.2538, + "step": 12580 + }, + { + "epoch": 20.274096840832367, + "grad_norm": 0.7429870963096619, + "learning_rate": 0.0001, + "loss": 60.5795, + "step": 12590 + }, + { + "epoch": 20.290220184410742, + "grad_norm": 0.7588251829147339, + "learning_rate": 0.0001, + "loss": 60.6261, + "step": 12600 + }, + { + "epoch": 20.306343527989117, + "grad_norm": 0.6430977582931519, + "learning_rate": 0.0001, + "loss": 61.0344, + "step": 12610 + }, + { + "epoch": 20.322466871567492, + "grad_norm": 0.6879712343215942, + "learning_rate": 0.0001, + "loss": 62.2668, + "step": 12620 + }, + { + "epoch": 20.338590215145867, + "grad_norm": 0.7650917172431946, + "learning_rate": 0.0001, + "loss": 60.3852, + "step": 12630 + }, + { + "epoch": 20.35471355872424, + "grad_norm": 0.5326449871063232, + "learning_rate": 0.0001, + "loss": 60.6138, + "step": 12640 + }, + { + "epoch": 20.370836902302614, + "grad_norm": 0.7018648982048035, + "learning_rate": 0.0001, + "loss": 62.1081, + "step": 12650 + }, + { + "epoch": 20.38696024588099, + "grad_norm": 0.7988570332527161, + "learning_rate": 0.0001, + "loss": 61.2661, + "step": 12660 + }, + { + "epoch": 20.403083589459364, + "grad_norm": 0.6800012588500977, + "learning_rate": 0.0001, + "loss": 59.9279, + "step": 12670 + }, + { + "epoch": 20.41920693303774, + "grad_norm": 0.6380598545074463, + "learning_rate": 0.0001, + "loss": 60.4877, + "step": 12680 + }, + { + "epoch": 20.435330276616114, + "grad_norm": 0.6755974888801575, + "learning_rate": 0.0001, + "loss": 60.9729, + "step": 12690 + }, + { + "epoch": 20.45145362019449, + "grad_norm": 0.8057810664176941, + "learning_rate": 0.0001, + "loss": 59.6747, + "step": 12700 + }, + { + "epoch": 20.467576963772864, + "grad_norm": 0.6942827105522156, + "learning_rate": 0.0001, + "loss": 60.7314, + "step": 12710 + }, + { + "epoch": 20.48370030735124, + "grad_norm": 1.107717752456665, + "learning_rate": 0.0001, + "loss": 59.1397, + "step": 12720 + }, + { + "epoch": 20.49982365092961, + "grad_norm": 2.6167991161346436, + "learning_rate": 0.0001, + "loss": 60.5604, + "step": 12730 + }, + { + "epoch": 20.515946994507985, + "grad_norm": 0.6041843891143799, + "learning_rate": 0.0001, + "loss": 60.6188, + "step": 12740 + }, + { + "epoch": 20.53207033808636, + "grad_norm": 0.5643273591995239, + "learning_rate": 0.0001, + "loss": 59.3172, + "step": 12750 + }, + { + "epoch": 20.548193681664735, + "grad_norm": 0.7615086436271667, + "learning_rate": 0.0001, + "loss": 60.9263, + "step": 12760 + }, + { + "epoch": 20.56431702524311, + "grad_norm": 0.692243754863739, + "learning_rate": 0.0001, + "loss": 58.3092, + "step": 12770 + }, + { + "epoch": 20.580440368821485, + "grad_norm": 0.6447681784629822, + "learning_rate": 0.0001, + "loss": 58.0169, + "step": 12780 + }, + { + "epoch": 20.59656371239986, + "grad_norm": 0.7043688297271729, + "learning_rate": 0.0001, + "loss": 58.8346, + "step": 12790 + }, + { + "epoch": 20.612687055978235, + "grad_norm": 0.8017414808273315, + "learning_rate": 0.0001, + "loss": 60.7451, + "step": 12800 + }, + { + "epoch": 20.62881039955661, + "grad_norm": 0.8983294367790222, + "learning_rate": 0.0001, + "loss": 59.6858, + "step": 12810 + }, + { + "epoch": 20.64493374313498, + "grad_norm": 1.0817819833755493, + "learning_rate": 0.0001, + "loss": 60.8232, + "step": 12820 + }, + { + "epoch": 20.661057086713356, + "grad_norm": 0.5739386081695557, + "learning_rate": 0.0001, + "loss": 60.7852, + "step": 12830 + }, + { + "epoch": 20.67718043029173, + "grad_norm": 1.007978916168213, + "learning_rate": 0.0001, + "loss": 61.7002, + "step": 12840 + }, + { + "epoch": 20.693303773870106, + "grad_norm": 0.7019670009613037, + "learning_rate": 0.0001, + "loss": 59.7211, + "step": 12850 + }, + { + "epoch": 20.70942711744848, + "grad_norm": 0.7891802191734314, + "learning_rate": 0.0001, + "loss": 61.4963, + "step": 12860 + }, + { + "epoch": 20.725550461026856, + "grad_norm": 0.6955318450927734, + "learning_rate": 0.0001, + "loss": 60.8528, + "step": 12870 + }, + { + "epoch": 20.74167380460523, + "grad_norm": 1.9988878965377808, + "learning_rate": 0.0001, + "loss": 58.7444, + "step": 12880 + }, + { + "epoch": 20.757797148183606, + "grad_norm": 0.7859933972358704, + "learning_rate": 0.0001, + "loss": 60.0696, + "step": 12890 + }, + { + "epoch": 20.77392049176198, + "grad_norm": 0.8253535628318787, + "learning_rate": 0.0001, + "loss": 61.3408, + "step": 12900 + }, + { + "epoch": 20.790043835340352, + "grad_norm": 0.6639575362205505, + "learning_rate": 0.0001, + "loss": 58.3044, + "step": 12910 + }, + { + "epoch": 20.806167178918727, + "grad_norm": 0.6754361391067505, + "learning_rate": 0.0001, + "loss": 59.8301, + "step": 12920 + }, + { + "epoch": 20.822290522497102, + "grad_norm": 0.7306725382804871, + "learning_rate": 0.0001, + "loss": 60.0021, + "step": 12930 + }, + { + "epoch": 20.838413866075477, + "grad_norm": 6.799670696258545, + "learning_rate": 0.0001, + "loss": 57.7934, + "step": 12940 + }, + { + "epoch": 20.854537209653852, + "grad_norm": 0.6870545744895935, + "learning_rate": 0.0001, + "loss": 59.2313, + "step": 12950 + }, + { + "epoch": 20.870660553232227, + "grad_norm": 0.6980645060539246, + "learning_rate": 0.0001, + "loss": 60.1903, + "step": 12960 + }, + { + "epoch": 20.886783896810602, + "grad_norm": 0.7006897926330566, + "learning_rate": 0.0001, + "loss": 60.9525, + "step": 12970 + }, + { + "epoch": 20.902907240388977, + "grad_norm": 0.7185907363891602, + "learning_rate": 0.0001, + "loss": 61.8833, + "step": 12980 + }, + { + "epoch": 20.91903058396735, + "grad_norm": 0.6029082536697388, + "learning_rate": 0.0001, + "loss": 60.3764, + "step": 12990 + }, + { + "epoch": 20.935153927545723, + "grad_norm": 0.8502999544143677, + "learning_rate": 0.0001, + "loss": 59.0051, + "step": 13000 + }, + { + "epoch": 20.9512772711241, + "grad_norm": 0.908899188041687, + "learning_rate": 0.0001, + "loss": 59.0677, + "step": 13010 + }, + { + "epoch": 20.967400614702473, + "grad_norm": 0.6983920335769653, + "learning_rate": 0.0001, + "loss": 60.8523, + "step": 13020 + }, + { + "epoch": 20.98352395828085, + "grad_norm": 0.9079043865203857, + "learning_rate": 0.0001, + "loss": 58.8014, + "step": 13030 + }, + { + "epoch": 20.999647301859223, + "grad_norm": 0.7139968872070312, + "learning_rate": 0.0001, + "loss": 59.3019, + "step": 13040 + }, + { + "epoch": 21.014511009220538, + "grad_norm": 0.7472447156906128, + "learning_rate": 0.0001, + "loss": 56.1178, + "step": 13050 + }, + { + "epoch": 21.030634352798913, + "grad_norm": 0.5645695328712463, + "learning_rate": 0.0001, + "loss": 60.5944, + "step": 13060 + }, + { + "epoch": 21.046757696377288, + "grad_norm": 0.6419529318809509, + "learning_rate": 0.0001, + "loss": 59.5792, + "step": 13070 + }, + { + "epoch": 21.06288103995566, + "grad_norm": 0.6508122682571411, + "learning_rate": 0.0001, + "loss": 58.995, + "step": 13080 + }, + { + "epoch": 21.079004383534034, + "grad_norm": 0.6605037450790405, + "learning_rate": 0.0001, + "loss": 60.9783, + "step": 13090 + }, + { + "epoch": 21.09512772711241, + "grad_norm": 0.6636376976966858, + "learning_rate": 0.0001, + "loss": 60.3516, + "step": 13100 + }, + { + "epoch": 21.111251070690784, + "grad_norm": 0.7671985030174255, + "learning_rate": 0.0001, + "loss": 61.8918, + "step": 13110 + }, + { + "epoch": 21.12737441426916, + "grad_norm": 0.859834611415863, + "learning_rate": 0.0001, + "loss": 61.0766, + "step": 13120 + }, + { + "epoch": 21.143497757847534, + "grad_norm": 0.7551945447921753, + "learning_rate": 0.0001, + "loss": 60.6674, + "step": 13130 + }, + { + "epoch": 21.15962110142591, + "grad_norm": 0.6225672960281372, + "learning_rate": 0.0001, + "loss": 60.4107, + "step": 13140 + }, + { + "epoch": 21.175744445004284, + "grad_norm": 0.6100388765335083, + "learning_rate": 0.0001, + "loss": 58.8803, + "step": 13150 + }, + { + "epoch": 21.19186778858266, + "grad_norm": 0.669247567653656, + "learning_rate": 0.0001, + "loss": 60.4534, + "step": 13160 + }, + { + "epoch": 21.20799113216103, + "grad_norm": 0.8899680972099304, + "learning_rate": 0.0001, + "loss": 59.3081, + "step": 13170 + }, + { + "epoch": 21.224114475739405, + "grad_norm": 0.6804965138435364, + "learning_rate": 0.0001, + "loss": 58.1406, + "step": 13180 + }, + { + "epoch": 21.24023781931778, + "grad_norm": 0.7442401051521301, + "learning_rate": 0.0001, + "loss": 61.9136, + "step": 13190 + }, + { + "epoch": 21.256361162896155, + "grad_norm": 0.6071797013282776, + "learning_rate": 0.0001, + "loss": 59.7979, + "step": 13200 + }, + { + "epoch": 21.27248450647453, + "grad_norm": 0.6128804683685303, + "learning_rate": 0.0001, + "loss": 58.802, + "step": 13210 + }, + { + "epoch": 21.288607850052905, + "grad_norm": 0.6430484652519226, + "learning_rate": 0.0001, + "loss": 59.9492, + "step": 13220 + }, + { + "epoch": 21.30473119363128, + "grad_norm": 0.6414241194725037, + "learning_rate": 0.0001, + "loss": 60.699, + "step": 13230 + }, + { + "epoch": 21.320854537209655, + "grad_norm": 0.7222250699996948, + "learning_rate": 0.0001, + "loss": 58.5909, + "step": 13240 + }, + { + "epoch": 21.33697788078803, + "grad_norm": 1.0913405418395996, + "learning_rate": 0.0001, + "loss": 59.9657, + "step": 13250 + }, + { + "epoch": 21.3531012243664, + "grad_norm": 0.8245854377746582, + "learning_rate": 0.0001, + "loss": 60.2276, + "step": 13260 + }, + { + "epoch": 21.369224567944777, + "grad_norm": 0.7204747200012207, + "learning_rate": 0.0001, + "loss": 60.1854, + "step": 13270 + }, + { + "epoch": 21.38534791152315, + "grad_norm": 0.5877585411071777, + "learning_rate": 0.0001, + "loss": 60.8509, + "step": 13280 + }, + { + "epoch": 21.401471255101526, + "grad_norm": 0.906605064868927, + "learning_rate": 0.0001, + "loss": 57.9792, + "step": 13290 + }, + { + "epoch": 21.4175945986799, + "grad_norm": 0.910301923751831, + "learning_rate": 0.0001, + "loss": 58.986, + "step": 13300 + }, + { + "epoch": 21.433717942258276, + "grad_norm": 0.7098278403282166, + "learning_rate": 0.0001, + "loss": 60.6727, + "step": 13310 + }, + { + "epoch": 21.44984128583665, + "grad_norm": 0.842569887638092, + "learning_rate": 0.0001, + "loss": 59.3789, + "step": 13320 + }, + { + "epoch": 21.465964629415026, + "grad_norm": 0.7320014834403992, + "learning_rate": 0.0001, + "loss": 60.7055, + "step": 13330 + }, + { + "epoch": 21.482087972993398, + "grad_norm": 0.7192698121070862, + "learning_rate": 0.0001, + "loss": 59.7494, + "step": 13340 + }, + { + "epoch": 21.498211316571773, + "grad_norm": 0.6317388415336609, + "learning_rate": 0.0001, + "loss": 59.2083, + "step": 13350 + }, + { + "epoch": 21.514334660150148, + "grad_norm": 0.7012689709663391, + "learning_rate": 0.0001, + "loss": 58.6733, + "step": 13360 + }, + { + "epoch": 21.530458003728523, + "grad_norm": 0.8148412704467773, + "learning_rate": 0.0001, + "loss": 60.6708, + "step": 13370 + }, + { + "epoch": 21.546581347306898, + "grad_norm": 0.6740885376930237, + "learning_rate": 0.0001, + "loss": 60.2317, + "step": 13380 + }, + { + "epoch": 21.562704690885273, + "grad_norm": 0.6835291981697083, + "learning_rate": 0.0001, + "loss": 60.1807, + "step": 13390 + }, + { + "epoch": 21.578828034463648, + "grad_norm": 0.9604494571685791, + "learning_rate": 0.0001, + "loss": 60.811, + "step": 13400 + }, + { + "epoch": 21.594951378042023, + "grad_norm": 0.765917956829071, + "learning_rate": 0.0001, + "loss": 60.4539, + "step": 13410 + }, + { + "epoch": 21.611074721620398, + "grad_norm": 0.7830843329429626, + "learning_rate": 0.0001, + "loss": 59.2483, + "step": 13420 + }, + { + "epoch": 21.62719806519877, + "grad_norm": 0.6538785099983215, + "learning_rate": 0.0001, + "loss": 60.4062, + "step": 13430 + }, + { + "epoch": 21.643321408777144, + "grad_norm": 0.7341951727867126, + "learning_rate": 0.0001, + "loss": 59.3654, + "step": 13440 + }, + { + "epoch": 21.65944475235552, + "grad_norm": 0.684467077255249, + "learning_rate": 0.0001, + "loss": 58.9634, + "step": 13450 + }, + { + "epoch": 21.675568095933894, + "grad_norm": 0.6942781805992126, + "learning_rate": 0.0001, + "loss": 60.3677, + "step": 13460 + }, + { + "epoch": 21.69169143951227, + "grad_norm": 0.7780037522315979, + "learning_rate": 0.0001, + "loss": 61.6705, + "step": 13470 + }, + { + "epoch": 21.707814783090644, + "grad_norm": 0.8363171219825745, + "learning_rate": 0.0001, + "loss": 59.7802, + "step": 13480 + }, + { + "epoch": 21.72393812666902, + "grad_norm": 0.7392274737358093, + "learning_rate": 0.0001, + "loss": 59.8264, + "step": 13490 + }, + { + "epoch": 21.740061470247394, + "grad_norm": 0.6051466464996338, + "learning_rate": 0.0001, + "loss": 59.6685, + "step": 13500 + }, + { + "epoch": 21.75618481382577, + "grad_norm": 0.7772730588912964, + "learning_rate": 0.0001, + "loss": 58.261, + "step": 13510 + }, + { + "epoch": 21.77230815740414, + "grad_norm": 0.7481459379196167, + "learning_rate": 0.0001, + "loss": 59.3212, + "step": 13520 + }, + { + "epoch": 21.788431500982515, + "grad_norm": 0.8020729422569275, + "learning_rate": 0.0001, + "loss": 59.0113, + "step": 13530 + }, + { + "epoch": 21.80455484456089, + "grad_norm": 0.8504058122634888, + "learning_rate": 0.0001, + "loss": 59.7243, + "step": 13540 + }, + { + "epoch": 21.820678188139265, + "grad_norm": 0.7256322503089905, + "learning_rate": 0.0001, + "loss": 60.7478, + "step": 13550 + }, + { + "epoch": 21.83680153171764, + "grad_norm": 0.6632773876190186, + "learning_rate": 0.0001, + "loss": 59.2331, + "step": 13560 + }, + { + "epoch": 21.852924875296015, + "grad_norm": 0.6225188970565796, + "learning_rate": 0.0001, + "loss": 59.8263, + "step": 13570 + }, + { + "epoch": 21.86904821887439, + "grad_norm": 0.6302409768104553, + "learning_rate": 0.0001, + "loss": 60.8574, + "step": 13580 + }, + { + "epoch": 21.885171562452765, + "grad_norm": 0.7654368877410889, + "learning_rate": 0.0001, + "loss": 59.8633, + "step": 13590 + }, + { + "epoch": 21.90129490603114, + "grad_norm": 1.227220892906189, + "learning_rate": 0.0001, + "loss": 60.7098, + "step": 13600 + }, + { + "epoch": 21.91741824960951, + "grad_norm": 0.6679421067237854, + "learning_rate": 0.0001, + "loss": 59.0663, + "step": 13610 + }, + { + "epoch": 21.933541593187886, + "grad_norm": 0.8533850312232971, + "learning_rate": 0.0001, + "loss": 60.8597, + "step": 13620 + }, + { + "epoch": 21.94966493676626, + "grad_norm": 0.6417121887207031, + "learning_rate": 0.0001, + "loss": 60.1464, + "step": 13630 + }, + { + "epoch": 21.965788280344636, + "grad_norm": 0.679053544998169, + "learning_rate": 0.0001, + "loss": 60.5066, + "step": 13640 + }, + { + "epoch": 21.98191162392301, + "grad_norm": 0.6914607286453247, + "learning_rate": 0.0001, + "loss": 59.6815, + "step": 13650 + }, + { + "epoch": 21.998034967501386, + "grad_norm": 0.5661413669586182, + "learning_rate": 0.0001, + "loss": 61.6076, + "step": 13660 + }, + { + "epoch": 22.0128986748627, + "grad_norm": 1.4387016296386719, + "learning_rate": 0.0001, + "loss": 55.0652, + "step": 13670 + }, + { + "epoch": 22.029022018441076, + "grad_norm": 0.7774295210838318, + "learning_rate": 0.0001, + "loss": 61.4395, + "step": 13680 + }, + { + "epoch": 22.045145362019447, + "grad_norm": 0.7438349723815918, + "learning_rate": 0.0001, + "loss": 60.998, + "step": 13690 + }, + { + "epoch": 22.061268705597822, + "grad_norm": 0.7506164312362671, + "learning_rate": 0.0001, + "loss": 60.6929, + "step": 13700 + }, + { + "epoch": 22.077392049176197, + "grad_norm": 0.6262985467910767, + "learning_rate": 0.0001, + "loss": 59.4921, + "step": 13710 + }, + { + "epoch": 22.093515392754572, + "grad_norm": 1.826015591621399, + "learning_rate": 0.0001, + "loss": 61.3791, + "step": 13720 + }, + { + "epoch": 22.109638736332947, + "grad_norm": 0.6410622596740723, + "learning_rate": 0.0001, + "loss": 62.5485, + "step": 13730 + }, + { + "epoch": 22.125762079911322, + "grad_norm": 0.7042442560195923, + "learning_rate": 0.0001, + "loss": 60.0654, + "step": 13740 + }, + { + "epoch": 22.141885423489697, + "grad_norm": 0.741142988204956, + "learning_rate": 0.0001, + "loss": 59.9831, + "step": 13750 + }, + { + "epoch": 22.158008767068072, + "grad_norm": 0.836445152759552, + "learning_rate": 0.0001, + "loss": 59.6805, + "step": 13760 + }, + { + "epoch": 22.174132110646447, + "grad_norm": 0.8353415727615356, + "learning_rate": 0.0001, + "loss": 60.1912, + "step": 13770 + }, + { + "epoch": 22.19025545422482, + "grad_norm": 0.6416126489639282, + "learning_rate": 0.0001, + "loss": 59.5518, + "step": 13780 + }, + { + "epoch": 22.206378797803193, + "grad_norm": 0.8702544569969177, + "learning_rate": 0.0001, + "loss": 57.7428, + "step": 13790 + }, + { + "epoch": 22.222502141381568, + "grad_norm": 0.6681593656539917, + "learning_rate": 0.0001, + "loss": 59.9499, + "step": 13800 + }, + { + "epoch": 22.238625484959943, + "grad_norm": 0.6185410022735596, + "learning_rate": 0.0001, + "loss": 59.8062, + "step": 13810 + }, + { + "epoch": 22.254748828538318, + "grad_norm": 0.7695328593254089, + "learning_rate": 0.0001, + "loss": 61.3512, + "step": 13820 + }, + { + "epoch": 22.270872172116693, + "grad_norm": 0.6550581455230713, + "learning_rate": 0.0001, + "loss": 59.3379, + "step": 13830 + }, + { + "epoch": 22.286995515695068, + "grad_norm": 0.7764340043067932, + "learning_rate": 0.0001, + "loss": 59.984, + "step": 13840 + }, + { + "epoch": 22.303118859273443, + "grad_norm": 0.6377913355827332, + "learning_rate": 0.0001, + "loss": 59.1862, + "step": 13850 + }, + { + "epoch": 22.319242202851818, + "grad_norm": 0.7837334275245667, + "learning_rate": 0.0001, + "loss": 60.8087, + "step": 13860 + }, + { + "epoch": 22.33536554643019, + "grad_norm": 0.7966547012329102, + "learning_rate": 0.0001, + "loss": 59.9109, + "step": 13870 + }, + { + "epoch": 22.351488890008564, + "grad_norm": 0.6303540468215942, + "learning_rate": 0.0001, + "loss": 61.0486, + "step": 13880 + }, + { + "epoch": 22.36761223358694, + "grad_norm": 0.6628289818763733, + "learning_rate": 0.0001, + "loss": 61.7723, + "step": 13890 + }, + { + "epoch": 22.383735577165314, + "grad_norm": 1.2156575918197632, + "learning_rate": 0.0001, + "loss": 58.0992, + "step": 13900 + }, + { + "epoch": 22.39985892074369, + "grad_norm": 1.0548591613769531, + "learning_rate": 0.0001, + "loss": 58.9139, + "step": 13910 + }, + { + "epoch": 22.415982264322064, + "grad_norm": 0.6389120817184448, + "learning_rate": 0.0001, + "loss": 59.1494, + "step": 13920 + }, + { + "epoch": 22.43210560790044, + "grad_norm": 0.6384537220001221, + "learning_rate": 0.0001, + "loss": 59.7135, + "step": 13930 + }, + { + "epoch": 22.448228951478814, + "grad_norm": 0.7434266209602356, + "learning_rate": 0.0001, + "loss": 60.7941, + "step": 13940 + }, + { + "epoch": 22.46435229505719, + "grad_norm": 0.7207310199737549, + "learning_rate": 0.0001, + "loss": 57.9466, + "step": 13950 + }, + { + "epoch": 22.48047563863556, + "grad_norm": 0.9680810570716858, + "learning_rate": 0.0001, + "loss": 57.5014, + "step": 13960 + }, + { + "epoch": 22.496598982213936, + "grad_norm": 0.5354718565940857, + "learning_rate": 0.0001, + "loss": 59.8771, + "step": 13970 + }, + { + "epoch": 22.51272232579231, + "grad_norm": 1.1624388694763184, + "learning_rate": 0.0001, + "loss": 60.2305, + "step": 13980 + }, + { + "epoch": 22.528845669370686, + "grad_norm": 0.6650974154472351, + "learning_rate": 0.0001, + "loss": 58.1649, + "step": 13990 + }, + { + "epoch": 22.54496901294906, + "grad_norm": 0.6151170134544373, + "learning_rate": 0.0001, + "loss": 60.4392, + "step": 14000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 33, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.712197053892285e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ade09d5233289ba4bc5076af39b8652078582dee --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016a4e9b5d71a1947317d3801b00fe5d0604edbef1f981d7b6c92f219131717c +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-14000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/latest new file mode 100644 index 0000000000000000000000000000000000000000..76800d601529826c2ec8d7453b614b895cf74f2a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/latest @@ -0,0 +1 @@ +global_step16000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b0662b91417e869365cd8cf8af22373de27dd18a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69a0c2e7a1dfd07d11f9ebe4093899aa4ae8cdddf1baf24180f526282714572d +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..574d63396e0368e2ffecaf35dda9c9d00f8ab0ea --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48110c834d4d6ef7bf27b8b49e7bd0fd8c0b824970ddb4c38690685c6a5feab4 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..06787f214640b0417e5813f866c361e5c17daf19 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51239f5468f8d0370346c49e1394c8c6eefad67b953ef6bc968857ae35985e5e +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..f9f8cd1fd4957185d8778eec33f2e502eb0f5375 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e970a1468ff581a08922ca09ec203e253134687e184f5f430b3f7b4b42097b80 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..bde7433c0c2d7c5642452a2c7951a5627dfc8286 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b244b8cef4b08b6673d7b79a074abcff2c1a091d2f4a4a2be18783642a80d1b3 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..ec158b41ff71281b35edf528920ca61a586f67a3 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db4e502ce7278ff21b948bc87c2da94a900cbdb86f3edc4ae0830d20332d612e +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..78006ce2fe2b56e5e1d74175545bd7559bc0bb80 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63b3a8abb43eb8c2463a035d963e620a05ea1a1dffeb66475d2ffcaa70457a24 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..c9f29f7b37b227da321e7f85059c698f82fe3e9e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d5e23632f52995d38771e2258aa3741638b54c26b6060ac8afe14726f7e8357 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..0b2311c9864661d5e724f54ece2c3621fd28cab7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d605c300f639f185db21de11682aa68ae1d1197670bbef25d57744bc266f0cf +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..40b93c36797b53b455f19ea4c1bc67022918d62f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a427e77702ae9121e994cdda9f4f553997fc61824162d506f30f54b748c208a5 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..021b94c065ce42ad509d48f4424b40215b62a36c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/trainer_state.json @@ -0,0 +1,11234 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 25.76585881997279, + "eval_steps": 2000, + "global_step": 16000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.016123343578374565, + "grad_norm": 48.11856460571289, + "learning_rate": 0.0001, + "loss": 299.1088, + "step": 10 + }, + { + "epoch": 0.03224668715674913, + "grad_norm": 9.07711410522461, + "learning_rate": 0.0001, + "loss": 234.0888, + "step": 20 + }, + { + "epoch": 0.0483700307351237, + "grad_norm": 3.5935330390930176, + "learning_rate": 0.0001, + "loss": 231.4619, + "step": 30 + }, + { + "epoch": 0.06449337431349826, + "grad_norm": 3.5784752368927, + "learning_rate": 0.0001, + "loss": 229.7082, + "step": 40 + }, + { + "epoch": 0.08061671789187283, + "grad_norm": 3.5757956504821777, + "learning_rate": 0.0001, + "loss": 230.1062, + "step": 50 + }, + { + "epoch": 0.0967400614702474, + "grad_norm": 3.8763978481292725, + "learning_rate": 0.0001, + "loss": 229.1914, + "step": 60 + }, + { + "epoch": 0.11286340504862195, + "grad_norm": 3.006988525390625, + "learning_rate": 0.0001, + "loss": 229.6482, + "step": 70 + }, + { + "epoch": 0.12898674862699652, + "grad_norm": 2.3527095317840576, + "learning_rate": 0.0001, + "loss": 228.1141, + "step": 80 + }, + { + "epoch": 0.14511009220537108, + "grad_norm": 5.353086471557617, + "learning_rate": 0.0001, + "loss": 228.1448, + "step": 90 + }, + { + "epoch": 0.16123343578374566, + "grad_norm": 2.251854419708252, + "learning_rate": 0.0001, + "loss": 228.3897, + "step": 100 + }, + { + "epoch": 0.17735677936212021, + "grad_norm": 1.5789915323257446, + "learning_rate": 0.0001, + "loss": 227.7662, + "step": 110 + }, + { + "epoch": 0.1934801229404948, + "grad_norm": 3.7646398544311523, + "learning_rate": 0.0001, + "loss": 223.6411, + "step": 120 + }, + { + "epoch": 0.20960346651886935, + "grad_norm": 3.444000720977783, + "learning_rate": 0.0001, + "loss": 221.2073, + "step": 130 + }, + { + "epoch": 0.2257268100972439, + "grad_norm": 2.1871185302734375, + "learning_rate": 0.0001, + "loss": 215.9944, + "step": 140 + }, + { + "epoch": 0.2418501536756185, + "grad_norm": 2.266244411468506, + "learning_rate": 0.0001, + "loss": 212.3066, + "step": 150 + }, + { + "epoch": 0.25797349725399304, + "grad_norm": 2.043832778930664, + "learning_rate": 0.0001, + "loss": 208.1343, + "step": 160 + }, + { + "epoch": 0.2740968408323676, + "grad_norm": 2.2641494274139404, + "learning_rate": 0.0001, + "loss": 203.1485, + "step": 170 + }, + { + "epoch": 0.29022018441074215, + "grad_norm": 2.9727790355682373, + "learning_rate": 0.0001, + "loss": 198.181, + "step": 180 + }, + { + "epoch": 0.30634352798911674, + "grad_norm": 2.751523494720459, + "learning_rate": 0.0001, + "loss": 194.8422, + "step": 190 + }, + { + "epoch": 0.3224668715674913, + "grad_norm": 1.9561562538146973, + "learning_rate": 0.0001, + "loss": 189.9401, + "step": 200 + }, + { + "epoch": 0.33859021514586585, + "grad_norm": 2.3264107704162598, + "learning_rate": 0.0001, + "loss": 186.1309, + "step": 210 + }, + { + "epoch": 0.35471355872424043, + "grad_norm": 5.212426662445068, + "learning_rate": 0.0001, + "loss": 182.2255, + "step": 220 + }, + { + "epoch": 0.370836902302615, + "grad_norm": 2.488032102584839, + "learning_rate": 0.0001, + "loss": 181.0103, + "step": 230 + }, + { + "epoch": 0.3869602458809896, + "grad_norm": 2.6173202991485596, + "learning_rate": 0.0001, + "loss": 177.6162, + "step": 240 + }, + { + "epoch": 0.4030835894593641, + "grad_norm": 2.6245291233062744, + "learning_rate": 0.0001, + "loss": 173.0431, + "step": 250 + }, + { + "epoch": 0.4192069330377387, + "grad_norm": 5.356006145477295, + "learning_rate": 0.0001, + "loss": 172.2371, + "step": 260 + }, + { + "epoch": 0.4353302766161133, + "grad_norm": 2.0453829765319824, + "learning_rate": 0.0001, + "loss": 169.7781, + "step": 270 + }, + { + "epoch": 0.4514536201944878, + "grad_norm": 1.7002843618392944, + "learning_rate": 0.0001, + "loss": 166.9564, + "step": 280 + }, + { + "epoch": 0.4675769637728624, + "grad_norm": 2.181410312652588, + "learning_rate": 0.0001, + "loss": 166.0094, + "step": 290 + }, + { + "epoch": 0.483700307351237, + "grad_norm": 2.2948265075683594, + "learning_rate": 0.0001, + "loss": 162.4388, + "step": 300 + }, + { + "epoch": 0.4998236509296115, + "grad_norm": 3.0272789001464844, + "learning_rate": 0.0001, + "loss": 160.3446, + "step": 310 + }, + { + "epoch": 0.5159469945079861, + "grad_norm": 1.948184847831726, + "learning_rate": 0.0001, + "loss": 158.488, + "step": 320 + }, + { + "epoch": 0.5320703380863606, + "grad_norm": 1.7981538772583008, + "learning_rate": 0.0001, + "loss": 156.2107, + "step": 330 + }, + { + "epoch": 0.5481936816647353, + "grad_norm": 2.049323797225952, + "learning_rate": 0.0001, + "loss": 155.5434, + "step": 340 + }, + { + "epoch": 0.5643170252431098, + "grad_norm": 1.9415442943572998, + "learning_rate": 0.0001, + "loss": 153.8682, + "step": 350 + }, + { + "epoch": 0.5804403688214843, + "grad_norm": 2.2168045043945312, + "learning_rate": 0.0001, + "loss": 150.4087, + "step": 360 + }, + { + "epoch": 0.596563712399859, + "grad_norm": 1.510750651359558, + "learning_rate": 0.0001, + "loss": 151.7655, + "step": 370 + }, + { + "epoch": 0.6126870559782335, + "grad_norm": 2.7469751834869385, + "learning_rate": 0.0001, + "loss": 147.4665, + "step": 380 + }, + { + "epoch": 0.628810399556608, + "grad_norm": 2.0436975955963135, + "learning_rate": 0.0001, + "loss": 147.4405, + "step": 390 + }, + { + "epoch": 0.6449337431349826, + "grad_norm": 1.8219181299209595, + "learning_rate": 0.0001, + "loss": 146.6332, + "step": 400 + }, + { + "epoch": 0.6610570867133572, + "grad_norm": 1.948431134223938, + "learning_rate": 0.0001, + "loss": 143.3298, + "step": 410 + }, + { + "epoch": 0.6771804302917317, + "grad_norm": 1.8887908458709717, + "learning_rate": 0.0001, + "loss": 140.5058, + "step": 420 + }, + { + "epoch": 0.6933037738701063, + "grad_norm": 1.8739320039749146, + "learning_rate": 0.0001, + "loss": 137.6334, + "step": 430 + }, + { + "epoch": 0.7094271174484809, + "grad_norm": 2.057689666748047, + "learning_rate": 0.0001, + "loss": 135.0343, + "step": 440 + }, + { + "epoch": 0.7255504610268555, + "grad_norm": 1.6926478147506714, + "learning_rate": 0.0001, + "loss": 132.8766, + "step": 450 + }, + { + "epoch": 0.74167380460523, + "grad_norm": 1.5039501190185547, + "learning_rate": 0.0001, + "loss": 131.6774, + "step": 460 + }, + { + "epoch": 0.7577971481836046, + "grad_norm": 1.6903547048568726, + "learning_rate": 0.0001, + "loss": 129.3116, + "step": 470 + }, + { + "epoch": 0.7739204917619792, + "grad_norm": 3.7062244415283203, + "learning_rate": 0.0001, + "loss": 127.9314, + "step": 480 + }, + { + "epoch": 0.7900438353403537, + "grad_norm": 2.0586934089660645, + "learning_rate": 0.0001, + "loss": 126.6398, + "step": 490 + }, + { + "epoch": 0.8061671789187282, + "grad_norm": 1.7598215341567993, + "learning_rate": 0.0001, + "loss": 125.1501, + "step": 500 + }, + { + "epoch": 0.8222905224971029, + "grad_norm": 1.5992189645767212, + "learning_rate": 0.0001, + "loss": 124.7039, + "step": 510 + }, + { + "epoch": 0.8384138660754774, + "grad_norm": 2.8735458850860596, + "learning_rate": 0.0001, + "loss": 122.1619, + "step": 520 + }, + { + "epoch": 0.8545372096538519, + "grad_norm": 1.867098093032837, + "learning_rate": 0.0001, + "loss": 122.4506, + "step": 530 + }, + { + "epoch": 0.8706605532322266, + "grad_norm": 2.2088418006896973, + "learning_rate": 0.0001, + "loss": 122.1047, + "step": 540 + }, + { + "epoch": 0.8867838968106011, + "grad_norm": 1.642657995223999, + "learning_rate": 0.0001, + "loss": 120.1141, + "step": 550 + }, + { + "epoch": 0.9029072403889756, + "grad_norm": 1.5567114353179932, + "learning_rate": 0.0001, + "loss": 118.532, + "step": 560 + }, + { + "epoch": 0.9190305839673503, + "grad_norm": 1.667843222618103, + "learning_rate": 0.0001, + "loss": 118.0734, + "step": 570 + }, + { + "epoch": 0.9351539275457248, + "grad_norm": 1.5304442644119263, + "learning_rate": 0.0001, + "loss": 114.4925, + "step": 580 + }, + { + "epoch": 0.9512772711240993, + "grad_norm": 1.6920255422592163, + "learning_rate": 0.0001, + "loss": 114.0729, + "step": 590 + }, + { + "epoch": 0.967400614702474, + "grad_norm": 1.6313318014144897, + "learning_rate": 0.0001, + "loss": 114.2879, + "step": 600 + }, + { + "epoch": 0.9835239582808485, + "grad_norm": 1.9033657312393188, + "learning_rate": 0.0001, + "loss": 110.5299, + "step": 610 + }, + { + "epoch": 0.999647301859223, + "grad_norm": 1.9010754823684692, + "learning_rate": 0.0001, + "loss": 109.1145, + "step": 620 + }, + { + "epoch": 1.0145110092205372, + "grad_norm": 3.5390989780426025, + "learning_rate": 0.0001, + "loss": 101.0258, + "step": 630 + }, + { + "epoch": 1.0306343527989117, + "grad_norm": 2.2925171852111816, + "learning_rate": 0.0001, + "loss": 107.3693, + "step": 640 + }, + { + "epoch": 1.0467576963772862, + "grad_norm": 2.329108238220215, + "learning_rate": 0.0001, + "loss": 104.3778, + "step": 650 + }, + { + "epoch": 1.0628810399556607, + "grad_norm": 1.7325661182403564, + "learning_rate": 0.0001, + "loss": 102.8409, + "step": 660 + }, + { + "epoch": 1.0790043835340353, + "grad_norm": 1.989586353302002, + "learning_rate": 0.0001, + "loss": 104.2577, + "step": 670 + }, + { + "epoch": 1.09512772711241, + "grad_norm": 1.900484323501587, + "learning_rate": 0.0001, + "loss": 102.3051, + "step": 680 + }, + { + "epoch": 1.1112510706907845, + "grad_norm": 2.015859842300415, + "learning_rate": 0.0001, + "loss": 100.893, + "step": 690 + }, + { + "epoch": 1.127374414269159, + "grad_norm": 1.7695508003234863, + "learning_rate": 0.0001, + "loss": 99.215, + "step": 700 + }, + { + "epoch": 1.1434977578475336, + "grad_norm": 2.025923490524292, + "learning_rate": 0.0001, + "loss": 98.8768, + "step": 710 + }, + { + "epoch": 1.1596211014259081, + "grad_norm": 2.946708917617798, + "learning_rate": 0.0001, + "loss": 96.5588, + "step": 720 + }, + { + "epoch": 1.1757444450042827, + "grad_norm": 2.4511258602142334, + "learning_rate": 0.0001, + "loss": 95.0545, + "step": 730 + }, + { + "epoch": 1.1918677885826574, + "grad_norm": 2.3094682693481445, + "learning_rate": 0.0001, + "loss": 97.004, + "step": 740 + }, + { + "epoch": 1.207991132161032, + "grad_norm": 2.1313347816467285, + "learning_rate": 0.0001, + "loss": 94.8819, + "step": 750 + }, + { + "epoch": 1.2241144757394065, + "grad_norm": 1.614182710647583, + "learning_rate": 0.0001, + "loss": 95.5339, + "step": 760 + }, + { + "epoch": 1.240237819317781, + "grad_norm": 2.029953956604004, + "learning_rate": 0.0001, + "loss": 93.7303, + "step": 770 + }, + { + "epoch": 1.2563611628961555, + "grad_norm": 1.7871663570404053, + "learning_rate": 0.0001, + "loss": 93.8354, + "step": 780 + }, + { + "epoch": 1.2724845064745303, + "grad_norm": 1.72455632686615, + "learning_rate": 0.0001, + "loss": 94.7348, + "step": 790 + }, + { + "epoch": 1.2886078500529048, + "grad_norm": 1.9776639938354492, + "learning_rate": 0.0001, + "loss": 93.5957, + "step": 800 + }, + { + "epoch": 1.3047311936312793, + "grad_norm": 1.7187533378601074, + "learning_rate": 0.0001, + "loss": 92.9583, + "step": 810 + }, + { + "epoch": 1.3208545372096538, + "grad_norm": 1.5949162244796753, + "learning_rate": 0.0001, + "loss": 93.3088, + "step": 820 + }, + { + "epoch": 1.3369778807880284, + "grad_norm": 1.4616206884384155, + "learning_rate": 0.0001, + "loss": 93.2125, + "step": 830 + }, + { + "epoch": 1.353101224366403, + "grad_norm": 2.832163095474243, + "learning_rate": 0.0001, + "loss": 90.3301, + "step": 840 + }, + { + "epoch": 1.3692245679447774, + "grad_norm": 2.174501419067383, + "learning_rate": 0.0001, + "loss": 91.6988, + "step": 850 + }, + { + "epoch": 1.3853479115231522, + "grad_norm": 1.2156339883804321, + "learning_rate": 0.0001, + "loss": 92.7419, + "step": 860 + }, + { + "epoch": 1.4014712551015267, + "grad_norm": 1.608261227607727, + "learning_rate": 0.0001, + "loss": 89.8255, + "step": 870 + }, + { + "epoch": 1.4175945986799012, + "grad_norm": 1.6671923398971558, + "learning_rate": 0.0001, + "loss": 90.9683, + "step": 880 + }, + { + "epoch": 1.4337179422582758, + "grad_norm": 1.2346433401107788, + "learning_rate": 0.0001, + "loss": 90.6092, + "step": 890 + }, + { + "epoch": 1.4498412858366505, + "grad_norm": 1.561085820198059, + "learning_rate": 0.0001, + "loss": 88.0842, + "step": 900 + }, + { + "epoch": 1.465964629415025, + "grad_norm": 1.5475201606750488, + "learning_rate": 0.0001, + "loss": 89.8786, + "step": 910 + }, + { + "epoch": 1.4820879729933996, + "grad_norm": 1.6044800281524658, + "learning_rate": 0.0001, + "loss": 88.0364, + "step": 920 + }, + { + "epoch": 1.498211316571774, + "grad_norm": 1.4081908464431763, + "learning_rate": 0.0001, + "loss": 88.8799, + "step": 930 + }, + { + "epoch": 1.5143346601501486, + "grad_norm": 2.341053009033203, + "learning_rate": 0.0001, + "loss": 88.2023, + "step": 940 + }, + { + "epoch": 1.5304580037285231, + "grad_norm": 1.3976799249649048, + "learning_rate": 0.0001, + "loss": 86.6493, + "step": 950 + }, + { + "epoch": 1.5465813473068977, + "grad_norm": 1.2485483884811401, + "learning_rate": 0.0001, + "loss": 88.9224, + "step": 960 + }, + { + "epoch": 1.5627046908852722, + "grad_norm": 1.6824235916137695, + "learning_rate": 0.0001, + "loss": 89.6484, + "step": 970 + }, + { + "epoch": 1.578828034463647, + "grad_norm": 1.64790940284729, + "learning_rate": 0.0001, + "loss": 87.2479, + "step": 980 + }, + { + "epoch": 1.5949513780420215, + "grad_norm": 1.2093443870544434, + "learning_rate": 0.0001, + "loss": 86.8241, + "step": 990 + }, + { + "epoch": 1.611074721620396, + "grad_norm": 1.5306812524795532, + "learning_rate": 0.0001, + "loss": 88.0006, + "step": 1000 + }, + { + "epoch": 1.6271980651987707, + "grad_norm": 1.2465763092041016, + "learning_rate": 0.0001, + "loss": 86.8143, + "step": 1010 + }, + { + "epoch": 1.6433214087771453, + "grad_norm": 1.5224069356918335, + "learning_rate": 0.0001, + "loss": 87.0102, + "step": 1020 + }, + { + "epoch": 1.6594447523555198, + "grad_norm": 1.312179684638977, + "learning_rate": 0.0001, + "loss": 88.7395, + "step": 1030 + }, + { + "epoch": 1.6755680959338943, + "grad_norm": 1.2343158721923828, + "learning_rate": 0.0001, + "loss": 87.2868, + "step": 1040 + }, + { + "epoch": 1.6916914395122689, + "grad_norm": 1.2778265476226807, + "learning_rate": 0.0001, + "loss": 87.043, + "step": 1050 + }, + { + "epoch": 1.7078147830906434, + "grad_norm": 1.2799627780914307, + "learning_rate": 0.0001, + "loss": 87.7843, + "step": 1060 + }, + { + "epoch": 1.723938126669018, + "grad_norm": 1.3688952922821045, + "learning_rate": 0.0001, + "loss": 85.8985, + "step": 1070 + }, + { + "epoch": 1.7400614702473924, + "grad_norm": 1.4317796230316162, + "learning_rate": 0.0001, + "loss": 85.5408, + "step": 1080 + }, + { + "epoch": 1.756184813825767, + "grad_norm": 1.164801001548767, + "learning_rate": 0.0001, + "loss": 85.8484, + "step": 1090 + }, + { + "epoch": 1.7723081574041417, + "grad_norm": 1.3929173946380615, + "learning_rate": 0.0001, + "loss": 86.9031, + "step": 1100 + }, + { + "epoch": 1.7884315009825162, + "grad_norm": 2.116119861602783, + "learning_rate": 0.0001, + "loss": 86.5343, + "step": 1110 + }, + { + "epoch": 1.8045548445608908, + "grad_norm": 1.4117276668548584, + "learning_rate": 0.0001, + "loss": 85.2678, + "step": 1120 + }, + { + "epoch": 1.8206781881392655, + "grad_norm": 1.3213504552841187, + "learning_rate": 0.0001, + "loss": 85.8935, + "step": 1130 + }, + { + "epoch": 1.83680153171764, + "grad_norm": 1.3589770793914795, + "learning_rate": 0.0001, + "loss": 85.032, + "step": 1140 + }, + { + "epoch": 1.8529248752960146, + "grad_norm": 1.3336007595062256, + "learning_rate": 0.0001, + "loss": 84.0993, + "step": 1150 + }, + { + "epoch": 1.869048218874389, + "grad_norm": 1.1081451177597046, + "learning_rate": 0.0001, + "loss": 84.2225, + "step": 1160 + }, + { + "epoch": 1.8851715624527636, + "grad_norm": 1.482038974761963, + "learning_rate": 0.0001, + "loss": 82.1439, + "step": 1170 + }, + { + "epoch": 1.9012949060311382, + "grad_norm": 1.1571478843688965, + "learning_rate": 0.0001, + "loss": 82.5045, + "step": 1180 + }, + { + "epoch": 1.9174182496095127, + "grad_norm": 1.2665468454360962, + "learning_rate": 0.0001, + "loss": 82.8761, + "step": 1190 + }, + { + "epoch": 1.9335415931878872, + "grad_norm": 1.0771509408950806, + "learning_rate": 0.0001, + "loss": 81.5162, + "step": 1200 + }, + { + "epoch": 1.949664936766262, + "grad_norm": 1.2992299795150757, + "learning_rate": 0.0001, + "loss": 82.4038, + "step": 1210 + }, + { + "epoch": 1.9657882803446365, + "grad_norm": 1.2946668863296509, + "learning_rate": 0.0001, + "loss": 83.4837, + "step": 1220 + }, + { + "epoch": 1.981911623923011, + "grad_norm": 1.3505686521530151, + "learning_rate": 0.0001, + "loss": 81.8993, + "step": 1230 + }, + { + "epoch": 1.9980349675013858, + "grad_norm": 1.1247715950012207, + "learning_rate": 0.0001, + "loss": 82.9006, + "step": 1240 + }, + { + "epoch": 2.0128986748627, + "grad_norm": 1.5652127265930176, + "learning_rate": 0.0001, + "loss": 75.8347, + "step": 1250 + }, + { + "epoch": 2.0290220184410743, + "grad_norm": 7.311964988708496, + "learning_rate": 0.0001, + "loss": 80.5435, + "step": 1260 + }, + { + "epoch": 2.045145362019449, + "grad_norm": 1.2229893207550049, + "learning_rate": 0.0001, + "loss": 83.3277, + "step": 1270 + }, + { + "epoch": 2.0612687055978234, + "grad_norm": 1.1387693881988525, + "learning_rate": 0.0001, + "loss": 81.3252, + "step": 1280 + }, + { + "epoch": 2.077392049176198, + "grad_norm": 1.307201623916626, + "learning_rate": 0.0001, + "loss": 82.3311, + "step": 1290 + }, + { + "epoch": 2.0935153927545724, + "grad_norm": 1.1502896547317505, + "learning_rate": 0.0001, + "loss": 81.1682, + "step": 1300 + }, + { + "epoch": 2.109638736332947, + "grad_norm": 0.9291583299636841, + "learning_rate": 0.0001, + "loss": 81.7017, + "step": 1310 + }, + { + "epoch": 2.1257620799113215, + "grad_norm": 1.0334666967391968, + "learning_rate": 0.0001, + "loss": 81.718, + "step": 1320 + }, + { + "epoch": 2.141885423489696, + "grad_norm": 1.1099056005477905, + "learning_rate": 0.0001, + "loss": 81.2232, + "step": 1330 + }, + { + "epoch": 2.1580087670680705, + "grad_norm": 1.289478063583374, + "learning_rate": 0.0001, + "loss": 81.8736, + "step": 1340 + }, + { + "epoch": 2.174132110646445, + "grad_norm": 1.1634794473648071, + "learning_rate": 0.0001, + "loss": 79.9434, + "step": 1350 + }, + { + "epoch": 2.19025545422482, + "grad_norm": 0.9563440680503845, + "learning_rate": 0.0001, + "loss": 80.628, + "step": 1360 + }, + { + "epoch": 2.2063787978031946, + "grad_norm": 1.1268844604492188, + "learning_rate": 0.0001, + "loss": 82.1389, + "step": 1370 + }, + { + "epoch": 2.222502141381569, + "grad_norm": 1.1050443649291992, + "learning_rate": 0.0001, + "loss": 78.9023, + "step": 1380 + }, + { + "epoch": 2.2386254849599436, + "grad_norm": 1.1936142444610596, + "learning_rate": 0.0001, + "loss": 81.2118, + "step": 1390 + }, + { + "epoch": 2.254748828538318, + "grad_norm": 1.7825465202331543, + "learning_rate": 0.0001, + "loss": 80.8804, + "step": 1400 + }, + { + "epoch": 2.2708721721166927, + "grad_norm": 2.437251329421997, + "learning_rate": 0.0001, + "loss": 79.3807, + "step": 1410 + }, + { + "epoch": 2.286995515695067, + "grad_norm": 1.0604854822158813, + "learning_rate": 0.0001, + "loss": 79.5881, + "step": 1420 + }, + { + "epoch": 2.3031188592734417, + "grad_norm": 1.2285529375076294, + "learning_rate": 0.0001, + "loss": 79.5517, + "step": 1430 + }, + { + "epoch": 2.3192422028518163, + "grad_norm": 1.1916660070419312, + "learning_rate": 0.0001, + "loss": 80.5887, + "step": 1440 + }, + { + "epoch": 2.335365546430191, + "grad_norm": 1.224304437637329, + "learning_rate": 0.0001, + "loss": 79.8109, + "step": 1450 + }, + { + "epoch": 2.3514888900085653, + "grad_norm": 0.9228448867797852, + "learning_rate": 0.0001, + "loss": 79.2787, + "step": 1460 + }, + { + "epoch": 2.3676122335869403, + "grad_norm": 1.2085226774215698, + "learning_rate": 0.0001, + "loss": 78.4983, + "step": 1470 + }, + { + "epoch": 2.383735577165315, + "grad_norm": 2.424083948135376, + "learning_rate": 0.0001, + "loss": 79.8657, + "step": 1480 + }, + { + "epoch": 2.3998589207436893, + "grad_norm": 1.0277849435806274, + "learning_rate": 0.0001, + "loss": 80.6915, + "step": 1490 + }, + { + "epoch": 2.415982264322064, + "grad_norm": 1.0226075649261475, + "learning_rate": 0.0001, + "loss": 78.9751, + "step": 1500 + }, + { + "epoch": 2.4321056079004384, + "grad_norm": 0.8860388398170471, + "learning_rate": 0.0001, + "loss": 78.3915, + "step": 1510 + }, + { + "epoch": 2.448228951478813, + "grad_norm": 1.265637755393982, + "learning_rate": 0.0001, + "loss": 78.8737, + "step": 1520 + }, + { + "epoch": 2.4643522950571874, + "grad_norm": 1.2518186569213867, + "learning_rate": 0.0001, + "loss": 76.5169, + "step": 1530 + }, + { + "epoch": 2.480475638635562, + "grad_norm": 1.074277400970459, + "learning_rate": 0.0001, + "loss": 77.9156, + "step": 1540 + }, + { + "epoch": 2.4965989822139365, + "grad_norm": 1.0666711330413818, + "learning_rate": 0.0001, + "loss": 78.4103, + "step": 1550 + }, + { + "epoch": 2.512722325792311, + "grad_norm": 0.9492248892784119, + "learning_rate": 0.0001, + "loss": 76.7393, + "step": 1560 + }, + { + "epoch": 2.5288456693706856, + "grad_norm": 1.127238154411316, + "learning_rate": 0.0001, + "loss": 79.729, + "step": 1570 + }, + { + "epoch": 2.5449690129490605, + "grad_norm": 1.1520992517471313, + "learning_rate": 0.0001, + "loss": 78.5618, + "step": 1580 + }, + { + "epoch": 2.5610923565274346, + "grad_norm": 0.8632035255432129, + "learning_rate": 0.0001, + "loss": 79.032, + "step": 1590 + }, + { + "epoch": 2.5772157001058096, + "grad_norm": 1.0506329536437988, + "learning_rate": 0.0001, + "loss": 77.6204, + "step": 1600 + }, + { + "epoch": 2.593339043684184, + "grad_norm": 1.019895315170288, + "learning_rate": 0.0001, + "loss": 79.2634, + "step": 1610 + }, + { + "epoch": 2.6094623872625586, + "grad_norm": 1.1685571670532227, + "learning_rate": 0.0001, + "loss": 78.8371, + "step": 1620 + }, + { + "epoch": 2.625585730840933, + "grad_norm": 1.147546648979187, + "learning_rate": 0.0001, + "loss": 78.2113, + "step": 1630 + }, + { + "epoch": 2.6417090744193077, + "grad_norm": 1.7336841821670532, + "learning_rate": 0.0001, + "loss": 78.7866, + "step": 1640 + }, + { + "epoch": 2.657832417997682, + "grad_norm": 1.2150614261627197, + "learning_rate": 0.0001, + "loss": 78.5621, + "step": 1650 + }, + { + "epoch": 2.6739557615760567, + "grad_norm": 1.2257503271102905, + "learning_rate": 0.0001, + "loss": 77.5835, + "step": 1660 + }, + { + "epoch": 2.6900791051544313, + "grad_norm": 1.0375311374664307, + "learning_rate": 0.0001, + "loss": 77.7509, + "step": 1670 + }, + { + "epoch": 2.706202448732806, + "grad_norm": 1.2551004886627197, + "learning_rate": 0.0001, + "loss": 77.8615, + "step": 1680 + }, + { + "epoch": 2.7223257923111808, + "grad_norm": 1.0464223623275757, + "learning_rate": 0.0001, + "loss": 77.2837, + "step": 1690 + }, + { + "epoch": 2.738449135889555, + "grad_norm": 1.0254806280136108, + "learning_rate": 0.0001, + "loss": 76.9616, + "step": 1700 + }, + { + "epoch": 2.75457247946793, + "grad_norm": 1.1665682792663574, + "learning_rate": 0.0001, + "loss": 77.9108, + "step": 1710 + }, + { + "epoch": 2.7706958230463044, + "grad_norm": 1.1349453926086426, + "learning_rate": 0.0001, + "loss": 78.7636, + "step": 1720 + }, + { + "epoch": 2.786819166624679, + "grad_norm": 1.4961057901382446, + "learning_rate": 0.0001, + "loss": 76.3862, + "step": 1730 + }, + { + "epoch": 2.8029425102030534, + "grad_norm": 1.10721755027771, + "learning_rate": 0.0001, + "loss": 77.4105, + "step": 1740 + }, + { + "epoch": 2.819065853781428, + "grad_norm": 0.9878576397895813, + "learning_rate": 0.0001, + "loss": 78.4855, + "step": 1750 + }, + { + "epoch": 2.8351891973598025, + "grad_norm": 1.1722687482833862, + "learning_rate": 0.0001, + "loss": 75.1589, + "step": 1760 + }, + { + "epoch": 2.851312540938177, + "grad_norm": 1.6624362468719482, + "learning_rate": 0.0001, + "loss": 77.1647, + "step": 1770 + }, + { + "epoch": 2.8674358845165515, + "grad_norm": 0.9542295932769775, + "learning_rate": 0.0001, + "loss": 77.1656, + "step": 1780 + }, + { + "epoch": 2.883559228094926, + "grad_norm": 2.328542947769165, + "learning_rate": 0.0001, + "loss": 76.4686, + "step": 1790 + }, + { + "epoch": 2.899682571673301, + "grad_norm": 1.2532029151916504, + "learning_rate": 0.0001, + "loss": 76.1727, + "step": 1800 + }, + { + "epoch": 2.915805915251675, + "grad_norm": 0.9604716897010803, + "learning_rate": 0.0001, + "loss": 74.3132, + "step": 1810 + }, + { + "epoch": 2.93192925883005, + "grad_norm": 0.9182595610618591, + "learning_rate": 0.0001, + "loss": 77.8689, + "step": 1820 + }, + { + "epoch": 2.9480526024084246, + "grad_norm": 0.9623042941093445, + "learning_rate": 0.0001, + "loss": 77.7531, + "step": 1830 + }, + { + "epoch": 2.964175945986799, + "grad_norm": 1.0500119924545288, + "learning_rate": 0.0001, + "loss": 77.6007, + "step": 1840 + }, + { + "epoch": 2.9802992895651736, + "grad_norm": 1.0874103307724, + "learning_rate": 0.0001, + "loss": 77.042, + "step": 1850 + }, + { + "epoch": 2.996422633143548, + "grad_norm": 0.9955475330352783, + "learning_rate": 0.0001, + "loss": 75.468, + "step": 1860 + }, + { + "epoch": 3.011286340504862, + "grad_norm": 1.4281760454177856, + "learning_rate": 0.0001, + "loss": 69.5972, + "step": 1870 + }, + { + "epoch": 3.0274096840832367, + "grad_norm": 1.0292123556137085, + "learning_rate": 0.0001, + "loss": 76.0186, + "step": 1880 + }, + { + "epoch": 3.0435330276616113, + "grad_norm": 0.9535816311836243, + "learning_rate": 0.0001, + "loss": 76.0979, + "step": 1890 + }, + { + "epoch": 3.059656371239986, + "grad_norm": 0.9099674820899963, + "learning_rate": 0.0001, + "loss": 75.9641, + "step": 1900 + }, + { + "epoch": 3.0757797148183603, + "grad_norm": 0.9568380117416382, + "learning_rate": 0.0001, + "loss": 75.0656, + "step": 1910 + }, + { + "epoch": 3.091903058396735, + "grad_norm": 0.9701060652732849, + "learning_rate": 0.0001, + "loss": 75.1422, + "step": 1920 + }, + { + "epoch": 3.1080264019751094, + "grad_norm": 1.3879098892211914, + "learning_rate": 0.0001, + "loss": 76.0731, + "step": 1930 + }, + { + "epoch": 3.1241497455534843, + "grad_norm": 0.9782084226608276, + "learning_rate": 0.0001, + "loss": 75.9666, + "step": 1940 + }, + { + "epoch": 3.140273089131859, + "grad_norm": 1.0856964588165283, + "learning_rate": 0.0001, + "loss": 75.0217, + "step": 1950 + }, + { + "epoch": 3.1563964327102334, + "grad_norm": 0.8506444692611694, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 1960 + }, + { + "epoch": 3.172519776288608, + "grad_norm": 1.044980525970459, + "learning_rate": 0.0001, + "loss": 75.3471, + "step": 1970 + }, + { + "epoch": 3.1886431198669825, + "grad_norm": 0.8432291746139526, + "learning_rate": 0.0001, + "loss": 75.3563, + "step": 1980 + }, + { + "epoch": 3.204766463445357, + "grad_norm": 0.9651834964752197, + "learning_rate": 0.0001, + "loss": 76.3991, + "step": 1990 + }, + { + "epoch": 3.2208898070237315, + "grad_norm": 0.957943320274353, + "learning_rate": 0.0001, + "loss": 77.2613, + "step": 2000 + }, + { + "epoch": 3.237013150602106, + "grad_norm": 0.9495745301246643, + "learning_rate": 0.0001, + "loss": 73.3801, + "step": 2010 + }, + { + "epoch": 3.2531364941804806, + "grad_norm": 1.182983160018921, + "learning_rate": 0.0001, + "loss": 74.5695, + "step": 2020 + }, + { + "epoch": 3.269259837758855, + "grad_norm": 0.828452467918396, + "learning_rate": 0.0001, + "loss": 75.4313, + "step": 2030 + }, + { + "epoch": 3.2853831813372296, + "grad_norm": 0.8340104818344116, + "learning_rate": 0.0001, + "loss": 75.1086, + "step": 2040 + }, + { + "epoch": 3.301506524915604, + "grad_norm": 1.1700762510299683, + "learning_rate": 0.0001, + "loss": 75.7496, + "step": 2050 + }, + { + "epoch": 3.317629868493979, + "grad_norm": 0.8709110617637634, + "learning_rate": 0.0001, + "loss": 73.8403, + "step": 2060 + }, + { + "epoch": 3.3337532120723536, + "grad_norm": 0.8928954601287842, + "learning_rate": 0.0001, + "loss": 75.6134, + "step": 2070 + }, + { + "epoch": 3.349876555650728, + "grad_norm": 0.9787353277206421, + "learning_rate": 0.0001, + "loss": 75.8116, + "step": 2080 + }, + { + "epoch": 3.3659998992291027, + "grad_norm": 0.9515123963356018, + "learning_rate": 0.0001, + "loss": 75.5024, + "step": 2090 + }, + { + "epoch": 3.3821232428074772, + "grad_norm": 0.9529262185096741, + "learning_rate": 0.0001, + "loss": 74.1553, + "step": 2100 + }, + { + "epoch": 3.3982465863858518, + "grad_norm": 0.8934710621833801, + "learning_rate": 0.0001, + "loss": 74.1501, + "step": 2110 + }, + { + "epoch": 3.4143699299642263, + "grad_norm": 1.089447021484375, + "learning_rate": 0.0001, + "loss": 74.9123, + "step": 2120 + }, + { + "epoch": 3.430493273542601, + "grad_norm": 1.2569622993469238, + "learning_rate": 0.0001, + "loss": 74.3962, + "step": 2130 + }, + { + "epoch": 3.4466166171209753, + "grad_norm": 1.020094394683838, + "learning_rate": 0.0001, + "loss": 74.6617, + "step": 2140 + }, + { + "epoch": 3.46273996069935, + "grad_norm": 1.1579667329788208, + "learning_rate": 0.0001, + "loss": 74.2745, + "step": 2150 + }, + { + "epoch": 3.4788633042777244, + "grad_norm": 1.448935627937317, + "learning_rate": 0.0001, + "loss": 71.5285, + "step": 2160 + }, + { + "epoch": 3.4949866478560994, + "grad_norm": 1.0030494928359985, + "learning_rate": 0.0001, + "loss": 74.6666, + "step": 2170 + }, + { + "epoch": 3.511109991434474, + "grad_norm": 0.9010888934135437, + "learning_rate": 0.0001, + "loss": 74.9355, + "step": 2180 + }, + { + "epoch": 3.5272333350128484, + "grad_norm": 1.1595124006271362, + "learning_rate": 0.0001, + "loss": 74.5049, + "step": 2190 + }, + { + "epoch": 3.543356678591223, + "grad_norm": 1.066405177116394, + "learning_rate": 0.0001, + "loss": 74.6336, + "step": 2200 + }, + { + "epoch": 3.5594800221695975, + "grad_norm": 1.2301596403121948, + "learning_rate": 0.0001, + "loss": 74.6664, + "step": 2210 + }, + { + "epoch": 3.575603365747972, + "grad_norm": 1.0073511600494385, + "learning_rate": 0.0001, + "loss": 73.1405, + "step": 2220 + }, + { + "epoch": 3.5917267093263465, + "grad_norm": 0.9913995862007141, + "learning_rate": 0.0001, + "loss": 75.7443, + "step": 2230 + }, + { + "epoch": 3.607850052904721, + "grad_norm": 0.9555648565292358, + "learning_rate": 0.0001, + "loss": 74.8036, + "step": 2240 + }, + { + "epoch": 3.6239733964830956, + "grad_norm": 0.9350649118423462, + "learning_rate": 0.0001, + "loss": 73.5783, + "step": 2250 + }, + { + "epoch": 3.64009674006147, + "grad_norm": 0.935840368270874, + "learning_rate": 0.0001, + "loss": 75.1694, + "step": 2260 + }, + { + "epoch": 3.6562200836398446, + "grad_norm": 0.9303803443908691, + "learning_rate": 0.0001, + "loss": 72.8282, + "step": 2270 + }, + { + "epoch": 3.6723434272182196, + "grad_norm": 0.937164843082428, + "learning_rate": 0.0001, + "loss": 70.3305, + "step": 2280 + }, + { + "epoch": 3.6884667707965937, + "grad_norm": 1.027329444885254, + "learning_rate": 0.0001, + "loss": 73.748, + "step": 2290 + }, + { + "epoch": 3.7045901143749687, + "grad_norm": 1.0915215015411377, + "learning_rate": 0.0001, + "loss": 72.2265, + "step": 2300 + }, + { + "epoch": 3.720713457953343, + "grad_norm": 1.1031261682510376, + "learning_rate": 0.0001, + "loss": 71.2795, + "step": 2310 + }, + { + "epoch": 3.7368368015317177, + "grad_norm": 1.1466140747070312, + "learning_rate": 0.0001, + "loss": 72.0116, + "step": 2320 + }, + { + "epoch": 3.7529601451100922, + "grad_norm": 1.0441662073135376, + "learning_rate": 0.0001, + "loss": 73.3735, + "step": 2330 + }, + { + "epoch": 3.7690834886884668, + "grad_norm": 0.8610432744026184, + "learning_rate": 0.0001, + "loss": 73.3433, + "step": 2340 + }, + { + "epoch": 3.7852068322668413, + "grad_norm": 0.885720431804657, + "learning_rate": 0.0001, + "loss": 73.575, + "step": 2350 + }, + { + "epoch": 3.801330175845216, + "grad_norm": 1.0023950338363647, + "learning_rate": 0.0001, + "loss": 73.4166, + "step": 2360 + }, + { + "epoch": 3.8174535194235903, + "grad_norm": 1.3820304870605469, + "learning_rate": 0.0001, + "loss": 72.9599, + "step": 2370 + }, + { + "epoch": 3.833576863001965, + "grad_norm": 0.9645710587501526, + "learning_rate": 0.0001, + "loss": 72.3922, + "step": 2380 + }, + { + "epoch": 3.84970020658034, + "grad_norm": 1.273369312286377, + "learning_rate": 0.0001, + "loss": 72.0158, + "step": 2390 + }, + { + "epoch": 3.865823550158714, + "grad_norm": 0.859863817691803, + "learning_rate": 0.0001, + "loss": 74.6642, + "step": 2400 + }, + { + "epoch": 3.881946893737089, + "grad_norm": 0.9396682381629944, + "learning_rate": 0.0001, + "loss": 71.9003, + "step": 2410 + }, + { + "epoch": 3.8980702373154634, + "grad_norm": 0.9976151585578918, + "learning_rate": 0.0001, + "loss": 73.0716, + "step": 2420 + }, + { + "epoch": 3.914193580893838, + "grad_norm": 1.0730767250061035, + "learning_rate": 0.0001, + "loss": 72.7847, + "step": 2430 + }, + { + "epoch": 3.9303169244722125, + "grad_norm": 1.002018690109253, + "learning_rate": 0.0001, + "loss": 71.7309, + "step": 2440 + }, + { + "epoch": 3.946440268050587, + "grad_norm": 0.9102753400802612, + "learning_rate": 0.0001, + "loss": 71.616, + "step": 2450 + }, + { + "epoch": 3.9625636116289615, + "grad_norm": 0.9827467799186707, + "learning_rate": 0.0001, + "loss": 72.22, + "step": 2460 + }, + { + "epoch": 3.978686955207336, + "grad_norm": 1.2781893014907837, + "learning_rate": 0.0001, + "loss": 73.2048, + "step": 2470 + }, + { + "epoch": 3.9948102987857106, + "grad_norm": 0.9042410850524902, + "learning_rate": 0.0001, + "loss": 72.9843, + "step": 2480 + }, + { + "epoch": 4.009674006147025, + "grad_norm": 0.8662117123603821, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 2490 + }, + { + "epoch": 4.0257973497254, + "grad_norm": 0.9448756575584412, + "learning_rate": 0.0001, + "loss": 73.36, + "step": 2500 + }, + { + "epoch": 4.041920693303774, + "grad_norm": 0.9054566621780396, + "learning_rate": 0.0001, + "loss": 73.94, + "step": 2510 + }, + { + "epoch": 4.058044036882149, + "grad_norm": 0.957398533821106, + "learning_rate": 0.0001, + "loss": 73.2019, + "step": 2520 + }, + { + "epoch": 4.074167380460523, + "grad_norm": 0.9444408416748047, + "learning_rate": 0.0001, + "loss": 70.6553, + "step": 2530 + }, + { + "epoch": 4.090290724038898, + "grad_norm": 0.8716063499450684, + "learning_rate": 0.0001, + "loss": 70.1048, + "step": 2540 + }, + { + "epoch": 4.106414067617272, + "grad_norm": 0.949249267578125, + "learning_rate": 0.0001, + "loss": 72.4453, + "step": 2550 + }, + { + "epoch": 4.122537411195647, + "grad_norm": 0.9843282699584961, + "learning_rate": 0.0001, + "loss": 72.1725, + "step": 2560 + }, + { + "epoch": 4.138660754774021, + "grad_norm": 0.8977480530738831, + "learning_rate": 0.0001, + "loss": 72.5272, + "step": 2570 + }, + { + "epoch": 4.154784098352396, + "grad_norm": 0.9957827925682068, + "learning_rate": 0.0001, + "loss": 72.289, + "step": 2580 + }, + { + "epoch": 4.170907441930771, + "grad_norm": 1.1358791589736938, + "learning_rate": 0.0001, + "loss": 72.1949, + "step": 2590 + }, + { + "epoch": 4.187030785509145, + "grad_norm": 0.9953081607818604, + "learning_rate": 0.0001, + "loss": 72.6004, + "step": 2600 + }, + { + "epoch": 4.20315412908752, + "grad_norm": 1.019061803817749, + "learning_rate": 0.0001, + "loss": 72.3772, + "step": 2610 + }, + { + "epoch": 4.219277472665894, + "grad_norm": 0.9062562584877014, + "learning_rate": 0.0001, + "loss": 72.0566, + "step": 2620 + }, + { + "epoch": 4.235400816244269, + "grad_norm": 1.0877611637115479, + "learning_rate": 0.0001, + "loss": 72.854, + "step": 2630 + }, + { + "epoch": 4.251524159822643, + "grad_norm": 1.2419648170471191, + "learning_rate": 0.0001, + "loss": 72.6498, + "step": 2640 + }, + { + "epoch": 4.267647503401018, + "grad_norm": 0.9507225751876831, + "learning_rate": 0.0001, + "loss": 68.9583, + "step": 2650 + }, + { + "epoch": 4.283770846979392, + "grad_norm": 1.121191382408142, + "learning_rate": 0.0001, + "loss": 72.8464, + "step": 2660 + }, + { + "epoch": 4.299894190557767, + "grad_norm": 1.0543361902236938, + "learning_rate": 0.0001, + "loss": 71.9882, + "step": 2670 + }, + { + "epoch": 4.316017534136141, + "grad_norm": 0.8622955083847046, + "learning_rate": 0.0001, + "loss": 71.7373, + "step": 2680 + }, + { + "epoch": 4.332140877714516, + "grad_norm": 0.911253809928894, + "learning_rate": 0.0001, + "loss": 71.1968, + "step": 2690 + }, + { + "epoch": 4.34826422129289, + "grad_norm": 4.405472755432129, + "learning_rate": 0.0001, + "loss": 70.4128, + "step": 2700 + }, + { + "epoch": 4.364387564871265, + "grad_norm": 0.9142985939979553, + "learning_rate": 0.0001, + "loss": 69.7548, + "step": 2710 + }, + { + "epoch": 4.38051090844964, + "grad_norm": 1.1022522449493408, + "learning_rate": 0.0001, + "loss": 71.8282, + "step": 2720 + }, + { + "epoch": 4.396634252028014, + "grad_norm": 1.018386721611023, + "learning_rate": 0.0001, + "loss": 71.0853, + "step": 2730 + }, + { + "epoch": 4.412757595606389, + "grad_norm": 1.0483211278915405, + "learning_rate": 0.0001, + "loss": 72.1478, + "step": 2740 + }, + { + "epoch": 4.428880939184763, + "grad_norm": 1.026861548423767, + "learning_rate": 0.0001, + "loss": 71.8947, + "step": 2750 + }, + { + "epoch": 4.445004282763138, + "grad_norm": 1.1858552694320679, + "learning_rate": 0.0001, + "loss": 71.7016, + "step": 2760 + }, + { + "epoch": 4.461127626341512, + "grad_norm": 1.1212221384048462, + "learning_rate": 0.0001, + "loss": 71.4454, + "step": 2770 + }, + { + "epoch": 4.477250969919887, + "grad_norm": 0.8062102794647217, + "learning_rate": 0.0001, + "loss": 73.2672, + "step": 2780 + }, + { + "epoch": 4.493374313498261, + "grad_norm": 1.104483962059021, + "learning_rate": 0.0001, + "loss": 70.548, + "step": 2790 + }, + { + "epoch": 4.509497657076636, + "grad_norm": 0.942057728767395, + "learning_rate": 0.0001, + "loss": 72.3584, + "step": 2800 + }, + { + "epoch": 4.525621000655011, + "grad_norm": 0.9028333425521851, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 2810 + }, + { + "epoch": 4.541744344233385, + "grad_norm": 0.9096795916557312, + "learning_rate": 0.0001, + "loss": 70.6596, + "step": 2820 + }, + { + "epoch": 4.55786768781176, + "grad_norm": 0.9038829207420349, + "learning_rate": 0.0001, + "loss": 72.6148, + "step": 2830 + }, + { + "epoch": 4.573991031390134, + "grad_norm": 1.8511613607406616, + "learning_rate": 0.0001, + "loss": 71.7205, + "step": 2840 + }, + { + "epoch": 4.590114374968509, + "grad_norm": 1.3492343425750732, + "learning_rate": 0.0001, + "loss": 72.1492, + "step": 2850 + }, + { + "epoch": 4.6062377185468835, + "grad_norm": 1.1469528675079346, + "learning_rate": 0.0001, + "loss": 70.5892, + "step": 2860 + }, + { + "epoch": 4.622361062125258, + "grad_norm": 1.1374379396438599, + "learning_rate": 0.0001, + "loss": 70.061, + "step": 2870 + }, + { + "epoch": 4.6384844057036325, + "grad_norm": 0.9597565531730652, + "learning_rate": 0.0001, + "loss": 71.0869, + "step": 2880 + }, + { + "epoch": 4.6546077492820075, + "grad_norm": 0.8356521129608154, + "learning_rate": 0.0001, + "loss": 72.1891, + "step": 2890 + }, + { + "epoch": 4.670731092860382, + "grad_norm": 0.8882647752761841, + "learning_rate": 0.0001, + "loss": 71.3845, + "step": 2900 + }, + { + "epoch": 4.6868544364387565, + "grad_norm": 1.1904003620147705, + "learning_rate": 0.0001, + "loss": 72.3086, + "step": 2910 + }, + { + "epoch": 4.702977780017131, + "grad_norm": 0.8010085821151733, + "learning_rate": 0.0001, + "loss": 71.3051, + "step": 2920 + }, + { + "epoch": 4.719101123595506, + "grad_norm": 0.7572135329246521, + "learning_rate": 0.0001, + "loss": 72.5605, + "step": 2930 + }, + { + "epoch": 4.735224467173881, + "grad_norm": 1.0866942405700684, + "learning_rate": 0.0001, + "loss": 70.0935, + "step": 2940 + }, + { + "epoch": 4.751347810752255, + "grad_norm": 0.9358721971511841, + "learning_rate": 0.0001, + "loss": 71.0391, + "step": 2950 + }, + { + "epoch": 4.76747115433063, + "grad_norm": 1.0276161432266235, + "learning_rate": 0.0001, + "loss": 69.8462, + "step": 2960 + }, + { + "epoch": 4.783594497909004, + "grad_norm": 0.9573063850402832, + "learning_rate": 0.0001, + "loss": 69.7089, + "step": 2970 + }, + { + "epoch": 4.799717841487379, + "grad_norm": 1.0638892650604248, + "learning_rate": 0.0001, + "loss": 71.8376, + "step": 2980 + }, + { + "epoch": 4.815841185065753, + "grad_norm": 0.9894681572914124, + "learning_rate": 0.0001, + "loss": 70.1764, + "step": 2990 + }, + { + "epoch": 4.831964528644128, + "grad_norm": 1.1265769004821777, + "learning_rate": 0.0001, + "loss": 71.7918, + "step": 3000 + }, + { + "epoch": 4.848087872222502, + "grad_norm": 2.2792513370513916, + "learning_rate": 0.0001, + "loss": 70.4341, + "step": 3010 + }, + { + "epoch": 4.864211215800877, + "grad_norm": 0.8469329476356506, + "learning_rate": 0.0001, + "loss": 72.1137, + "step": 3020 + }, + { + "epoch": 4.880334559379252, + "grad_norm": 0.9333013296127319, + "learning_rate": 0.0001, + "loss": 71.0789, + "step": 3030 + }, + { + "epoch": 4.896457902957626, + "grad_norm": 0.7888994216918945, + "learning_rate": 0.0001, + "loss": 70.8197, + "step": 3040 + }, + { + "epoch": 4.912581246536, + "grad_norm": 0.9194874167442322, + "learning_rate": 0.0001, + "loss": 72.071, + "step": 3050 + }, + { + "epoch": 4.928704590114375, + "grad_norm": 1.1252421140670776, + "learning_rate": 0.0001, + "loss": 71.6943, + "step": 3060 + }, + { + "epoch": 4.94482793369275, + "grad_norm": 1.1801071166992188, + "learning_rate": 0.0001, + "loss": 70.8937, + "step": 3070 + }, + { + "epoch": 4.960951277271124, + "grad_norm": 0.9162415266036987, + "learning_rate": 0.0001, + "loss": 70.024, + "step": 3080 + }, + { + "epoch": 4.977074620849499, + "grad_norm": 0.8833363652229309, + "learning_rate": 0.0001, + "loss": 71.3835, + "step": 3090 + }, + { + "epoch": 4.993197964427873, + "grad_norm": 0.9985883831977844, + "learning_rate": 0.0001, + "loss": 71.635, + "step": 3100 + }, + { + "epoch": 5.0080616717891875, + "grad_norm": 1.097298264503479, + "learning_rate": 0.0001, + "loss": 65.3542, + "step": 3110 + }, + { + "epoch": 5.024185015367562, + "grad_norm": 0.9703903198242188, + "learning_rate": 0.0001, + "loss": 69.8587, + "step": 3120 + }, + { + "epoch": 5.0403083589459365, + "grad_norm": 0.8894070386886597, + "learning_rate": 0.0001, + "loss": 71.2868, + "step": 3130 + }, + { + "epoch": 5.056431702524311, + "grad_norm": 0.6984127163887024, + "learning_rate": 0.0001, + "loss": 69.0425, + "step": 3140 + }, + { + "epoch": 5.072555046102686, + "grad_norm": 0.8607762455940247, + "learning_rate": 0.0001, + "loss": 69.5354, + "step": 3150 + }, + { + "epoch": 5.08867838968106, + "grad_norm": 1.1215914487838745, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 3160 + }, + { + "epoch": 5.104801733259435, + "grad_norm": 0.9114409685134888, + "learning_rate": 0.0001, + "loss": 68.4186, + "step": 3170 + }, + { + "epoch": 5.12092507683781, + "grad_norm": 1.010400652885437, + "learning_rate": 0.0001, + "loss": 68.3965, + "step": 3180 + }, + { + "epoch": 5.137048420416184, + "grad_norm": 0.8741958141326904, + "learning_rate": 0.0001, + "loss": 70.6728, + "step": 3190 + }, + { + "epoch": 5.153171763994559, + "grad_norm": 0.8760207891464233, + "learning_rate": 0.0001, + "loss": 72.0704, + "step": 3200 + }, + { + "epoch": 5.169295107572933, + "grad_norm": 1.328247308731079, + "learning_rate": 0.0001, + "loss": 69.9082, + "step": 3210 + }, + { + "epoch": 5.185418451151308, + "grad_norm": 0.8263765573501587, + "learning_rate": 0.0001, + "loss": 70.3539, + "step": 3220 + }, + { + "epoch": 5.201541794729682, + "grad_norm": 0.9358834028244019, + "learning_rate": 0.0001, + "loss": 70.4221, + "step": 3230 + }, + { + "epoch": 5.217665138308057, + "grad_norm": 1.0769531726837158, + "learning_rate": 0.0001, + "loss": 68.8167, + "step": 3240 + }, + { + "epoch": 5.233788481886431, + "grad_norm": 1.1438790559768677, + "learning_rate": 0.0001, + "loss": 68.9288, + "step": 3250 + }, + { + "epoch": 5.249911825464806, + "grad_norm": 0.9534218311309814, + "learning_rate": 0.0001, + "loss": 70.3277, + "step": 3260 + }, + { + "epoch": 5.26603516904318, + "grad_norm": 0.88532954454422, + "learning_rate": 0.0001, + "loss": 67.5781, + "step": 3270 + }, + { + "epoch": 5.282158512621555, + "grad_norm": 0.7986133694648743, + "learning_rate": 0.0001, + "loss": 69.759, + "step": 3280 + }, + { + "epoch": 5.29828185619993, + "grad_norm": 0.9100566506385803, + "learning_rate": 0.0001, + "loss": 68.6368, + "step": 3290 + }, + { + "epoch": 5.314405199778304, + "grad_norm": 0.8861745595932007, + "learning_rate": 0.0001, + "loss": 70.5756, + "step": 3300 + }, + { + "epoch": 5.330528543356679, + "grad_norm": 1.0729680061340332, + "learning_rate": 0.0001, + "loss": 70.3178, + "step": 3310 + }, + { + "epoch": 5.346651886935053, + "grad_norm": 1.0288280248641968, + "learning_rate": 0.0001, + "loss": 69.422, + "step": 3320 + }, + { + "epoch": 5.362775230513428, + "grad_norm": 0.8335140347480774, + "learning_rate": 0.0001, + "loss": 69.8281, + "step": 3330 + }, + { + "epoch": 5.378898574091802, + "grad_norm": 0.9789717197418213, + "learning_rate": 0.0001, + "loss": 71.1532, + "step": 3340 + }, + { + "epoch": 5.395021917670177, + "grad_norm": 0.9884677529335022, + "learning_rate": 0.0001, + "loss": 69.5673, + "step": 3350 + }, + { + "epoch": 5.411145261248551, + "grad_norm": 1.050154447555542, + "learning_rate": 0.0001, + "loss": 70.8015, + "step": 3360 + }, + { + "epoch": 5.427268604826926, + "grad_norm": 0.9645342230796814, + "learning_rate": 0.0001, + "loss": 69.1615, + "step": 3370 + }, + { + "epoch": 5.4433919484053, + "grad_norm": 0.8383159041404724, + "learning_rate": 0.0001, + "loss": 70.6299, + "step": 3380 + }, + { + "epoch": 5.459515291983675, + "grad_norm": 0.9250556826591492, + "learning_rate": 0.0001, + "loss": 69.231, + "step": 3390 + }, + { + "epoch": 5.475638635562049, + "grad_norm": 0.8108687400817871, + "learning_rate": 0.0001, + "loss": 70.5858, + "step": 3400 + }, + { + "epoch": 5.491761979140424, + "grad_norm": 0.8748818039894104, + "learning_rate": 0.0001, + "loss": 69.9001, + "step": 3410 + }, + { + "epoch": 5.507885322718799, + "grad_norm": 0.7411381006240845, + "learning_rate": 0.0001, + "loss": 68.3248, + "step": 3420 + }, + { + "epoch": 5.524008666297173, + "grad_norm": 1.0753326416015625, + "learning_rate": 0.0001, + "loss": 69.5831, + "step": 3430 + }, + { + "epoch": 5.540132009875548, + "grad_norm": 0.875254213809967, + "learning_rate": 0.0001, + "loss": 69.7516, + "step": 3440 + }, + { + "epoch": 5.556255353453922, + "grad_norm": 0.7637733817100525, + "learning_rate": 0.0001, + "loss": 68.0078, + "step": 3450 + }, + { + "epoch": 5.572378697032297, + "grad_norm": 0.968203604221344, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 3460 + }, + { + "epoch": 5.588502040610671, + "grad_norm": 0.8903141617774963, + "learning_rate": 0.0001, + "loss": 68.5835, + "step": 3470 + }, + { + "epoch": 5.604625384189046, + "grad_norm": 0.9536195397377014, + "learning_rate": 0.0001, + "loss": 71.0259, + "step": 3480 + }, + { + "epoch": 5.62074872776742, + "grad_norm": 0.8951714038848877, + "learning_rate": 0.0001, + "loss": 69.4422, + "step": 3490 + }, + { + "epoch": 5.636872071345795, + "grad_norm": 0.9639522433280945, + "learning_rate": 0.0001, + "loss": 66.5173, + "step": 3500 + }, + { + "epoch": 5.65299541492417, + "grad_norm": 1.0022730827331543, + "learning_rate": 0.0001, + "loss": 68.3443, + "step": 3510 + }, + { + "epoch": 5.669118758502544, + "grad_norm": 1.0654487609863281, + "learning_rate": 0.0001, + "loss": 69.0835, + "step": 3520 + }, + { + "epoch": 5.685242102080919, + "grad_norm": 0.9934629797935486, + "learning_rate": 0.0001, + "loss": 69.2278, + "step": 3530 + }, + { + "epoch": 5.7013654456592935, + "grad_norm": 0.8773566484451294, + "learning_rate": 0.0001, + "loss": 68.9339, + "step": 3540 + }, + { + "epoch": 5.7174887892376685, + "grad_norm": 0.7756998538970947, + "learning_rate": 0.0001, + "loss": 69.2683, + "step": 3550 + }, + { + "epoch": 5.7336121328160425, + "grad_norm": 0.8819359540939331, + "learning_rate": 0.0001, + "loss": 68.5071, + "step": 3560 + }, + { + "epoch": 5.7497354763944175, + "grad_norm": 1.1069825887680054, + "learning_rate": 0.0001, + "loss": 68.5236, + "step": 3570 + }, + { + "epoch": 5.765858819972792, + "grad_norm": 0.8006312847137451, + "learning_rate": 0.0001, + "loss": 68.7192, + "step": 3580 + }, + { + "epoch": 5.781982163551167, + "grad_norm": 1.0278719663619995, + "learning_rate": 0.0001, + "loss": 69.2545, + "step": 3590 + }, + { + "epoch": 5.798105507129541, + "grad_norm": 0.9205041527748108, + "learning_rate": 0.0001, + "loss": 69.7698, + "step": 3600 + }, + { + "epoch": 5.814228850707916, + "grad_norm": 1.0769038200378418, + "learning_rate": 0.0001, + "loss": 67.6568, + "step": 3610 + }, + { + "epoch": 5.83035219428629, + "grad_norm": 1.0565663576126099, + "learning_rate": 0.0001, + "loss": 68.9166, + "step": 3620 + }, + { + "epoch": 5.846475537864665, + "grad_norm": 0.9008663296699524, + "learning_rate": 0.0001, + "loss": 69.5837, + "step": 3630 + }, + { + "epoch": 5.86259888144304, + "grad_norm": 0.8369411826133728, + "learning_rate": 0.0001, + "loss": 68.5661, + "step": 3640 + }, + { + "epoch": 5.878722225021414, + "grad_norm": 1.0472379922866821, + "learning_rate": 0.0001, + "loss": 68.3122, + "step": 3650 + }, + { + "epoch": 5.894845568599789, + "grad_norm": 0.965277373790741, + "learning_rate": 0.0001, + "loss": 69.8062, + "step": 3660 + }, + { + "epoch": 5.910968912178163, + "grad_norm": 0.8936495780944824, + "learning_rate": 0.0001, + "loss": 67.1016, + "step": 3670 + }, + { + "epoch": 5.927092255756538, + "grad_norm": 0.8559162020683289, + "learning_rate": 0.0001, + "loss": 69.9986, + "step": 3680 + }, + { + "epoch": 5.943215599334912, + "grad_norm": 1.4815773963928223, + "learning_rate": 0.0001, + "loss": 66.7648, + "step": 3690 + }, + { + "epoch": 5.959338942913287, + "grad_norm": 0.9034469723701477, + "learning_rate": 0.0001, + "loss": 68.0767, + "step": 3700 + }, + { + "epoch": 5.975462286491661, + "grad_norm": 1.838799238204956, + "learning_rate": 0.0001, + "loss": 66.5848, + "step": 3710 + }, + { + "epoch": 5.991585630070036, + "grad_norm": 0.9274787902832031, + "learning_rate": 0.0001, + "loss": 70.0396, + "step": 3720 + }, + { + "epoch": 6.0064493374313495, + "grad_norm": 1.016974687576294, + "learning_rate": 0.0001, + "loss": 62.2268, + "step": 3730 + }, + { + "epoch": 6.022572681009724, + "grad_norm": 1.0522642135620117, + "learning_rate": 0.0001, + "loss": 69.3381, + "step": 3740 + }, + { + "epoch": 6.0386960245880985, + "grad_norm": 1.1157186031341553, + "learning_rate": 0.0001, + "loss": 68.8969, + "step": 3750 + }, + { + "epoch": 6.0548193681664735, + "grad_norm": 0.9913288950920105, + "learning_rate": 0.0001, + "loss": 67.3634, + "step": 3760 + }, + { + "epoch": 6.0709427117448485, + "grad_norm": 0.8823490142822266, + "learning_rate": 0.0001, + "loss": 70.978, + "step": 3770 + }, + { + "epoch": 6.0870660553232225, + "grad_norm": 0.853837788105011, + "learning_rate": 0.0001, + "loss": 68.9924, + "step": 3780 + }, + { + "epoch": 6.1031893989015975, + "grad_norm": 0.7937766909599304, + "learning_rate": 0.0001, + "loss": 68.1994, + "step": 3790 + }, + { + "epoch": 6.119312742479972, + "grad_norm": 1.7638057470321655, + "learning_rate": 0.0001, + "loss": 68.5078, + "step": 3800 + }, + { + "epoch": 6.135436086058347, + "grad_norm": 1.0542887449264526, + "learning_rate": 0.0001, + "loss": 68.9362, + "step": 3810 + }, + { + "epoch": 6.151559429636721, + "grad_norm": 1.06019926071167, + "learning_rate": 0.0001, + "loss": 67.2493, + "step": 3820 + }, + { + "epoch": 6.167682773215096, + "grad_norm": 0.9765111804008484, + "learning_rate": 0.0001, + "loss": 70.4797, + "step": 3830 + }, + { + "epoch": 6.18380611679347, + "grad_norm": 1.006372332572937, + "learning_rate": 0.0001, + "loss": 69.4233, + "step": 3840 + }, + { + "epoch": 6.199929460371845, + "grad_norm": 0.8274241089820862, + "learning_rate": 0.0001, + "loss": 67.9907, + "step": 3850 + }, + { + "epoch": 6.216052803950219, + "grad_norm": 0.9905765056610107, + "learning_rate": 0.0001, + "loss": 69.6988, + "step": 3860 + }, + { + "epoch": 6.232176147528594, + "grad_norm": 1.0065921545028687, + "learning_rate": 0.0001, + "loss": 68.7678, + "step": 3870 + }, + { + "epoch": 6.248299491106969, + "grad_norm": 0.8845255970954895, + "learning_rate": 0.0001, + "loss": 68.3039, + "step": 3880 + }, + { + "epoch": 6.264422834685343, + "grad_norm": 0.9030968546867371, + "learning_rate": 0.0001, + "loss": 68.671, + "step": 3890 + }, + { + "epoch": 6.280546178263718, + "grad_norm": 0.8650258183479309, + "learning_rate": 0.0001, + "loss": 68.5309, + "step": 3900 + }, + { + "epoch": 6.296669521842092, + "grad_norm": 0.8961272835731506, + "learning_rate": 0.0001, + "loss": 67.5611, + "step": 3910 + }, + { + "epoch": 6.312792865420467, + "grad_norm": 1.1702407598495483, + "learning_rate": 0.0001, + "loss": 68.2768, + "step": 3920 + }, + { + "epoch": 6.328916208998841, + "grad_norm": 0.8598962426185608, + "learning_rate": 0.0001, + "loss": 69.1961, + "step": 3930 + }, + { + "epoch": 6.345039552577216, + "grad_norm": 0.8714122176170349, + "learning_rate": 0.0001, + "loss": 66.8628, + "step": 3940 + }, + { + "epoch": 6.36116289615559, + "grad_norm": 0.7757895588874817, + "learning_rate": 0.0001, + "loss": 68.6908, + "step": 3950 + }, + { + "epoch": 6.377286239733965, + "grad_norm": 0.9189490675926208, + "learning_rate": 0.0001, + "loss": 67.8685, + "step": 3960 + }, + { + "epoch": 6.393409583312339, + "grad_norm": 0.8161299228668213, + "learning_rate": 0.0001, + "loss": 66.816, + "step": 3970 + }, + { + "epoch": 6.409532926890714, + "grad_norm": 0.8559791445732117, + "learning_rate": 0.0001, + "loss": 68.4302, + "step": 3980 + }, + { + "epoch": 6.425656270469089, + "grad_norm": 0.8087984919548035, + "learning_rate": 0.0001, + "loss": 66.2646, + "step": 3990 + }, + { + "epoch": 6.441779614047463, + "grad_norm": 0.7733110785484314, + "learning_rate": 0.0001, + "loss": 69.2769, + "step": 4000 + }, + { + "epoch": 6.457902957625838, + "grad_norm": 0.9335984587669373, + "learning_rate": 0.0001, + "loss": 67.1199, + "step": 4010 + }, + { + "epoch": 6.474026301204212, + "grad_norm": 0.9546418786048889, + "learning_rate": 0.0001, + "loss": 68.8706, + "step": 4020 + }, + { + "epoch": 6.490149644782587, + "grad_norm": 0.9670593738555908, + "learning_rate": 0.0001, + "loss": 68.2329, + "step": 4030 + }, + { + "epoch": 6.506272988360961, + "grad_norm": 0.8428588509559631, + "learning_rate": 0.0001, + "loss": 68.0105, + "step": 4040 + }, + { + "epoch": 6.522396331939336, + "grad_norm": 0.9755622744560242, + "learning_rate": 0.0001, + "loss": 68.656, + "step": 4050 + }, + { + "epoch": 6.53851967551771, + "grad_norm": 0.8972346186637878, + "learning_rate": 0.0001, + "loss": 69.3665, + "step": 4060 + }, + { + "epoch": 6.554643019096085, + "grad_norm": 0.9716668128967285, + "learning_rate": 0.0001, + "loss": 68.1053, + "step": 4070 + }, + { + "epoch": 6.570766362674459, + "grad_norm": 0.8147910833358765, + "learning_rate": 0.0001, + "loss": 67.9778, + "step": 4080 + }, + { + "epoch": 6.586889706252834, + "grad_norm": 0.8304234147071838, + "learning_rate": 0.0001, + "loss": 67.7069, + "step": 4090 + }, + { + "epoch": 6.603013049831208, + "grad_norm": 0.9356856346130371, + "learning_rate": 0.0001, + "loss": 68.1942, + "step": 4100 + }, + { + "epoch": 6.619136393409583, + "grad_norm": 1.017134189605713, + "learning_rate": 0.0001, + "loss": 67.4211, + "step": 4110 + }, + { + "epoch": 6.635259736987958, + "grad_norm": 1.142558217048645, + "learning_rate": 0.0001, + "loss": 68.4413, + "step": 4120 + }, + { + "epoch": 6.651383080566332, + "grad_norm": 0.9071153998374939, + "learning_rate": 0.0001, + "loss": 69.3666, + "step": 4130 + }, + { + "epoch": 6.667506424144707, + "grad_norm": 0.9620397686958313, + "learning_rate": 0.0001, + "loss": 68.208, + "step": 4140 + }, + { + "epoch": 6.683629767723081, + "grad_norm": 0.8282128572463989, + "learning_rate": 0.0001, + "loss": 67.1702, + "step": 4150 + }, + { + "epoch": 6.699753111301456, + "grad_norm": 0.8838199973106384, + "learning_rate": 0.0001, + "loss": 69.0074, + "step": 4160 + }, + { + "epoch": 6.71587645487983, + "grad_norm": 0.8864356279373169, + "learning_rate": 0.0001, + "loss": 67.8052, + "step": 4170 + }, + { + "epoch": 6.731999798458205, + "grad_norm": 0.9373385906219482, + "learning_rate": 0.0001, + "loss": 67.446, + "step": 4180 + }, + { + "epoch": 6.7481231420365795, + "grad_norm": 0.8308956027030945, + "learning_rate": 0.0001, + "loss": 68.716, + "step": 4190 + }, + { + "epoch": 6.7642464856149545, + "grad_norm": 0.8031372427940369, + "learning_rate": 0.0001, + "loss": 68.0482, + "step": 4200 + }, + { + "epoch": 6.780369829193329, + "grad_norm": 0.9258586764335632, + "learning_rate": 0.0001, + "loss": 68.5607, + "step": 4210 + }, + { + "epoch": 6.7964931727717035, + "grad_norm": 0.8147327899932861, + "learning_rate": 0.0001, + "loss": 67.7265, + "step": 4220 + }, + { + "epoch": 6.8126165163500785, + "grad_norm": 0.8717535734176636, + "learning_rate": 0.0001, + "loss": 67.8097, + "step": 4230 + }, + { + "epoch": 6.828739859928453, + "grad_norm": 0.9876209497451782, + "learning_rate": 0.0001, + "loss": 67.9068, + "step": 4240 + }, + { + "epoch": 6.8448632035068275, + "grad_norm": 0.718376874923706, + "learning_rate": 0.0001, + "loss": 68.1762, + "step": 4250 + }, + { + "epoch": 6.860986547085202, + "grad_norm": 0.8264320492744446, + "learning_rate": 0.0001, + "loss": 67.8391, + "step": 4260 + }, + { + "epoch": 6.877109890663577, + "grad_norm": 0.7478179335594177, + "learning_rate": 0.0001, + "loss": 67.6635, + "step": 4270 + }, + { + "epoch": 6.893233234241951, + "grad_norm": 0.8394957184791565, + "learning_rate": 0.0001, + "loss": 68.6073, + "step": 4280 + }, + { + "epoch": 6.909356577820326, + "grad_norm": 0.8534071445465088, + "learning_rate": 0.0001, + "loss": 69.6206, + "step": 4290 + }, + { + "epoch": 6.9254799213987, + "grad_norm": 0.9137582778930664, + "learning_rate": 0.0001, + "loss": 67.4362, + "step": 4300 + }, + { + "epoch": 6.941603264977075, + "grad_norm": 0.8869971036911011, + "learning_rate": 0.0001, + "loss": 67.6811, + "step": 4310 + }, + { + "epoch": 6.957726608555449, + "grad_norm": 0.889025866985321, + "learning_rate": 0.0001, + "loss": 65.3354, + "step": 4320 + }, + { + "epoch": 6.973849952133824, + "grad_norm": 0.7968549728393555, + "learning_rate": 0.0001, + "loss": 66.192, + "step": 4330 + }, + { + "epoch": 6.989973295712199, + "grad_norm": 1.0077433586120605, + "learning_rate": 0.0001, + "loss": 67.5216, + "step": 4340 + }, + { + "epoch": 7.004837003073512, + "grad_norm": 0.7900649309158325, + "learning_rate": 0.0001, + "loss": 62.2421, + "step": 4350 + }, + { + "epoch": 7.020960346651887, + "grad_norm": 1.0160410404205322, + "learning_rate": 0.0001, + "loss": 68.0978, + "step": 4360 + }, + { + "epoch": 7.037083690230261, + "grad_norm": 0.841387927532196, + "learning_rate": 0.0001, + "loss": 68.1008, + "step": 4370 + }, + { + "epoch": 7.053207033808636, + "grad_norm": 0.87884122133255, + "learning_rate": 0.0001, + "loss": 66.4667, + "step": 4380 + }, + { + "epoch": 7.06933037738701, + "grad_norm": 0.8580564856529236, + "learning_rate": 0.0001, + "loss": 67.2002, + "step": 4390 + }, + { + "epoch": 7.085453720965385, + "grad_norm": 0.7560063600540161, + "learning_rate": 0.0001, + "loss": 67.6743, + "step": 4400 + }, + { + "epoch": 7.1015770645437595, + "grad_norm": 0.855478048324585, + "learning_rate": 0.0001, + "loss": 67.822, + "step": 4410 + }, + { + "epoch": 7.1177004081221344, + "grad_norm": 0.7994911670684814, + "learning_rate": 0.0001, + "loss": 66.0094, + "step": 4420 + }, + { + "epoch": 7.1338237517005085, + "grad_norm": 0.8728616833686829, + "learning_rate": 0.0001, + "loss": 66.405, + "step": 4430 + }, + { + "epoch": 7.1499470952788835, + "grad_norm": 0.8434081673622131, + "learning_rate": 0.0001, + "loss": 67.0535, + "step": 4440 + }, + { + "epoch": 7.166070438857258, + "grad_norm": 0.8543094396591187, + "learning_rate": 0.0001, + "loss": 66.2417, + "step": 4450 + }, + { + "epoch": 7.182193782435633, + "grad_norm": 0.7209319472312927, + "learning_rate": 0.0001, + "loss": 67.3958, + "step": 4460 + }, + { + "epoch": 7.1983171260140075, + "grad_norm": 0.6735154390335083, + "learning_rate": 0.0001, + "loss": 67.4086, + "step": 4470 + }, + { + "epoch": 7.214440469592382, + "grad_norm": 0.9126104116439819, + "learning_rate": 0.0001, + "loss": 68.4191, + "step": 4480 + }, + { + "epoch": 7.230563813170757, + "grad_norm": 0.9334300756454468, + "learning_rate": 0.0001, + "loss": 68.3247, + "step": 4490 + }, + { + "epoch": 7.246687156749131, + "grad_norm": 1.5956038236618042, + "learning_rate": 0.0001, + "loss": 68.2779, + "step": 4500 + }, + { + "epoch": 7.262810500327506, + "grad_norm": 1.5012775659561157, + "learning_rate": 0.0001, + "loss": 68.3533, + "step": 4510 + }, + { + "epoch": 7.27893384390588, + "grad_norm": 1.1376632452011108, + "learning_rate": 0.0001, + "loss": 65.3627, + "step": 4520 + }, + { + "epoch": 7.295057187484255, + "grad_norm": 0.8597291111946106, + "learning_rate": 0.0001, + "loss": 67.8653, + "step": 4530 + }, + { + "epoch": 7.311180531062629, + "grad_norm": 0.7645048499107361, + "learning_rate": 0.0001, + "loss": 65.4592, + "step": 4540 + }, + { + "epoch": 7.327303874641004, + "grad_norm": 0.7717318534851074, + "learning_rate": 0.0001, + "loss": 66.902, + "step": 4550 + }, + { + "epoch": 7.343427218219378, + "grad_norm": 0.8884950876235962, + "learning_rate": 0.0001, + "loss": 67.1858, + "step": 4560 + }, + { + "epoch": 7.359550561797753, + "grad_norm": 0.7826898694038391, + "learning_rate": 0.0001, + "loss": 68.4673, + "step": 4570 + }, + { + "epoch": 7.375673905376128, + "grad_norm": 0.7484182119369507, + "learning_rate": 0.0001, + "loss": 67.4859, + "step": 4580 + }, + { + "epoch": 7.391797248954502, + "grad_norm": 0.918194591999054, + "learning_rate": 0.0001, + "loss": 67.7393, + "step": 4590 + }, + { + "epoch": 7.407920592532877, + "grad_norm": 1.0696896314620972, + "learning_rate": 0.0001, + "loss": 67.0101, + "step": 4600 + }, + { + "epoch": 7.424043936111251, + "grad_norm": 0.8190692067146301, + "learning_rate": 0.0001, + "loss": 68.5358, + "step": 4610 + }, + { + "epoch": 7.440167279689626, + "grad_norm": 0.9063783884048462, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 4620 + }, + { + "epoch": 7.456290623268, + "grad_norm": 0.7588100433349609, + "learning_rate": 0.0001, + "loss": 68.0066, + "step": 4630 + }, + { + "epoch": 7.472413966846375, + "grad_norm": 0.897281289100647, + "learning_rate": 0.0001, + "loss": 67.2499, + "step": 4640 + }, + { + "epoch": 7.488537310424749, + "grad_norm": 0.9505569338798523, + "learning_rate": 0.0001, + "loss": 67.5452, + "step": 4650 + }, + { + "epoch": 7.504660654003124, + "grad_norm": 0.8409159183502197, + "learning_rate": 0.0001, + "loss": 66.6293, + "step": 4660 + }, + { + "epoch": 7.520783997581498, + "grad_norm": 0.9878574013710022, + "learning_rate": 0.0001, + "loss": 65.3379, + "step": 4670 + }, + { + "epoch": 7.536907341159873, + "grad_norm": 1.0410685539245605, + "learning_rate": 0.0001, + "loss": 65.8028, + "step": 4680 + }, + { + "epoch": 7.553030684738248, + "grad_norm": 0.8417725563049316, + "learning_rate": 0.0001, + "loss": 65.4283, + "step": 4690 + }, + { + "epoch": 7.569154028316622, + "grad_norm": 0.7660863399505615, + "learning_rate": 0.0001, + "loss": 66.6765, + "step": 4700 + }, + { + "epoch": 7.585277371894997, + "grad_norm": 0.8074233531951904, + "learning_rate": 0.0001, + "loss": 67.1082, + "step": 4710 + }, + { + "epoch": 7.601400715473371, + "grad_norm": 0.7653326988220215, + "learning_rate": 0.0001, + "loss": 66.028, + "step": 4720 + }, + { + "epoch": 7.617524059051746, + "grad_norm": 1.1764041185379028, + "learning_rate": 0.0001, + "loss": 65.2587, + "step": 4730 + }, + { + "epoch": 7.63364740263012, + "grad_norm": 1.7604950666427612, + "learning_rate": 0.0001, + "loss": 66.0543, + "step": 4740 + }, + { + "epoch": 7.649770746208495, + "grad_norm": 0.941438615322113, + "learning_rate": 0.0001, + "loss": 65.6954, + "step": 4750 + }, + { + "epoch": 7.665894089786869, + "grad_norm": 0.740069568157196, + "learning_rate": 0.0001, + "loss": 67.9701, + "step": 4760 + }, + { + "epoch": 7.682017433365244, + "grad_norm": 0.9867332577705383, + "learning_rate": 0.0001, + "loss": 66.1445, + "step": 4770 + }, + { + "epoch": 7.698140776943618, + "grad_norm": 0.8156166672706604, + "learning_rate": 0.0001, + "loss": 67.2647, + "step": 4780 + }, + { + "epoch": 7.714264120521993, + "grad_norm": 0.7689579129219055, + "learning_rate": 0.0001, + "loss": 67.0961, + "step": 4790 + }, + { + "epoch": 7.730387464100367, + "grad_norm": 0.8159700036048889, + "learning_rate": 0.0001, + "loss": 66.4974, + "step": 4800 + }, + { + "epoch": 7.746510807678742, + "grad_norm": 0.8554222583770752, + "learning_rate": 0.0001, + "loss": 64.5057, + "step": 4810 + }, + { + "epoch": 7.762634151257117, + "grad_norm": 0.856629490852356, + "learning_rate": 0.0001, + "loss": 67.7819, + "step": 4820 + }, + { + "epoch": 7.778757494835491, + "grad_norm": 1.157179832458496, + "learning_rate": 0.0001, + "loss": 66.0122, + "step": 4830 + }, + { + "epoch": 7.794880838413866, + "grad_norm": 0.9597711563110352, + "learning_rate": 0.0001, + "loss": 66.8248, + "step": 4840 + }, + { + "epoch": 7.8110041819922404, + "grad_norm": 0.9175744652748108, + "learning_rate": 0.0001, + "loss": 65.7998, + "step": 4850 + }, + { + "epoch": 7.827127525570615, + "grad_norm": 0.8467298150062561, + "learning_rate": 0.0001, + "loss": 67.5363, + "step": 4860 + }, + { + "epoch": 7.8432508691489895, + "grad_norm": 0.863922655582428, + "learning_rate": 0.0001, + "loss": 67.7219, + "step": 4870 + }, + { + "epoch": 7.8593742127273645, + "grad_norm": 0.9896290302276611, + "learning_rate": 0.0001, + "loss": 66.7098, + "step": 4880 + }, + { + "epoch": 7.8754975563057386, + "grad_norm": 0.8828520774841309, + "learning_rate": 0.0001, + "loss": 66.0192, + "step": 4890 + }, + { + "epoch": 7.8916208998841135, + "grad_norm": 0.9093656539916992, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 4900 + }, + { + "epoch": 7.9077442434624885, + "grad_norm": 0.8665091395378113, + "learning_rate": 0.0001, + "loss": 66.5851, + "step": 4910 + }, + { + "epoch": 7.923867587040863, + "grad_norm": 0.8027398586273193, + "learning_rate": 0.0001, + "loss": 64.8607, + "step": 4920 + }, + { + "epoch": 7.9399909306192376, + "grad_norm": 0.7808436751365662, + "learning_rate": 0.0001, + "loss": 65.052, + "step": 4930 + }, + { + "epoch": 7.956114274197612, + "grad_norm": 0.7246070504188538, + "learning_rate": 0.0001, + "loss": 64.789, + "step": 4940 + }, + { + "epoch": 7.972237617775987, + "grad_norm": 1.0405027866363525, + "learning_rate": 0.0001, + "loss": 66.7735, + "step": 4950 + }, + { + "epoch": 7.988360961354361, + "grad_norm": 0.812921404838562, + "learning_rate": 0.0001, + "loss": 64.1286, + "step": 4960 + }, + { + "epoch": 8.003224668715674, + "grad_norm": 0.884215772151947, + "learning_rate": 0.0001, + "loss": 61.0812, + "step": 4970 + }, + { + "epoch": 8.01934801229405, + "grad_norm": 0.8519641757011414, + "learning_rate": 0.0001, + "loss": 66.2784, + "step": 4980 + }, + { + "epoch": 8.035471355872424, + "grad_norm": 0.8331199884414673, + "learning_rate": 0.0001, + "loss": 67.1405, + "step": 4990 + }, + { + "epoch": 8.0515946994508, + "grad_norm": 0.8568746447563171, + "learning_rate": 0.0001, + "loss": 66.7835, + "step": 5000 + }, + { + "epoch": 8.067718043029172, + "grad_norm": 1.170482873916626, + "learning_rate": 0.0001, + "loss": 64.2269, + "step": 5010 + }, + { + "epoch": 8.083841386607547, + "grad_norm": 0.9171113967895508, + "learning_rate": 0.0001, + "loss": 66.5567, + "step": 5020 + }, + { + "epoch": 8.099964730185922, + "grad_norm": 0.8988714814186096, + "learning_rate": 0.0001, + "loss": 66.0471, + "step": 5030 + }, + { + "epoch": 8.116088073764297, + "grad_norm": 0.7651671171188354, + "learning_rate": 0.0001, + "loss": 65.7185, + "step": 5040 + }, + { + "epoch": 8.132211417342672, + "grad_norm": 0.6890843510627747, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 5050 + }, + { + "epoch": 8.148334760921045, + "grad_norm": 0.7683818340301514, + "learning_rate": 0.0001, + "loss": 65.895, + "step": 5060 + }, + { + "epoch": 8.16445810449942, + "grad_norm": 0.9398573637008667, + "learning_rate": 0.0001, + "loss": 64.674, + "step": 5070 + }, + { + "epoch": 8.180581448077795, + "grad_norm": 0.7708531022071838, + "learning_rate": 0.0001, + "loss": 65.0637, + "step": 5080 + }, + { + "epoch": 8.19670479165617, + "grad_norm": 0.8840665221214294, + "learning_rate": 0.0001, + "loss": 65.6449, + "step": 5090 + }, + { + "epoch": 8.212828135234544, + "grad_norm": 0.8390524983406067, + "learning_rate": 0.0001, + "loss": 65.4738, + "step": 5100 + }, + { + "epoch": 8.228951478812919, + "grad_norm": 1.7559128999710083, + "learning_rate": 0.0001, + "loss": 68.911, + "step": 5110 + }, + { + "epoch": 8.245074822391294, + "grad_norm": 0.8293817043304443, + "learning_rate": 0.0001, + "loss": 65.2601, + "step": 5120 + }, + { + "epoch": 8.261198165969668, + "grad_norm": 0.8947123289108276, + "learning_rate": 0.0001, + "loss": 65.8611, + "step": 5130 + }, + { + "epoch": 8.277321509548042, + "grad_norm": 0.8526618480682373, + "learning_rate": 0.0001, + "loss": 65.559, + "step": 5140 + }, + { + "epoch": 8.293444853126417, + "grad_norm": 1.7423408031463623, + "learning_rate": 0.0001, + "loss": 64.5007, + "step": 5150 + }, + { + "epoch": 8.309568196704792, + "grad_norm": 1.0512502193450928, + "learning_rate": 0.0001, + "loss": 66.2593, + "step": 5160 + }, + { + "epoch": 8.325691540283167, + "grad_norm": 0.8129869103431702, + "learning_rate": 0.0001, + "loss": 64.7989, + "step": 5170 + }, + { + "epoch": 8.341814883861542, + "grad_norm": 0.852199912071228, + "learning_rate": 0.0001, + "loss": 65.9627, + "step": 5180 + }, + { + "epoch": 8.357938227439915, + "grad_norm": 0.8932474851608276, + "learning_rate": 0.0001, + "loss": 64.1035, + "step": 5190 + }, + { + "epoch": 8.37406157101829, + "grad_norm": 1.2631597518920898, + "learning_rate": 0.0001, + "loss": 65.3406, + "step": 5200 + }, + { + "epoch": 8.390184914596665, + "grad_norm": 0.9062619209289551, + "learning_rate": 0.0001, + "loss": 65.1841, + "step": 5210 + }, + { + "epoch": 8.40630825817504, + "grad_norm": 0.9153940081596375, + "learning_rate": 0.0001, + "loss": 66.3852, + "step": 5220 + }, + { + "epoch": 8.422431601753413, + "grad_norm": 0.8056164979934692, + "learning_rate": 0.0001, + "loss": 66.511, + "step": 5230 + }, + { + "epoch": 8.438554945331788, + "grad_norm": 0.7506036162376404, + "learning_rate": 0.0001, + "loss": 65.275, + "step": 5240 + }, + { + "epoch": 8.454678288910163, + "grad_norm": 0.7994081377983093, + "learning_rate": 0.0001, + "loss": 64.9946, + "step": 5250 + }, + { + "epoch": 8.470801632488538, + "grad_norm": 0.9887873530387878, + "learning_rate": 0.0001, + "loss": 67.2057, + "step": 5260 + }, + { + "epoch": 8.486924976066913, + "grad_norm": 0.8495044112205505, + "learning_rate": 0.0001, + "loss": 66.2481, + "step": 5270 + }, + { + "epoch": 8.503048319645286, + "grad_norm": 0.7353820204734802, + "learning_rate": 0.0001, + "loss": 65.6163, + "step": 5280 + }, + { + "epoch": 8.519171663223661, + "grad_norm": 0.795216977596283, + "learning_rate": 0.0001, + "loss": 64.808, + "step": 5290 + }, + { + "epoch": 8.535295006802036, + "grad_norm": 0.8121851682662964, + "learning_rate": 0.0001, + "loss": 65.9755, + "step": 5300 + }, + { + "epoch": 8.55141835038041, + "grad_norm": 0.8164243698120117, + "learning_rate": 0.0001, + "loss": 66.0583, + "step": 5310 + }, + { + "epoch": 8.567541693958784, + "grad_norm": 0.9893137216567993, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 5320 + }, + { + "epoch": 8.583665037537159, + "grad_norm": 1.3616434335708618, + "learning_rate": 0.0001, + "loss": 64.1444, + "step": 5330 + }, + { + "epoch": 8.599788381115534, + "grad_norm": 0.7002740502357483, + "learning_rate": 0.0001, + "loss": 65.0906, + "step": 5340 + }, + { + "epoch": 8.615911724693909, + "grad_norm": 0.838424563407898, + "learning_rate": 0.0001, + "loss": 65.4578, + "step": 5350 + }, + { + "epoch": 8.632035068272282, + "grad_norm": 0.988902747631073, + "learning_rate": 0.0001, + "loss": 64.3405, + "step": 5360 + }, + { + "epoch": 8.648158411850657, + "grad_norm": 0.7941252589225769, + "learning_rate": 0.0001, + "loss": 67.0146, + "step": 5370 + }, + { + "epoch": 8.664281755429032, + "grad_norm": 0.7727184295654297, + "learning_rate": 0.0001, + "loss": 64.8011, + "step": 5380 + }, + { + "epoch": 8.680405099007407, + "grad_norm": 0.9514367580413818, + "learning_rate": 0.0001, + "loss": 65.8165, + "step": 5390 + }, + { + "epoch": 8.69652844258578, + "grad_norm": 0.870544970035553, + "learning_rate": 0.0001, + "loss": 65.803, + "step": 5400 + }, + { + "epoch": 8.712651786164155, + "grad_norm": 0.7348082065582275, + "learning_rate": 0.0001, + "loss": 65.8409, + "step": 5410 + }, + { + "epoch": 8.72877512974253, + "grad_norm": 0.7543814778327942, + "learning_rate": 0.0001, + "loss": 65.5958, + "step": 5420 + }, + { + "epoch": 8.744898473320905, + "grad_norm": 0.7520368099212646, + "learning_rate": 0.0001, + "loss": 65.3323, + "step": 5430 + }, + { + "epoch": 8.76102181689928, + "grad_norm": 0.7610031366348267, + "learning_rate": 0.0001, + "loss": 65.4386, + "step": 5440 + }, + { + "epoch": 8.777145160477653, + "grad_norm": 0.8854930400848389, + "learning_rate": 0.0001, + "loss": 66.181, + "step": 5450 + }, + { + "epoch": 8.793268504056028, + "grad_norm": 0.7659342885017395, + "learning_rate": 0.0001, + "loss": 65.2586, + "step": 5460 + }, + { + "epoch": 8.809391847634403, + "grad_norm": 0.8592572212219238, + "learning_rate": 0.0001, + "loss": 66.3306, + "step": 5470 + }, + { + "epoch": 8.825515191212778, + "grad_norm": 0.8169172406196594, + "learning_rate": 0.0001, + "loss": 65.8283, + "step": 5480 + }, + { + "epoch": 8.841638534791151, + "grad_norm": 0.7380614876747131, + "learning_rate": 0.0001, + "loss": 65.8711, + "step": 5490 + }, + { + "epoch": 8.857761878369526, + "grad_norm": 0.9993331432342529, + "learning_rate": 0.0001, + "loss": 64.9378, + "step": 5500 + }, + { + "epoch": 8.873885221947901, + "grad_norm": 0.6252862811088562, + "learning_rate": 0.0001, + "loss": 64.1655, + "step": 5510 + }, + { + "epoch": 8.890008565526276, + "grad_norm": 0.8441852331161499, + "learning_rate": 0.0001, + "loss": 64.9795, + "step": 5520 + }, + { + "epoch": 8.906131909104651, + "grad_norm": 0.7075499892234802, + "learning_rate": 0.0001, + "loss": 65.5843, + "step": 5530 + }, + { + "epoch": 8.922255252683025, + "grad_norm": 0.7374144196510315, + "learning_rate": 0.0001, + "loss": 67.5064, + "step": 5540 + }, + { + "epoch": 8.9383785962614, + "grad_norm": 0.8073632121086121, + "learning_rate": 0.0001, + "loss": 67.0182, + "step": 5550 + }, + { + "epoch": 8.954501939839774, + "grad_norm": 0.7576195001602173, + "learning_rate": 0.0001, + "loss": 65.9139, + "step": 5560 + }, + { + "epoch": 8.97062528341815, + "grad_norm": 1.6479626893997192, + "learning_rate": 0.0001, + "loss": 66.3517, + "step": 5570 + }, + { + "epoch": 8.986748626996523, + "grad_norm": 0.846513569355011, + "learning_rate": 0.0001, + "loss": 64.2908, + "step": 5580 + }, + { + "epoch": 9.001612334357837, + "grad_norm": 0.9551037549972534, + "learning_rate": 0.0001, + "loss": 61.7312, + "step": 5590 + }, + { + "epoch": 9.017735677936212, + "grad_norm": 0.8250304460525513, + "learning_rate": 0.0001, + "loss": 66.3351, + "step": 5600 + }, + { + "epoch": 9.033859021514587, + "grad_norm": 0.7932960391044617, + "learning_rate": 0.0001, + "loss": 66.7753, + "step": 5610 + }, + { + "epoch": 9.04998236509296, + "grad_norm": 0.9839881658554077, + "learning_rate": 0.0001, + "loss": 64.5146, + "step": 5620 + }, + { + "epoch": 9.066105708671335, + "grad_norm": 0.7337850332260132, + "learning_rate": 0.0001, + "loss": 65.0544, + "step": 5630 + }, + { + "epoch": 9.08222905224971, + "grad_norm": 0.7496634721755981, + "learning_rate": 0.0001, + "loss": 64.1451, + "step": 5640 + }, + { + "epoch": 9.098352395828085, + "grad_norm": 0.9457297921180725, + "learning_rate": 0.0001, + "loss": 65.3036, + "step": 5650 + }, + { + "epoch": 9.11447573940646, + "grad_norm": 0.8315348625183105, + "learning_rate": 0.0001, + "loss": 66.0585, + "step": 5660 + }, + { + "epoch": 9.130599082984833, + "grad_norm": 0.9666195511817932, + "learning_rate": 0.0001, + "loss": 64.6802, + "step": 5670 + }, + { + "epoch": 9.146722426563208, + "grad_norm": 0.7416576147079468, + "learning_rate": 0.0001, + "loss": 65.1557, + "step": 5680 + }, + { + "epoch": 9.162845770141583, + "grad_norm": 0.7723799347877502, + "learning_rate": 0.0001, + "loss": 66.2871, + "step": 5690 + }, + { + "epoch": 9.178969113719958, + "grad_norm": 0.7461305856704712, + "learning_rate": 0.0001, + "loss": 65.5795, + "step": 5700 + }, + { + "epoch": 9.195092457298331, + "grad_norm": 0.6663411259651184, + "learning_rate": 0.0001, + "loss": 66.4094, + "step": 5710 + }, + { + "epoch": 9.211215800876706, + "grad_norm": 0.9340134859085083, + "learning_rate": 0.0001, + "loss": 63.5753, + "step": 5720 + }, + { + "epoch": 9.227339144455081, + "grad_norm": 0.7643963694572449, + "learning_rate": 0.0001, + "loss": 65.3359, + "step": 5730 + }, + { + "epoch": 9.243462488033456, + "grad_norm": 0.6931570768356323, + "learning_rate": 0.0001, + "loss": 65.0755, + "step": 5740 + }, + { + "epoch": 9.259585831611831, + "grad_norm": 0.9096988439559937, + "learning_rate": 0.0001, + "loss": 64.7983, + "step": 5750 + }, + { + "epoch": 9.275709175190205, + "grad_norm": 0.8360128402709961, + "learning_rate": 0.0001, + "loss": 64.3604, + "step": 5760 + }, + { + "epoch": 9.29183251876858, + "grad_norm": 0.9302591681480408, + "learning_rate": 0.0001, + "loss": 66.2754, + "step": 5770 + }, + { + "epoch": 9.307955862346954, + "grad_norm": 0.7000735998153687, + "learning_rate": 0.0001, + "loss": 65.2241, + "step": 5780 + }, + { + "epoch": 9.32407920592533, + "grad_norm": 0.7321175932884216, + "learning_rate": 0.0001, + "loss": 65.3558, + "step": 5790 + }, + { + "epoch": 9.340202549503703, + "grad_norm": 0.7948715090751648, + "learning_rate": 0.0001, + "loss": 64.2793, + "step": 5800 + }, + { + "epoch": 9.356325893082078, + "grad_norm": 0.7676129341125488, + "learning_rate": 0.0001, + "loss": 66.4778, + "step": 5810 + }, + { + "epoch": 9.372449236660453, + "grad_norm": 0.977236270904541, + "learning_rate": 0.0001, + "loss": 64.5587, + "step": 5820 + }, + { + "epoch": 9.388572580238828, + "grad_norm": 0.86808842420578, + "learning_rate": 0.0001, + "loss": 65.8026, + "step": 5830 + }, + { + "epoch": 9.4046959238172, + "grad_norm": 0.7455324530601501, + "learning_rate": 0.0001, + "loss": 65.066, + "step": 5840 + }, + { + "epoch": 9.420819267395576, + "grad_norm": 0.6784212589263916, + "learning_rate": 0.0001, + "loss": 65.4167, + "step": 5850 + }, + { + "epoch": 9.43694261097395, + "grad_norm": 1.033731460571289, + "learning_rate": 0.0001, + "loss": 66.4502, + "step": 5860 + }, + { + "epoch": 9.453065954552326, + "grad_norm": 0.7946107983589172, + "learning_rate": 0.0001, + "loss": 63.7567, + "step": 5870 + }, + { + "epoch": 9.4691892981307, + "grad_norm": 0.7826364636421204, + "learning_rate": 0.0001, + "loss": 65.4484, + "step": 5880 + }, + { + "epoch": 9.485312641709074, + "grad_norm": 0.7605016827583313, + "learning_rate": 0.0001, + "loss": 66.8459, + "step": 5890 + }, + { + "epoch": 9.501435985287449, + "grad_norm": 0.7383126020431519, + "learning_rate": 0.0001, + "loss": 64.6873, + "step": 5900 + }, + { + "epoch": 9.517559328865824, + "grad_norm": 0.8355686664581299, + "learning_rate": 0.0001, + "loss": 63.5145, + "step": 5910 + }, + { + "epoch": 9.533682672444199, + "grad_norm": 0.8827091455459595, + "learning_rate": 0.0001, + "loss": 67.2207, + "step": 5920 + }, + { + "epoch": 9.549806016022572, + "grad_norm": 0.6988142728805542, + "learning_rate": 0.0001, + "loss": 64.2289, + "step": 5930 + }, + { + "epoch": 9.565929359600947, + "grad_norm": 0.7609000205993652, + "learning_rate": 0.0001, + "loss": 63.9324, + "step": 5940 + }, + { + "epoch": 9.582052703179322, + "grad_norm": 0.8053853511810303, + "learning_rate": 0.0001, + "loss": 62.5564, + "step": 5950 + }, + { + "epoch": 9.598176046757697, + "grad_norm": 0.7684671878814697, + "learning_rate": 0.0001, + "loss": 66.2669, + "step": 5960 + }, + { + "epoch": 9.614299390336072, + "grad_norm": 0.8411065340042114, + "learning_rate": 0.0001, + "loss": 64.5087, + "step": 5970 + }, + { + "epoch": 9.630422733914445, + "grad_norm": 0.963498592376709, + "learning_rate": 0.0001, + "loss": 68.1736, + "step": 5980 + }, + { + "epoch": 9.64654607749282, + "grad_norm": 0.8715981841087341, + "learning_rate": 0.0001, + "loss": 64.1686, + "step": 5990 + }, + { + "epoch": 9.662669421071195, + "grad_norm": 0.8217800855636597, + "learning_rate": 0.0001, + "loss": 64.2211, + "step": 6000 + }, + { + "epoch": 9.67879276464957, + "grad_norm": 0.7824742197990417, + "learning_rate": 0.0001, + "loss": 65.9052, + "step": 6010 + }, + { + "epoch": 9.694916108227943, + "grad_norm": 0.7440009117126465, + "learning_rate": 0.0001, + "loss": 63.3749, + "step": 6020 + }, + { + "epoch": 9.711039451806318, + "grad_norm": 0.8759479522705078, + "learning_rate": 0.0001, + "loss": 63.4488, + "step": 6030 + }, + { + "epoch": 9.727162795384693, + "grad_norm": 3.662144184112549, + "learning_rate": 0.0001, + "loss": 62.8787, + "step": 6040 + }, + { + "epoch": 9.743286138963068, + "grad_norm": 1.007149577140808, + "learning_rate": 0.0001, + "loss": 66.0053, + "step": 6050 + }, + { + "epoch": 9.759409482541441, + "grad_norm": 0.8369242548942566, + "learning_rate": 0.0001, + "loss": 67.233, + "step": 6060 + }, + { + "epoch": 9.775532826119816, + "grad_norm": 0.8160434365272522, + "learning_rate": 0.0001, + "loss": 65.2089, + "step": 6070 + }, + { + "epoch": 9.791656169698191, + "grad_norm": 0.8482319712638855, + "learning_rate": 0.0001, + "loss": 65.0718, + "step": 6080 + }, + { + "epoch": 9.807779513276566, + "grad_norm": 0.8800129890441895, + "learning_rate": 0.0001, + "loss": 65.145, + "step": 6090 + }, + { + "epoch": 9.82390285685494, + "grad_norm": 0.8911405801773071, + "learning_rate": 0.0001, + "loss": 65.7952, + "step": 6100 + }, + { + "epoch": 9.840026200433314, + "grad_norm": 0.8108358979225159, + "learning_rate": 0.0001, + "loss": 65.1928, + "step": 6110 + }, + { + "epoch": 9.85614954401169, + "grad_norm": 0.8919743895530701, + "learning_rate": 0.0001, + "loss": 65.8129, + "step": 6120 + }, + { + "epoch": 9.872272887590064, + "grad_norm": 0.73152756690979, + "learning_rate": 0.0001, + "loss": 65.7141, + "step": 6130 + }, + { + "epoch": 9.88839623116844, + "grad_norm": 0.8583903312683105, + "learning_rate": 0.0001, + "loss": 64.8231, + "step": 6140 + }, + { + "epoch": 9.904519574746812, + "grad_norm": 0.798543393611908, + "learning_rate": 0.0001, + "loss": 64.5073, + "step": 6150 + }, + { + "epoch": 9.920642918325187, + "grad_norm": 0.8657358288764954, + "learning_rate": 0.0001, + "loss": 64.1207, + "step": 6160 + }, + { + "epoch": 9.936766261903562, + "grad_norm": 0.9324677586555481, + "learning_rate": 0.0001, + "loss": 64.7129, + "step": 6170 + }, + { + "epoch": 9.952889605481937, + "grad_norm": 0.7371154427528381, + "learning_rate": 0.0001, + "loss": 64.0251, + "step": 6180 + }, + { + "epoch": 9.96901294906031, + "grad_norm": 0.8273237347602844, + "learning_rate": 0.0001, + "loss": 65.8633, + "step": 6190 + }, + { + "epoch": 9.985136292638686, + "grad_norm": 0.7219365239143372, + "learning_rate": 0.0001, + "loss": 63.4029, + "step": 6200 + }, + { + "epoch": 10.0, + "grad_norm": 0.76242995262146, + "learning_rate": 0.0001, + "loss": 60.9994, + "step": 6210 + }, + { + "epoch": 10.016123343578375, + "grad_norm": 0.8677287697792053, + "learning_rate": 0.0001, + "loss": 65.9687, + "step": 6220 + }, + { + "epoch": 10.03224668715675, + "grad_norm": 0.7073521018028259, + "learning_rate": 0.0001, + "loss": 63.0083, + "step": 6230 + }, + { + "epoch": 10.048370030735123, + "grad_norm": 0.794535756111145, + "learning_rate": 0.0001, + "loss": 63.0616, + "step": 6240 + }, + { + "epoch": 10.064493374313498, + "grad_norm": 0.6974631547927856, + "learning_rate": 0.0001, + "loss": 64.6579, + "step": 6250 + }, + { + "epoch": 10.080616717891873, + "grad_norm": 0.8279150128364563, + "learning_rate": 0.0001, + "loss": 65.1069, + "step": 6260 + }, + { + "epoch": 10.096740061470248, + "grad_norm": 0.8682932257652283, + "learning_rate": 0.0001, + "loss": 64.5019, + "step": 6270 + }, + { + "epoch": 10.112863405048621, + "grad_norm": 1.0158190727233887, + "learning_rate": 0.0001, + "loss": 65.8854, + "step": 6280 + }, + { + "epoch": 10.128986748626996, + "grad_norm": 0.6911011934280396, + "learning_rate": 0.0001, + "loss": 62.4879, + "step": 6290 + }, + { + "epoch": 10.145110092205371, + "grad_norm": 0.7394022941589355, + "learning_rate": 0.0001, + "loss": 64.2604, + "step": 6300 + }, + { + "epoch": 10.161233435783746, + "grad_norm": 0.7941621541976929, + "learning_rate": 0.0001, + "loss": 64.5133, + "step": 6310 + }, + { + "epoch": 10.17735677936212, + "grad_norm": 1.0238994359970093, + "learning_rate": 0.0001, + "loss": 63.8594, + "step": 6320 + }, + { + "epoch": 10.193480122940494, + "grad_norm": 0.7332146763801575, + "learning_rate": 0.0001, + "loss": 66.5434, + "step": 6330 + }, + { + "epoch": 10.20960346651887, + "grad_norm": 0.9375078082084656, + "learning_rate": 0.0001, + "loss": 63.4793, + "step": 6340 + }, + { + "epoch": 10.225726810097244, + "grad_norm": 0.715931236743927, + "learning_rate": 0.0001, + "loss": 65.8812, + "step": 6350 + }, + { + "epoch": 10.24185015367562, + "grad_norm": 1.291536808013916, + "learning_rate": 0.0001, + "loss": 64.9125, + "step": 6360 + }, + { + "epoch": 10.257973497253992, + "grad_norm": 0.8942781686782837, + "learning_rate": 0.0001, + "loss": 67.1706, + "step": 6370 + }, + { + "epoch": 10.274096840832367, + "grad_norm": 0.8272124528884888, + "learning_rate": 0.0001, + "loss": 62.1721, + "step": 6380 + }, + { + "epoch": 10.290220184410742, + "grad_norm": 0.8191152215003967, + "learning_rate": 0.0001, + "loss": 64.8931, + "step": 6390 + }, + { + "epoch": 10.306343527989117, + "grad_norm": 1.7500241994857788, + "learning_rate": 0.0001, + "loss": 62.9693, + "step": 6400 + }, + { + "epoch": 10.32246687156749, + "grad_norm": 1.0159430503845215, + "learning_rate": 0.0001, + "loss": 63.0889, + "step": 6410 + }, + { + "epoch": 10.338590215145866, + "grad_norm": 0.7300348281860352, + "learning_rate": 0.0001, + "loss": 65.1742, + "step": 6420 + }, + { + "epoch": 10.35471355872424, + "grad_norm": 1.3132387399673462, + "learning_rate": 0.0001, + "loss": 64.8867, + "step": 6430 + }, + { + "epoch": 10.370836902302615, + "grad_norm": 0.7502451539039612, + "learning_rate": 0.0001, + "loss": 66.5094, + "step": 6440 + }, + { + "epoch": 10.38696024588099, + "grad_norm": 0.9408686757087708, + "learning_rate": 0.0001, + "loss": 65.1515, + "step": 6450 + }, + { + "epoch": 10.403083589459364, + "grad_norm": 0.769598126411438, + "learning_rate": 0.0001, + "loss": 65.1478, + "step": 6460 + }, + { + "epoch": 10.419206933037739, + "grad_norm": 0.9086162447929382, + "learning_rate": 0.0001, + "loss": 63.5263, + "step": 6470 + }, + { + "epoch": 10.435330276616114, + "grad_norm": 0.7899322509765625, + "learning_rate": 0.0001, + "loss": 63.3939, + "step": 6480 + }, + { + "epoch": 10.451453620194489, + "grad_norm": 0.9135434031486511, + "learning_rate": 0.0001, + "loss": 63.867, + "step": 6490 + }, + { + "epoch": 10.467576963772862, + "grad_norm": 0.9312737584114075, + "learning_rate": 0.0001, + "loss": 63.6939, + "step": 6500 + }, + { + "epoch": 10.483700307351237, + "grad_norm": 0.861764132976532, + "learning_rate": 0.0001, + "loss": 63.4523, + "step": 6510 + }, + { + "epoch": 10.499823650929612, + "grad_norm": 0.7056366801261902, + "learning_rate": 0.0001, + "loss": 63.2889, + "step": 6520 + }, + { + "epoch": 10.515946994507987, + "grad_norm": 0.7007710337638855, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 6530 + }, + { + "epoch": 10.53207033808636, + "grad_norm": 0.7091413736343384, + "learning_rate": 0.0001, + "loss": 63.4185, + "step": 6540 + }, + { + "epoch": 10.548193681664735, + "grad_norm": 0.7147234678268433, + "learning_rate": 0.0001, + "loss": 65.0726, + "step": 6550 + }, + { + "epoch": 10.56431702524311, + "grad_norm": 0.7954673767089844, + "learning_rate": 0.0001, + "loss": 65.0334, + "step": 6560 + }, + { + "epoch": 10.580440368821485, + "grad_norm": 0.725352942943573, + "learning_rate": 0.0001, + "loss": 62.6663, + "step": 6570 + }, + { + "epoch": 10.59656371239986, + "grad_norm": 0.9563508629798889, + "learning_rate": 0.0001, + "loss": 64.0705, + "step": 6580 + }, + { + "epoch": 10.612687055978233, + "grad_norm": 0.7778734564781189, + "learning_rate": 0.0001, + "loss": 63.7619, + "step": 6590 + }, + { + "epoch": 10.628810399556608, + "grad_norm": 0.8414369821548462, + "learning_rate": 0.0001, + "loss": 63.9238, + "step": 6600 + }, + { + "epoch": 10.644933743134983, + "grad_norm": 0.8070562481880188, + "learning_rate": 0.0001, + "loss": 63.9591, + "step": 6610 + }, + { + "epoch": 10.661057086713358, + "grad_norm": 0.6493734121322632, + "learning_rate": 0.0001, + "loss": 63.7237, + "step": 6620 + }, + { + "epoch": 10.677180430291731, + "grad_norm": 0.8263372778892517, + "learning_rate": 0.0001, + "loss": 64.2262, + "step": 6630 + }, + { + "epoch": 10.693303773870106, + "grad_norm": 1.0090227127075195, + "learning_rate": 0.0001, + "loss": 63.5469, + "step": 6640 + }, + { + "epoch": 10.709427117448481, + "grad_norm": 0.998319685459137, + "learning_rate": 0.0001, + "loss": 62.9113, + "step": 6650 + }, + { + "epoch": 10.725550461026856, + "grad_norm": 1.006489634513855, + "learning_rate": 0.0001, + "loss": 66.1458, + "step": 6660 + }, + { + "epoch": 10.741673804605231, + "grad_norm": 0.9575757384300232, + "learning_rate": 0.0001, + "loss": 63.0913, + "step": 6670 + }, + { + "epoch": 10.757797148183604, + "grad_norm": 0.6832238435745239, + "learning_rate": 0.0001, + "loss": 66.7501, + "step": 6680 + }, + { + "epoch": 10.773920491761979, + "grad_norm": 0.8075717687606812, + "learning_rate": 0.0001, + "loss": 64.2839, + "step": 6690 + }, + { + "epoch": 10.790043835340354, + "grad_norm": 0.9402624368667603, + "learning_rate": 0.0001, + "loss": 64.9296, + "step": 6700 + }, + { + "epoch": 10.806167178918729, + "grad_norm": 0.7681158185005188, + "learning_rate": 0.0001, + "loss": 63.5686, + "step": 6710 + }, + { + "epoch": 10.822290522497102, + "grad_norm": 0.8451285362243652, + "learning_rate": 0.0001, + "loss": 64.5404, + "step": 6720 + }, + { + "epoch": 10.838413866075477, + "grad_norm": 0.7278769016265869, + "learning_rate": 0.0001, + "loss": 64.1341, + "step": 6730 + }, + { + "epoch": 10.854537209653852, + "grad_norm": 0.8524075746536255, + "learning_rate": 0.0001, + "loss": 64.0483, + "step": 6740 + }, + { + "epoch": 10.870660553232227, + "grad_norm": 0.703598141670227, + "learning_rate": 0.0001, + "loss": 64.0072, + "step": 6750 + }, + { + "epoch": 10.8867838968106, + "grad_norm": 0.7740384936332703, + "learning_rate": 0.0001, + "loss": 63.4171, + "step": 6760 + }, + { + "epoch": 10.902907240388975, + "grad_norm": 0.8302356004714966, + "learning_rate": 0.0001, + "loss": 65.5159, + "step": 6770 + }, + { + "epoch": 10.91903058396735, + "grad_norm": 1.0289733409881592, + "learning_rate": 0.0001, + "loss": 64.9785, + "step": 6780 + }, + { + "epoch": 10.935153927545725, + "grad_norm": 0.6721987128257751, + "learning_rate": 0.0001, + "loss": 63.965, + "step": 6790 + }, + { + "epoch": 10.951277271124098, + "grad_norm": 0.7614948749542236, + "learning_rate": 0.0001, + "loss": 64.1375, + "step": 6800 + }, + { + "epoch": 10.967400614702473, + "grad_norm": 0.7342216372489929, + "learning_rate": 0.0001, + "loss": 64.4342, + "step": 6810 + }, + { + "epoch": 10.983523958280848, + "grad_norm": 0.8808079957962036, + "learning_rate": 0.0001, + "loss": 65.2864, + "step": 6820 + }, + { + "epoch": 10.999647301859223, + "grad_norm": 0.818477988243103, + "learning_rate": 0.0001, + "loss": 63.9931, + "step": 6830 + }, + { + "epoch": 11.014511009220538, + "grad_norm": 0.7867474555969238, + "learning_rate": 0.0001, + "loss": 59.2426, + "step": 6840 + }, + { + "epoch": 11.030634352798911, + "grad_norm": 0.6163678765296936, + "learning_rate": 0.0001, + "loss": 65.0149, + "step": 6850 + }, + { + "epoch": 11.046757696377286, + "grad_norm": 0.8334609866142273, + "learning_rate": 0.0001, + "loss": 63.5017, + "step": 6860 + }, + { + "epoch": 11.062881039955661, + "grad_norm": 0.7335264086723328, + "learning_rate": 0.0001, + "loss": 64.6307, + "step": 6870 + }, + { + "epoch": 11.079004383534036, + "grad_norm": 0.7723846435546875, + "learning_rate": 0.0001, + "loss": 64.316, + "step": 6880 + }, + { + "epoch": 11.09512772711241, + "grad_norm": 0.7972145080566406, + "learning_rate": 0.0001, + "loss": 63.1469, + "step": 6890 + }, + { + "epoch": 11.111251070690784, + "grad_norm": 0.7696133852005005, + "learning_rate": 0.0001, + "loss": 64.0338, + "step": 6900 + }, + { + "epoch": 11.127374414269159, + "grad_norm": 0.8972133994102478, + "learning_rate": 0.0001, + "loss": 65.5727, + "step": 6910 + }, + { + "epoch": 11.143497757847534, + "grad_norm": 0.6776295900344849, + "learning_rate": 0.0001, + "loss": 66.176, + "step": 6920 + }, + { + "epoch": 11.159621101425909, + "grad_norm": 0.7345874905586243, + "learning_rate": 0.0001, + "loss": 64.0949, + "step": 6930 + }, + { + "epoch": 11.175744445004282, + "grad_norm": 0.7988374829292297, + "learning_rate": 0.0001, + "loss": 64.7721, + "step": 6940 + }, + { + "epoch": 11.191867788582657, + "grad_norm": 0.7552387714385986, + "learning_rate": 0.0001, + "loss": 63.1053, + "step": 6950 + }, + { + "epoch": 11.207991132161032, + "grad_norm": 0.6388214230537415, + "learning_rate": 0.0001, + "loss": 63.7871, + "step": 6960 + }, + { + "epoch": 11.224114475739407, + "grad_norm": 0.6837960481643677, + "learning_rate": 0.0001, + "loss": 63.906, + "step": 6970 + }, + { + "epoch": 11.24023781931778, + "grad_norm": 0.7050679326057434, + "learning_rate": 0.0001, + "loss": 64.2803, + "step": 6980 + }, + { + "epoch": 11.256361162896155, + "grad_norm": 0.8619478940963745, + "learning_rate": 0.0001, + "loss": 62.8786, + "step": 6990 + }, + { + "epoch": 11.27248450647453, + "grad_norm": 0.9759813547134399, + "learning_rate": 0.0001, + "loss": 65.2299, + "step": 7000 + }, + { + "epoch": 11.288607850052905, + "grad_norm": 0.7584514021873474, + "learning_rate": 0.0001, + "loss": 61.4178, + "step": 7010 + }, + { + "epoch": 11.304731193631278, + "grad_norm": 0.8467874526977539, + "learning_rate": 0.0001, + "loss": 65.6229, + "step": 7020 + }, + { + "epoch": 11.320854537209653, + "grad_norm": 0.7121654152870178, + "learning_rate": 0.0001, + "loss": 63.9061, + "step": 7030 + }, + { + "epoch": 11.336977880788028, + "grad_norm": 0.8885557055473328, + "learning_rate": 0.0001, + "loss": 65.1436, + "step": 7040 + }, + { + "epoch": 11.353101224366403, + "grad_norm": 0.6320177912712097, + "learning_rate": 0.0001, + "loss": 62.8535, + "step": 7050 + }, + { + "epoch": 11.369224567944778, + "grad_norm": 0.9287629723548889, + "learning_rate": 0.0001, + "loss": 62.7407, + "step": 7060 + }, + { + "epoch": 11.385347911523152, + "grad_norm": 0.7705608606338501, + "learning_rate": 0.0001, + "loss": 63.469, + "step": 7070 + }, + { + "epoch": 11.401471255101526, + "grad_norm": 0.6687246561050415, + "learning_rate": 0.0001, + "loss": 62.722, + "step": 7080 + }, + { + "epoch": 11.417594598679901, + "grad_norm": 0.7774648666381836, + "learning_rate": 0.0001, + "loss": 63.1145, + "step": 7090 + }, + { + "epoch": 11.433717942258276, + "grad_norm": 0.7808363437652588, + "learning_rate": 0.0001, + "loss": 63.9342, + "step": 7100 + }, + { + "epoch": 11.44984128583665, + "grad_norm": 0.8033562302589417, + "learning_rate": 0.0001, + "loss": 65.7876, + "step": 7110 + }, + { + "epoch": 11.465964629415025, + "grad_norm": 0.8143486976623535, + "learning_rate": 0.0001, + "loss": 64.8782, + "step": 7120 + }, + { + "epoch": 11.4820879729934, + "grad_norm": 0.7855561971664429, + "learning_rate": 0.0001, + "loss": 63.4944, + "step": 7130 + }, + { + "epoch": 11.498211316571775, + "grad_norm": 0.7525351047515869, + "learning_rate": 0.0001, + "loss": 62.4088, + "step": 7140 + }, + { + "epoch": 11.514334660150148, + "grad_norm": 0.7193494439125061, + "learning_rate": 0.0001, + "loss": 64.2386, + "step": 7150 + }, + { + "epoch": 11.530458003728523, + "grad_norm": 0.892281174659729, + "learning_rate": 0.0001, + "loss": 63.8075, + "step": 7160 + }, + { + "epoch": 11.546581347306898, + "grad_norm": 0.720287561416626, + "learning_rate": 0.0001, + "loss": 63.3259, + "step": 7170 + }, + { + "epoch": 11.562704690885273, + "grad_norm": 1.4404689073562622, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 7180 + }, + { + "epoch": 11.578828034463648, + "grad_norm": 0.9082726240158081, + "learning_rate": 0.0001, + "loss": 64.6877, + "step": 7190 + }, + { + "epoch": 11.59495137804202, + "grad_norm": 0.7137670516967773, + "learning_rate": 0.0001, + "loss": 63.4832, + "step": 7200 + }, + { + "epoch": 11.611074721620396, + "grad_norm": 0.6848503947257996, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 7210 + }, + { + "epoch": 11.62719806519877, + "grad_norm": 0.7483687996864319, + "learning_rate": 0.0001, + "loss": 63.294, + "step": 7220 + }, + { + "epoch": 11.643321408777146, + "grad_norm": 0.935321569442749, + "learning_rate": 0.0001, + "loss": 62.8669, + "step": 7230 + }, + { + "epoch": 11.659444752355519, + "grad_norm": 0.8359326124191284, + "learning_rate": 0.0001, + "loss": 64.4482, + "step": 7240 + }, + { + "epoch": 11.675568095933894, + "grad_norm": 0.7838515043258667, + "learning_rate": 0.0001, + "loss": 64.1975, + "step": 7250 + }, + { + "epoch": 11.691691439512269, + "grad_norm": 0.9128443598747253, + "learning_rate": 0.0001, + "loss": 63.1822, + "step": 7260 + }, + { + "epoch": 11.707814783090644, + "grad_norm": 1.0545419454574585, + "learning_rate": 0.0001, + "loss": 63.3112, + "step": 7270 + }, + { + "epoch": 11.723938126669019, + "grad_norm": 0.8572486639022827, + "learning_rate": 0.0001, + "loss": 64.9605, + "step": 7280 + }, + { + "epoch": 11.740061470247392, + "grad_norm": 1.0894116163253784, + "learning_rate": 0.0001, + "loss": 64.3671, + "step": 7290 + }, + { + "epoch": 11.756184813825767, + "grad_norm": 0.7500355839729309, + "learning_rate": 0.0001, + "loss": 62.6381, + "step": 7300 + }, + { + "epoch": 11.772308157404142, + "grad_norm": 0.7315185070037842, + "learning_rate": 0.0001, + "loss": 66.1438, + "step": 7310 + }, + { + "epoch": 11.788431500982517, + "grad_norm": 0.8123629093170166, + "learning_rate": 0.0001, + "loss": 62.6012, + "step": 7320 + }, + { + "epoch": 11.80455484456089, + "grad_norm": 0.8072763085365295, + "learning_rate": 0.0001, + "loss": 63.4424, + "step": 7330 + }, + { + "epoch": 11.820678188139265, + "grad_norm": 1.1077874898910522, + "learning_rate": 0.0001, + "loss": 63.2812, + "step": 7340 + }, + { + "epoch": 11.83680153171764, + "grad_norm": 0.7116726636886597, + "learning_rate": 0.0001, + "loss": 63.6077, + "step": 7350 + }, + { + "epoch": 11.852924875296015, + "grad_norm": 0.8817790746688843, + "learning_rate": 0.0001, + "loss": 63.1938, + "step": 7360 + }, + { + "epoch": 11.86904821887439, + "grad_norm": 1.021730661392212, + "learning_rate": 0.0001, + "loss": 65.2215, + "step": 7370 + }, + { + "epoch": 11.885171562452763, + "grad_norm": 0.9871250987052917, + "learning_rate": 0.0001, + "loss": 64.6468, + "step": 7380 + }, + { + "epoch": 11.901294906031138, + "grad_norm": 1.3591430187225342, + "learning_rate": 0.0001, + "loss": 61.5546, + "step": 7390 + }, + { + "epoch": 11.917418249609513, + "grad_norm": 0.8114227056503296, + "learning_rate": 0.0001, + "loss": 61.8215, + "step": 7400 + }, + { + "epoch": 11.933541593187888, + "grad_norm": 0.7151288986206055, + "learning_rate": 0.0001, + "loss": 61.4856, + "step": 7410 + }, + { + "epoch": 11.949664936766261, + "grad_norm": 0.7494287490844727, + "learning_rate": 0.0001, + "loss": 63.6451, + "step": 7420 + }, + { + "epoch": 11.965788280344636, + "grad_norm": 0.7377745509147644, + "learning_rate": 0.0001, + "loss": 63.002, + "step": 7430 + }, + { + "epoch": 11.981911623923011, + "grad_norm": 0.6534659266471863, + "learning_rate": 0.0001, + "loss": 63.6404, + "step": 7440 + }, + { + "epoch": 11.998034967501386, + "grad_norm": 0.7548736333847046, + "learning_rate": 0.0001, + "loss": 64.1986, + "step": 7450 + }, + { + "epoch": 12.012898674862699, + "grad_norm": 0.7227553129196167, + "learning_rate": 0.0001, + "loss": 56.8327, + "step": 7460 + }, + { + "epoch": 12.029022018441074, + "grad_norm": 0.6348695158958435, + "learning_rate": 0.0001, + "loss": 62.4472, + "step": 7470 + }, + { + "epoch": 12.045145362019449, + "grad_norm": 0.8249921202659607, + "learning_rate": 0.0001, + "loss": 65.1877, + "step": 7480 + }, + { + "epoch": 12.061268705597824, + "grad_norm": 0.7152777314186096, + "learning_rate": 0.0001, + "loss": 64.2609, + "step": 7490 + }, + { + "epoch": 12.077392049176197, + "grad_norm": 0.7399184703826904, + "learning_rate": 0.0001, + "loss": 63.3997, + "step": 7500 + }, + { + "epoch": 12.093515392754572, + "grad_norm": 0.6972354650497437, + "learning_rate": 0.0001, + "loss": 61.3573, + "step": 7510 + }, + { + "epoch": 12.109638736332947, + "grad_norm": 0.7666720151901245, + "learning_rate": 0.0001, + "loss": 63.8438, + "step": 7520 + }, + { + "epoch": 12.125762079911322, + "grad_norm": 0.7309302687644958, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 7530 + }, + { + "epoch": 12.141885423489697, + "grad_norm": 1.3805177211761475, + "learning_rate": 0.0001, + "loss": 64.68, + "step": 7540 + }, + { + "epoch": 12.15800876706807, + "grad_norm": 1.0315922498703003, + "learning_rate": 0.0001, + "loss": 63.6372, + "step": 7550 + }, + { + "epoch": 12.174132110646445, + "grad_norm": 1.2012155055999756, + "learning_rate": 0.0001, + "loss": 62.5765, + "step": 7560 + }, + { + "epoch": 12.19025545422482, + "grad_norm": 0.7769846320152283, + "learning_rate": 0.0001, + "loss": 63.9737, + "step": 7570 + }, + { + "epoch": 12.206378797803195, + "grad_norm": 0.6587379574775696, + "learning_rate": 0.0001, + "loss": 63.382, + "step": 7580 + }, + { + "epoch": 12.222502141381568, + "grad_norm": 0.8309435248374939, + "learning_rate": 0.0001, + "loss": 64.4027, + "step": 7590 + }, + { + "epoch": 12.238625484959943, + "grad_norm": 0.8221393823623657, + "learning_rate": 0.0001, + "loss": 62.8561, + "step": 7600 + }, + { + "epoch": 12.254748828538318, + "grad_norm": 0.8629311323165894, + "learning_rate": 0.0001, + "loss": 64.3673, + "step": 7610 + }, + { + "epoch": 12.270872172116693, + "grad_norm": 0.698113739490509, + "learning_rate": 0.0001, + "loss": 63.6153, + "step": 7620 + }, + { + "epoch": 12.286995515695068, + "grad_norm": 0.72906094789505, + "learning_rate": 0.0001, + "loss": 64.8439, + "step": 7630 + }, + { + "epoch": 12.303118859273441, + "grad_norm": 0.7402384877204895, + "learning_rate": 0.0001, + "loss": 64.2103, + "step": 7640 + }, + { + "epoch": 12.319242202851816, + "grad_norm": 0.9972009062767029, + "learning_rate": 0.0001, + "loss": 63.803, + "step": 7650 + }, + { + "epoch": 12.335365546430191, + "grad_norm": 0.7852274775505066, + "learning_rate": 0.0001, + "loss": 64.8144, + "step": 7660 + }, + { + "epoch": 12.351488890008566, + "grad_norm": 0.7433444857597351, + "learning_rate": 0.0001, + "loss": 63.0123, + "step": 7670 + }, + { + "epoch": 12.36761223358694, + "grad_norm": 0.8548759818077087, + "learning_rate": 0.0001, + "loss": 63.2215, + "step": 7680 + }, + { + "epoch": 12.383735577165314, + "grad_norm": 0.6759565472602844, + "learning_rate": 0.0001, + "loss": 62.1246, + "step": 7690 + }, + { + "epoch": 12.39985892074369, + "grad_norm": 0.9079536199569702, + "learning_rate": 0.0001, + "loss": 64.1692, + "step": 7700 + }, + { + "epoch": 12.415982264322064, + "grad_norm": 0.6890071034431458, + "learning_rate": 0.0001, + "loss": 62.5602, + "step": 7710 + }, + { + "epoch": 12.432105607900438, + "grad_norm": 0.8096348643302917, + "learning_rate": 0.0001, + "loss": 61.0103, + "step": 7720 + }, + { + "epoch": 12.448228951478812, + "grad_norm": 0.7359873056411743, + "learning_rate": 0.0001, + "loss": 62.1914, + "step": 7730 + }, + { + "epoch": 12.464352295057187, + "grad_norm": 1.0701091289520264, + "learning_rate": 0.0001, + "loss": 61.7295, + "step": 7740 + }, + { + "epoch": 12.480475638635562, + "grad_norm": 0.791928768157959, + "learning_rate": 0.0001, + "loss": 62.6812, + "step": 7750 + }, + { + "epoch": 12.496598982213937, + "grad_norm": 0.7719446420669556, + "learning_rate": 0.0001, + "loss": 61.9114, + "step": 7760 + }, + { + "epoch": 12.51272232579231, + "grad_norm": 0.8720338344573975, + "learning_rate": 0.0001, + "loss": 64.8107, + "step": 7770 + }, + { + "epoch": 12.528845669370686, + "grad_norm": 0.9999920129776001, + "learning_rate": 0.0001, + "loss": 61.198, + "step": 7780 + }, + { + "epoch": 12.54496901294906, + "grad_norm": 0.710754930973053, + "learning_rate": 0.0001, + "loss": 64.1462, + "step": 7790 + }, + { + "epoch": 12.561092356527435, + "grad_norm": 0.7401363849639893, + "learning_rate": 0.0001, + "loss": 61.8988, + "step": 7800 + }, + { + "epoch": 12.577215700105809, + "grad_norm": 0.9183729290962219, + "learning_rate": 0.0001, + "loss": 62.6392, + "step": 7810 + }, + { + "epoch": 12.593339043684184, + "grad_norm": 0.7341399788856506, + "learning_rate": 0.0001, + "loss": 62.7068, + "step": 7820 + }, + { + "epoch": 12.609462387262559, + "grad_norm": 0.8066722750663757, + "learning_rate": 0.0001, + "loss": 64.0625, + "step": 7830 + }, + { + "epoch": 12.625585730840934, + "grad_norm": 0.6088835597038269, + "learning_rate": 0.0001, + "loss": 64.3208, + "step": 7840 + }, + { + "epoch": 12.641709074419307, + "grad_norm": 0.6969302892684937, + "learning_rate": 0.0001, + "loss": 62.5817, + "step": 7850 + }, + { + "epoch": 12.657832417997682, + "grad_norm": 0.82635098695755, + "learning_rate": 0.0001, + "loss": 62.0061, + "step": 7860 + }, + { + "epoch": 12.673955761576057, + "grad_norm": 0.7016922235488892, + "learning_rate": 0.0001, + "loss": 62.5166, + "step": 7870 + }, + { + "epoch": 12.690079105154432, + "grad_norm": 0.8035777807235718, + "learning_rate": 0.0001, + "loss": 63.6543, + "step": 7880 + }, + { + "epoch": 12.706202448732807, + "grad_norm": 0.8573809862136841, + "learning_rate": 0.0001, + "loss": 62.5258, + "step": 7890 + }, + { + "epoch": 12.72232579231118, + "grad_norm": 0.9497472643852234, + "learning_rate": 0.0001, + "loss": 63.1133, + "step": 7900 + }, + { + "epoch": 12.738449135889555, + "grad_norm": 0.7494457960128784, + "learning_rate": 0.0001, + "loss": 63.8157, + "step": 7910 + }, + { + "epoch": 12.75457247946793, + "grad_norm": 0.7657570242881775, + "learning_rate": 0.0001, + "loss": 62.6255, + "step": 7920 + }, + { + "epoch": 12.770695823046305, + "grad_norm": 0.7848380208015442, + "learning_rate": 0.0001, + "loss": 62.9095, + "step": 7930 + }, + { + "epoch": 12.786819166624678, + "grad_norm": 0.8386877775192261, + "learning_rate": 0.0001, + "loss": 63.2254, + "step": 7940 + }, + { + "epoch": 12.802942510203053, + "grad_norm": 0.6774776577949524, + "learning_rate": 0.0001, + "loss": 64.3929, + "step": 7950 + }, + { + "epoch": 12.819065853781428, + "grad_norm": 0.798547625541687, + "learning_rate": 0.0001, + "loss": 62.9884, + "step": 7960 + }, + { + "epoch": 12.835189197359803, + "grad_norm": 0.9340888261795044, + "learning_rate": 0.0001, + "loss": 62.6197, + "step": 7970 + }, + { + "epoch": 12.851312540938178, + "grad_norm": 0.8211432695388794, + "learning_rate": 0.0001, + "loss": 62.7756, + "step": 7980 + }, + { + "epoch": 12.867435884516551, + "grad_norm": 0.7612001299858093, + "learning_rate": 0.0001, + "loss": 62.0792, + "step": 7990 + }, + { + "epoch": 12.883559228094926, + "grad_norm": 0.7743386626243591, + "learning_rate": 0.0001, + "loss": 63.9203, + "step": 8000 + }, + { + "epoch": 12.899682571673301, + "grad_norm": 0.7419672608375549, + "learning_rate": 0.0001, + "loss": 62.1705, + "step": 8010 + }, + { + "epoch": 12.915805915251676, + "grad_norm": 0.6556256413459778, + "learning_rate": 0.0001, + "loss": 64.5863, + "step": 8020 + }, + { + "epoch": 12.93192925883005, + "grad_norm": 1.136175274848938, + "learning_rate": 0.0001, + "loss": 63.2619, + "step": 8030 + }, + { + "epoch": 12.948052602408424, + "grad_norm": 0.8631499409675598, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8040 + }, + { + "epoch": 12.9641759459868, + "grad_norm": 0.8386172652244568, + "learning_rate": 0.0001, + "loss": 63.0598, + "step": 8050 + }, + { + "epoch": 12.980299289565174, + "grad_norm": 0.8078306913375854, + "learning_rate": 0.0001, + "loss": 62.6877, + "step": 8060 + }, + { + "epoch": 12.996422633143549, + "grad_norm": 0.8486189842224121, + "learning_rate": 0.0001, + "loss": 62.8643, + "step": 8070 + }, + { + "epoch": 13.011286340504862, + "grad_norm": 0.8717893958091736, + "learning_rate": 0.0001, + "loss": 57.6947, + "step": 8080 + }, + { + "epoch": 13.027409684083237, + "grad_norm": 1.2023789882659912, + "learning_rate": 0.0001, + "loss": 60.4728, + "step": 8090 + }, + { + "epoch": 13.043533027661612, + "grad_norm": 0.6879557967185974, + "learning_rate": 0.0001, + "loss": 61.7643, + "step": 8100 + }, + { + "epoch": 13.059656371239987, + "grad_norm": 1.0918916463851929, + "learning_rate": 0.0001, + "loss": 65.3682, + "step": 8110 + }, + { + "epoch": 13.07577971481836, + "grad_norm": 0.85753333568573, + "learning_rate": 0.0001, + "loss": 63.0815, + "step": 8120 + }, + { + "epoch": 13.091903058396735, + "grad_norm": 0.6568583846092224, + "learning_rate": 0.0001, + "loss": 64.7167, + "step": 8130 + }, + { + "epoch": 13.10802640197511, + "grad_norm": 0.6178144812583923, + "learning_rate": 0.0001, + "loss": 62.4294, + "step": 8140 + }, + { + "epoch": 13.124149745553485, + "grad_norm": 0.7930784821510315, + "learning_rate": 0.0001, + "loss": 64.542, + "step": 8150 + }, + { + "epoch": 13.140273089131858, + "grad_norm": 0.7675037980079651, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8160 + }, + { + "epoch": 13.156396432710233, + "grad_norm": 0.786997377872467, + "learning_rate": 0.0001, + "loss": 64.1495, + "step": 8170 + }, + { + "epoch": 13.172519776288608, + "grad_norm": 0.7752560377120972, + "learning_rate": 0.0001, + "loss": 61.6483, + "step": 8180 + }, + { + "epoch": 13.188643119866983, + "grad_norm": 0.8455203771591187, + "learning_rate": 0.0001, + "loss": 63.3551, + "step": 8190 + }, + { + "epoch": 13.204766463445356, + "grad_norm": 0.7223097681999207, + "learning_rate": 0.0001, + "loss": 63.1473, + "step": 8200 + }, + { + "epoch": 13.220889807023731, + "grad_norm": 0.7688442468643188, + "learning_rate": 0.0001, + "loss": 62.6723, + "step": 8210 + }, + { + "epoch": 13.237013150602106, + "grad_norm": 0.7730395793914795, + "learning_rate": 0.0001, + "loss": 64.3424, + "step": 8220 + }, + { + "epoch": 13.253136494180481, + "grad_norm": 0.8479727506637573, + "learning_rate": 0.0001, + "loss": 61.4336, + "step": 8230 + }, + { + "epoch": 13.269259837758856, + "grad_norm": 0.9070560932159424, + "learning_rate": 0.0001, + "loss": 63.1221, + "step": 8240 + }, + { + "epoch": 13.28538318133723, + "grad_norm": 0.8924600481987, + "learning_rate": 0.0001, + "loss": 63.9172, + "step": 8250 + }, + { + "epoch": 13.301506524915604, + "grad_norm": 1.0688443183898926, + "learning_rate": 0.0001, + "loss": 63.7944, + "step": 8260 + }, + { + "epoch": 13.31762986849398, + "grad_norm": 0.6866253018379211, + "learning_rate": 0.0001, + "loss": 65.5847, + "step": 8270 + }, + { + "epoch": 13.333753212072354, + "grad_norm": 0.711171567440033, + "learning_rate": 0.0001, + "loss": 62.698, + "step": 8280 + }, + { + "epoch": 13.349876555650727, + "grad_norm": 0.8128448724746704, + "learning_rate": 0.0001, + "loss": 61.5992, + "step": 8290 + }, + { + "epoch": 13.365999899229102, + "grad_norm": 0.7483755350112915, + "learning_rate": 0.0001, + "loss": 63.782, + "step": 8300 + }, + { + "epoch": 13.382123242807477, + "grad_norm": 0.7872599363327026, + "learning_rate": 0.0001, + "loss": 65.4922, + "step": 8310 + }, + { + "epoch": 13.398246586385852, + "grad_norm": 0.6861543655395508, + "learning_rate": 0.0001, + "loss": 62.6946, + "step": 8320 + }, + { + "epoch": 13.414369929964227, + "grad_norm": 0.7192599773406982, + "learning_rate": 0.0001, + "loss": 62.894, + "step": 8330 + }, + { + "epoch": 13.4304932735426, + "grad_norm": 0.7665286064147949, + "learning_rate": 0.0001, + "loss": 62.6253, + "step": 8340 + }, + { + "epoch": 13.446616617120975, + "grad_norm": 0.7909309267997742, + "learning_rate": 0.0001, + "loss": 62.5355, + "step": 8350 + }, + { + "epoch": 13.46273996069935, + "grad_norm": 0.7339541912078857, + "learning_rate": 0.0001, + "loss": 62.8514, + "step": 8360 + }, + { + "epoch": 13.478863304277725, + "grad_norm": 0.9893004298210144, + "learning_rate": 0.0001, + "loss": 63.7751, + "step": 8370 + }, + { + "epoch": 13.494986647856098, + "grad_norm": 0.9109547734260559, + "learning_rate": 0.0001, + "loss": 62.6697, + "step": 8380 + }, + { + "epoch": 13.511109991434473, + "grad_norm": 0.6807034015655518, + "learning_rate": 0.0001, + "loss": 62.1418, + "step": 8390 + }, + { + "epoch": 13.527233335012848, + "grad_norm": 0.915533185005188, + "learning_rate": 0.0001, + "loss": 61.7645, + "step": 8400 + }, + { + "epoch": 13.543356678591223, + "grad_norm": 0.8662852048873901, + "learning_rate": 0.0001, + "loss": 62.5546, + "step": 8410 + }, + { + "epoch": 13.559480022169598, + "grad_norm": 0.8678365349769592, + "learning_rate": 0.0001, + "loss": 63.8795, + "step": 8420 + }, + { + "epoch": 13.575603365747972, + "grad_norm": 0.6808590888977051, + "learning_rate": 0.0001, + "loss": 64.158, + "step": 8430 + }, + { + "epoch": 13.591726709326347, + "grad_norm": 0.7896480560302734, + "learning_rate": 0.0001, + "loss": 62.1052, + "step": 8440 + }, + { + "epoch": 13.607850052904721, + "grad_norm": 0.8973979949951172, + "learning_rate": 0.0001, + "loss": 64.478, + "step": 8450 + }, + { + "epoch": 13.623973396483096, + "grad_norm": 0.8228474259376526, + "learning_rate": 0.0001, + "loss": 61.2416, + "step": 8460 + }, + { + "epoch": 13.64009674006147, + "grad_norm": 0.9414492845535278, + "learning_rate": 0.0001, + "loss": 62.816, + "step": 8470 + }, + { + "epoch": 13.656220083639845, + "grad_norm": 0.9369436502456665, + "learning_rate": 0.0001, + "loss": 63.4951, + "step": 8480 + }, + { + "epoch": 13.67234342721822, + "grad_norm": 0.7152880430221558, + "learning_rate": 0.0001, + "loss": 62.4737, + "step": 8490 + }, + { + "epoch": 13.688466770796595, + "grad_norm": 0.7364594340324402, + "learning_rate": 0.0001, + "loss": 63.8688, + "step": 8500 + }, + { + "epoch": 13.704590114374968, + "grad_norm": 0.8499143123626709, + "learning_rate": 0.0001, + "loss": 64.5654, + "step": 8510 + }, + { + "epoch": 13.720713457953343, + "grad_norm": 0.7699156999588013, + "learning_rate": 0.0001, + "loss": 63.3726, + "step": 8520 + }, + { + "epoch": 13.736836801531718, + "grad_norm": 0.9146804809570312, + "learning_rate": 0.0001, + "loss": 63.6565, + "step": 8530 + }, + { + "epoch": 13.752960145110093, + "grad_norm": 0.773318350315094, + "learning_rate": 0.0001, + "loss": 65.0719, + "step": 8540 + }, + { + "epoch": 13.769083488688466, + "grad_norm": 0.6158450245857239, + "learning_rate": 0.0001, + "loss": 60.7291, + "step": 8550 + }, + { + "epoch": 13.78520683226684, + "grad_norm": 0.8053523302078247, + "learning_rate": 0.0001, + "loss": 62.5622, + "step": 8560 + }, + { + "epoch": 13.801330175845216, + "grad_norm": 0.8983457684516907, + "learning_rate": 0.0001, + "loss": 64.2921, + "step": 8570 + }, + { + "epoch": 13.81745351942359, + "grad_norm": 0.8214695453643799, + "learning_rate": 0.0001, + "loss": 62.2649, + "step": 8580 + }, + { + "epoch": 13.833576863001966, + "grad_norm": 0.8079172372817993, + "learning_rate": 0.0001, + "loss": 64.0828, + "step": 8590 + }, + { + "epoch": 13.849700206580339, + "grad_norm": 1.126568078994751, + "learning_rate": 0.0001, + "loss": 62.3927, + "step": 8600 + }, + { + "epoch": 13.865823550158714, + "grad_norm": 0.7597793936729431, + "learning_rate": 0.0001, + "loss": 62.5922, + "step": 8610 + }, + { + "epoch": 13.881946893737089, + "grad_norm": 0.671531617641449, + "learning_rate": 0.0001, + "loss": 61.9885, + "step": 8620 + }, + { + "epoch": 13.898070237315464, + "grad_norm": 0.823423445224762, + "learning_rate": 0.0001, + "loss": 62.0632, + "step": 8630 + }, + { + "epoch": 13.914193580893837, + "grad_norm": 0.7694658637046814, + "learning_rate": 0.0001, + "loss": 64.5485, + "step": 8640 + }, + { + "epoch": 13.930316924472212, + "grad_norm": 0.7394612431526184, + "learning_rate": 0.0001, + "loss": 61.9626, + "step": 8650 + }, + { + "epoch": 13.946440268050587, + "grad_norm": 0.724054217338562, + "learning_rate": 0.0001, + "loss": 62.3761, + "step": 8660 + }, + { + "epoch": 13.962563611628962, + "grad_norm": 0.7160128951072693, + "learning_rate": 0.0001, + "loss": 60.5142, + "step": 8670 + }, + { + "epoch": 13.978686955207337, + "grad_norm": 1.0045746564865112, + "learning_rate": 0.0001, + "loss": 62.7059, + "step": 8680 + }, + { + "epoch": 13.99481029878571, + "grad_norm": 0.6980776190757751, + "learning_rate": 0.0001, + "loss": 62.9427, + "step": 8690 + }, + { + "epoch": 14.009674006147025, + "grad_norm": 0.810961127281189, + "learning_rate": 0.0001, + "loss": 58.7552, + "step": 8700 + }, + { + "epoch": 14.0257973497254, + "grad_norm": 0.9210551381111145, + "learning_rate": 0.0001, + "loss": 63.7412, + "step": 8710 + }, + { + "epoch": 14.041920693303775, + "grad_norm": 0.7805785536766052, + "learning_rate": 0.0001, + "loss": 61.477, + "step": 8720 + }, + { + "epoch": 14.058044036882148, + "grad_norm": 0.8528208136558533, + "learning_rate": 0.0001, + "loss": 62.6746, + "step": 8730 + }, + { + "epoch": 14.074167380460523, + "grad_norm": 0.7467619776725769, + "learning_rate": 0.0001, + "loss": 63.8744, + "step": 8740 + }, + { + "epoch": 14.090290724038898, + "grad_norm": 0.6434402465820312, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 8750 + }, + { + "epoch": 14.106414067617273, + "grad_norm": 0.7217757701873779, + "learning_rate": 0.0001, + "loss": 64.0428, + "step": 8760 + }, + { + "epoch": 14.122537411195646, + "grad_norm": 0.8665125966072083, + "learning_rate": 0.0001, + "loss": 62.2772, + "step": 8770 + }, + { + "epoch": 14.13866075477402, + "grad_norm": 0.7868321537971497, + "learning_rate": 0.0001, + "loss": 62.0509, + "step": 8780 + }, + { + "epoch": 14.154784098352396, + "grad_norm": 0.8716760873794556, + "learning_rate": 0.0001, + "loss": 61.6021, + "step": 8790 + }, + { + "epoch": 14.17090744193077, + "grad_norm": 1.0922521352767944, + "learning_rate": 0.0001, + "loss": 64.4197, + "step": 8800 + }, + { + "epoch": 14.187030785509146, + "grad_norm": 0.6545577049255371, + "learning_rate": 0.0001, + "loss": 62.9259, + "step": 8810 + }, + { + "epoch": 14.203154129087519, + "grad_norm": 0.7634063959121704, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 8820 + }, + { + "epoch": 14.219277472665894, + "grad_norm": 0.9075300097465515, + "learning_rate": 0.0001, + "loss": 62.9128, + "step": 8830 + }, + { + "epoch": 14.235400816244269, + "grad_norm": 3.475320816040039, + "learning_rate": 0.0001, + "loss": 62.4841, + "step": 8840 + }, + { + "epoch": 14.251524159822644, + "grad_norm": 0.6839455962181091, + "learning_rate": 0.0001, + "loss": 61.5898, + "step": 8850 + }, + { + "epoch": 14.267647503401017, + "grad_norm": 0.8024231791496277, + "learning_rate": 0.0001, + "loss": 61.1875, + "step": 8860 + }, + { + "epoch": 14.283770846979392, + "grad_norm": 0.8948174118995667, + "learning_rate": 0.0001, + "loss": 62.3386, + "step": 8870 + }, + { + "epoch": 14.299894190557767, + "grad_norm": 0.7571985721588135, + "learning_rate": 0.0001, + "loss": 60.579, + "step": 8880 + }, + { + "epoch": 14.316017534136142, + "grad_norm": 1.432371973991394, + "learning_rate": 0.0001, + "loss": 62.4405, + "step": 8890 + }, + { + "epoch": 14.332140877714515, + "grad_norm": 0.6696875691413879, + "learning_rate": 0.0001, + "loss": 62.9984, + "step": 8900 + }, + { + "epoch": 14.34826422129289, + "grad_norm": 0.749396026134491, + "learning_rate": 0.0001, + "loss": 61.492, + "step": 8910 + }, + { + "epoch": 14.364387564871265, + "grad_norm": 0.664193332195282, + "learning_rate": 0.0001, + "loss": 62.9893, + "step": 8920 + }, + { + "epoch": 14.38051090844964, + "grad_norm": 0.7032697796821594, + "learning_rate": 0.0001, + "loss": 61.1682, + "step": 8930 + }, + { + "epoch": 14.396634252028015, + "grad_norm": 0.813119649887085, + "learning_rate": 0.0001, + "loss": 62.906, + "step": 8940 + }, + { + "epoch": 14.412757595606388, + "grad_norm": 0.7699596881866455, + "learning_rate": 0.0001, + "loss": 64.8087, + "step": 8950 + }, + { + "epoch": 14.428880939184763, + "grad_norm": 0.6424354910850525, + "learning_rate": 0.0001, + "loss": 62.2418, + "step": 8960 + }, + { + "epoch": 14.445004282763138, + "grad_norm": 0.8741731643676758, + "learning_rate": 0.0001, + "loss": 62.656, + "step": 8970 + }, + { + "epoch": 14.461127626341513, + "grad_norm": 0.710617184638977, + "learning_rate": 0.0001, + "loss": 60.8979, + "step": 8980 + }, + { + "epoch": 14.477250969919886, + "grad_norm": 0.6493995785713196, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 8990 + }, + { + "epoch": 14.493374313498261, + "grad_norm": 0.6655835509300232, + "learning_rate": 0.0001, + "loss": 64.151, + "step": 9000 + }, + { + "epoch": 14.509497657076636, + "grad_norm": 0.8140881657600403, + "learning_rate": 0.0001, + "loss": 62.4447, + "step": 9010 + }, + { + "epoch": 14.525621000655011, + "grad_norm": 0.7288498878479004, + "learning_rate": 0.0001, + "loss": 62.5189, + "step": 9020 + }, + { + "epoch": 14.541744344233386, + "grad_norm": 0.6972017884254456, + "learning_rate": 0.0001, + "loss": 61.5441, + "step": 9030 + }, + { + "epoch": 14.55786768781176, + "grad_norm": 0.7444366216659546, + "learning_rate": 0.0001, + "loss": 62.3233, + "step": 9040 + }, + { + "epoch": 14.573991031390134, + "grad_norm": 0.6602845191955566, + "learning_rate": 0.0001, + "loss": 61.4176, + "step": 9050 + }, + { + "epoch": 14.59011437496851, + "grad_norm": 0.7149770855903625, + "learning_rate": 0.0001, + "loss": 62.159, + "step": 9060 + }, + { + "epoch": 14.606237718546884, + "grad_norm": 0.6975772976875305, + "learning_rate": 0.0001, + "loss": 63.1857, + "step": 9070 + }, + { + "epoch": 14.622361062125258, + "grad_norm": 0.7835262417793274, + "learning_rate": 0.0001, + "loss": 61.6795, + "step": 9080 + }, + { + "epoch": 14.638484405703633, + "grad_norm": 5.790656566619873, + "learning_rate": 0.0001, + "loss": 62.7087, + "step": 9090 + }, + { + "epoch": 14.654607749282007, + "grad_norm": 1.0082288980484009, + "learning_rate": 0.0001, + "loss": 61.6808, + "step": 9100 + }, + { + "epoch": 14.670731092860382, + "grad_norm": 0.7758098840713501, + "learning_rate": 0.0001, + "loss": 61.0734, + "step": 9110 + }, + { + "epoch": 14.686854436438756, + "grad_norm": 0.7841722965240479, + "learning_rate": 0.0001, + "loss": 62.8882, + "step": 9120 + }, + { + "epoch": 14.70297778001713, + "grad_norm": 0.7695890069007874, + "learning_rate": 0.0001, + "loss": 60.7811, + "step": 9130 + }, + { + "epoch": 14.719101123595506, + "grad_norm": 0.7444761395454407, + "learning_rate": 0.0001, + "loss": 61.692, + "step": 9140 + }, + { + "epoch": 14.73522446717388, + "grad_norm": 0.7036821246147156, + "learning_rate": 0.0001, + "loss": 63.1698, + "step": 9150 + }, + { + "epoch": 14.751347810752256, + "grad_norm": 0.7891719937324524, + "learning_rate": 0.0001, + "loss": 63.8575, + "step": 9160 + }, + { + "epoch": 14.767471154330629, + "grad_norm": 0.6371109485626221, + "learning_rate": 0.0001, + "loss": 61.3358, + "step": 9170 + }, + { + "epoch": 14.783594497909004, + "grad_norm": 0.6551769375801086, + "learning_rate": 0.0001, + "loss": 62.2705, + "step": 9180 + }, + { + "epoch": 14.799717841487379, + "grad_norm": 0.6922310590744019, + "learning_rate": 0.0001, + "loss": 61.6442, + "step": 9190 + }, + { + "epoch": 14.815841185065754, + "grad_norm": 0.9615464210510254, + "learning_rate": 0.0001, + "loss": 60.932, + "step": 9200 + }, + { + "epoch": 14.831964528644127, + "grad_norm": 0.6367932558059692, + "learning_rate": 0.0001, + "loss": 61.3475, + "step": 9210 + }, + { + "epoch": 14.848087872222502, + "grad_norm": 0.5634505748748779, + "learning_rate": 0.0001, + "loss": 61.7562, + "step": 9220 + }, + { + "epoch": 14.864211215800877, + "grad_norm": 0.7021980285644531, + "learning_rate": 0.0001, + "loss": 62.0672, + "step": 9230 + }, + { + "epoch": 14.880334559379252, + "grad_norm": 0.8832064867019653, + "learning_rate": 0.0001, + "loss": 60.9882, + "step": 9240 + }, + { + "epoch": 14.896457902957625, + "grad_norm": 0.9043561220169067, + "learning_rate": 0.0001, + "loss": 63.0968, + "step": 9250 + }, + { + "epoch": 14.912581246536, + "grad_norm": 0.8958032131195068, + "learning_rate": 0.0001, + "loss": 61.5431, + "step": 9260 + }, + { + "epoch": 14.928704590114375, + "grad_norm": 0.7181615829467773, + "learning_rate": 0.0001, + "loss": 62.3626, + "step": 9270 + }, + { + "epoch": 14.94482793369275, + "grad_norm": 0.7541075348854065, + "learning_rate": 0.0001, + "loss": 61.7186, + "step": 9280 + }, + { + "epoch": 14.960951277271125, + "grad_norm": 1.5627169609069824, + "learning_rate": 0.0001, + "loss": 61.654, + "step": 9290 + }, + { + "epoch": 14.977074620849498, + "grad_norm": 0.7126879096031189, + "learning_rate": 0.0001, + "loss": 61.6806, + "step": 9300 + }, + { + "epoch": 14.993197964427873, + "grad_norm": 0.7397864460945129, + "learning_rate": 0.0001, + "loss": 61.2669, + "step": 9310 + }, + { + "epoch": 15.008061671789187, + "grad_norm": 0.7282766699790955, + "learning_rate": 0.0001, + "loss": 56.2289, + "step": 9320 + }, + { + "epoch": 15.024185015367562, + "grad_norm": 0.8346580862998962, + "learning_rate": 0.0001, + "loss": 61.4335, + "step": 9330 + }, + { + "epoch": 15.040308358945936, + "grad_norm": 0.9117271304130554, + "learning_rate": 0.0001, + "loss": 60.4636, + "step": 9340 + }, + { + "epoch": 15.05643170252431, + "grad_norm": 0.8483272194862366, + "learning_rate": 0.0001, + "loss": 61.3003, + "step": 9350 + }, + { + "epoch": 15.072555046102686, + "grad_norm": 1.1898932456970215, + "learning_rate": 0.0001, + "loss": 62.7169, + "step": 9360 + }, + { + "epoch": 15.08867838968106, + "grad_norm": 0.7258167862892151, + "learning_rate": 0.0001, + "loss": 63.5285, + "step": 9370 + }, + { + "epoch": 15.104801733259436, + "grad_norm": 0.7810865044593811, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 9380 + }, + { + "epoch": 15.120925076837809, + "grad_norm": 1.0359688997268677, + "learning_rate": 0.0001, + "loss": 62.5435, + "step": 9390 + }, + { + "epoch": 15.137048420416184, + "grad_norm": 0.7371546030044556, + "learning_rate": 0.0001, + "loss": 60.5583, + "step": 9400 + }, + { + "epoch": 15.153171763994559, + "grad_norm": 0.7578926086425781, + "learning_rate": 0.0001, + "loss": 61.2853, + "step": 9410 + }, + { + "epoch": 15.169295107572934, + "grad_norm": 0.6161713004112244, + "learning_rate": 0.0001, + "loss": 62.9725, + "step": 9420 + }, + { + "epoch": 15.185418451151307, + "grad_norm": 0.8325598835945129, + "learning_rate": 0.0001, + "loss": 62.857, + "step": 9430 + }, + { + "epoch": 15.201541794729682, + "grad_norm": 1.1438454389572144, + "learning_rate": 0.0001, + "loss": 61.8167, + "step": 9440 + }, + { + "epoch": 15.217665138308057, + "grad_norm": 1.1820837259292603, + "learning_rate": 0.0001, + "loss": 60.9899, + "step": 9450 + }, + { + "epoch": 15.233788481886432, + "grad_norm": 0.6057641506195068, + "learning_rate": 0.0001, + "loss": 62.5144, + "step": 9460 + }, + { + "epoch": 15.249911825464805, + "grad_norm": 0.6661948561668396, + "learning_rate": 0.0001, + "loss": 62.0835, + "step": 9470 + }, + { + "epoch": 15.26603516904318, + "grad_norm": 0.7623168230056763, + "learning_rate": 0.0001, + "loss": 61.9491, + "step": 9480 + }, + { + "epoch": 15.282158512621555, + "grad_norm": 0.6784029006958008, + "learning_rate": 0.0001, + "loss": 63.6006, + "step": 9490 + }, + { + "epoch": 15.29828185619993, + "grad_norm": 0.9433781504631042, + "learning_rate": 0.0001, + "loss": 61.6677, + "step": 9500 + }, + { + "epoch": 15.314405199778305, + "grad_norm": 0.6544079184532166, + "learning_rate": 0.0001, + "loss": 63.1439, + "step": 9510 + }, + { + "epoch": 15.330528543356678, + "grad_norm": 0.7979472279548645, + "learning_rate": 0.0001, + "loss": 62.719, + "step": 9520 + }, + { + "epoch": 15.346651886935053, + "grad_norm": 0.6641252040863037, + "learning_rate": 0.0001, + "loss": 63.2543, + "step": 9530 + }, + { + "epoch": 15.362775230513428, + "grad_norm": 0.7780525088310242, + "learning_rate": 0.0001, + "loss": 62.5659, + "step": 9540 + }, + { + "epoch": 15.378898574091803, + "grad_norm": 0.842349648475647, + "learning_rate": 0.0001, + "loss": 60.041, + "step": 9550 + }, + { + "epoch": 15.395021917670176, + "grad_norm": 0.8991017937660217, + "learning_rate": 0.0001, + "loss": 62.9075, + "step": 9560 + }, + { + "epoch": 15.411145261248551, + "grad_norm": 0.6734052300453186, + "learning_rate": 0.0001, + "loss": 60.8647, + "step": 9570 + }, + { + "epoch": 15.427268604826926, + "grad_norm": 0.7585113644599915, + "learning_rate": 0.0001, + "loss": 62.105, + "step": 9580 + }, + { + "epoch": 15.443391948405301, + "grad_norm": 0.7915840148925781, + "learning_rate": 0.0001, + "loss": 62.2741, + "step": 9590 + }, + { + "epoch": 15.459515291983674, + "grad_norm": 0.8676218390464783, + "learning_rate": 0.0001, + "loss": 62.4548, + "step": 9600 + }, + { + "epoch": 15.47563863556205, + "grad_norm": 0.7698650360107422, + "learning_rate": 0.0001, + "loss": 61.3412, + "step": 9610 + }, + { + "epoch": 15.491761979140424, + "grad_norm": 0.6600908637046814, + "learning_rate": 0.0001, + "loss": 61.1561, + "step": 9620 + }, + { + "epoch": 15.5078853227188, + "grad_norm": 0.5995001792907715, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 9630 + }, + { + "epoch": 15.524008666297174, + "grad_norm": 0.7258557677268982, + "learning_rate": 0.0001, + "loss": 62.4659, + "step": 9640 + }, + { + "epoch": 15.540132009875547, + "grad_norm": 0.6562618613243103, + "learning_rate": 0.0001, + "loss": 63.288, + "step": 9650 + }, + { + "epoch": 15.556255353453922, + "grad_norm": 0.9224995374679565, + "learning_rate": 0.0001, + "loss": 61.4861, + "step": 9660 + }, + { + "epoch": 15.572378697032297, + "grad_norm": 0.7053484916687012, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 9670 + }, + { + "epoch": 15.588502040610672, + "grad_norm": 1.095229148864746, + "learning_rate": 0.0001, + "loss": 62.1542, + "step": 9680 + }, + { + "epoch": 15.604625384189045, + "grad_norm": 0.7563759088516235, + "learning_rate": 0.0001, + "loss": 61.1644, + "step": 9690 + }, + { + "epoch": 15.62074872776742, + "grad_norm": 0.6508835554122925, + "learning_rate": 0.0001, + "loss": 60.1374, + "step": 9700 + }, + { + "epoch": 15.636872071345795, + "grad_norm": 0.664799153804779, + "learning_rate": 0.0001, + "loss": 60.2331, + "step": 9710 + }, + { + "epoch": 15.65299541492417, + "grad_norm": 0.6525771617889404, + "learning_rate": 0.0001, + "loss": 60.6142, + "step": 9720 + }, + { + "epoch": 15.669118758502545, + "grad_norm": 0.7395381331443787, + "learning_rate": 0.0001, + "loss": 63.5623, + "step": 9730 + }, + { + "epoch": 15.685242102080919, + "grad_norm": 0.7063724398612976, + "learning_rate": 0.0001, + "loss": 63.7133, + "step": 9740 + }, + { + "epoch": 15.701365445659293, + "grad_norm": 0.7918921113014221, + "learning_rate": 0.0001, + "loss": 62.5957, + "step": 9750 + }, + { + "epoch": 15.717488789237668, + "grad_norm": 0.8135486245155334, + "learning_rate": 0.0001, + "loss": 62.9146, + "step": 9760 + }, + { + "epoch": 15.733612132816043, + "grad_norm": 0.8422091007232666, + "learning_rate": 0.0001, + "loss": 61.1466, + "step": 9770 + }, + { + "epoch": 15.749735476394417, + "grad_norm": 0.6329447627067566, + "learning_rate": 0.0001, + "loss": 61.7247, + "step": 9780 + }, + { + "epoch": 15.765858819972792, + "grad_norm": 1.0627039670944214, + "learning_rate": 0.0001, + "loss": 60.6789, + "step": 9790 + }, + { + "epoch": 15.781982163551167, + "grad_norm": 0.7316949367523193, + "learning_rate": 0.0001, + "loss": 61.9896, + "step": 9800 + }, + { + "epoch": 15.798105507129542, + "grad_norm": 0.6968954205513, + "learning_rate": 0.0001, + "loss": 60.8571, + "step": 9810 + }, + { + "epoch": 15.814228850707915, + "grad_norm": 0.7226986289024353, + "learning_rate": 0.0001, + "loss": 62.2664, + "step": 9820 + }, + { + "epoch": 15.83035219428629, + "grad_norm": 0.6542951464653015, + "learning_rate": 0.0001, + "loss": 61.3316, + "step": 9830 + }, + { + "epoch": 15.846475537864665, + "grad_norm": 0.7662289142608643, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 9840 + }, + { + "epoch": 15.86259888144304, + "grad_norm": 0.7060043215751648, + "learning_rate": 0.0001, + "loss": 62.2647, + "step": 9850 + }, + { + "epoch": 15.878722225021415, + "grad_norm": 0.6948843002319336, + "learning_rate": 0.0001, + "loss": 62.8678, + "step": 9860 + }, + { + "epoch": 15.894845568599788, + "grad_norm": 0.7787327766418457, + "learning_rate": 0.0001, + "loss": 62.9154, + "step": 9870 + }, + { + "epoch": 15.910968912178163, + "grad_norm": 0.7077026963233948, + "learning_rate": 0.0001, + "loss": 62.9537, + "step": 9880 + }, + { + "epoch": 15.927092255756538, + "grad_norm": 0.9957109093666077, + "learning_rate": 0.0001, + "loss": 60.9461, + "step": 9890 + }, + { + "epoch": 15.943215599334913, + "grad_norm": 0.9767161011695862, + "learning_rate": 0.0001, + "loss": 62.5944, + "step": 9900 + }, + { + "epoch": 15.959338942913286, + "grad_norm": 0.6016744375228882, + "learning_rate": 0.0001, + "loss": 62.5889, + "step": 9910 + }, + { + "epoch": 15.975462286491661, + "grad_norm": 0.5464991331100464, + "learning_rate": 0.0001, + "loss": 61.3138, + "step": 9920 + }, + { + "epoch": 15.991585630070036, + "grad_norm": 0.8242359757423401, + "learning_rate": 0.0001, + "loss": 61.8091, + "step": 9930 + }, + { + "epoch": 16.00644933743135, + "grad_norm": 0.7031534314155579, + "learning_rate": 0.0001, + "loss": 57.4847, + "step": 9940 + }, + { + "epoch": 16.022572681009724, + "grad_norm": 0.8429487943649292, + "learning_rate": 0.0001, + "loss": 61.7286, + "step": 9950 + }, + { + "epoch": 16.0386960245881, + "grad_norm": 0.7200850248336792, + "learning_rate": 0.0001, + "loss": 62.6968, + "step": 9960 + }, + { + "epoch": 16.054819368166473, + "grad_norm": 0.7412559390068054, + "learning_rate": 0.0001, + "loss": 62.413, + "step": 9970 + }, + { + "epoch": 16.07094271174485, + "grad_norm": 0.8179170489311218, + "learning_rate": 0.0001, + "loss": 62.7044, + "step": 9980 + }, + { + "epoch": 16.087066055323223, + "grad_norm": 0.6939452886581421, + "learning_rate": 0.0001, + "loss": 61.9062, + "step": 9990 + }, + { + "epoch": 16.1031893989016, + "grad_norm": 0.7145045399665833, + "learning_rate": 0.0001, + "loss": 61.6201, + "step": 10000 + }, + { + "epoch": 16.119312742479973, + "grad_norm": 0.7874398231506348, + "learning_rate": 0.0001, + "loss": 60.4299, + "step": 10010 + }, + { + "epoch": 16.135436086058345, + "grad_norm": 0.6640581488609314, + "learning_rate": 0.0001, + "loss": 62.0195, + "step": 10020 + }, + { + "epoch": 16.15155942963672, + "grad_norm": 0.7903703451156616, + "learning_rate": 0.0001, + "loss": 62.501, + "step": 10030 + }, + { + "epoch": 16.167682773215095, + "grad_norm": 0.6669818162918091, + "learning_rate": 0.0001, + "loss": 59.1489, + "step": 10040 + }, + { + "epoch": 16.18380611679347, + "grad_norm": 0.8380771279335022, + "learning_rate": 0.0001, + "loss": 61.0985, + "step": 10050 + }, + { + "epoch": 16.199929460371845, + "grad_norm": 0.7925304174423218, + "learning_rate": 0.0001, + "loss": 62.9251, + "step": 10060 + }, + { + "epoch": 16.21605280395022, + "grad_norm": 1.2176182270050049, + "learning_rate": 0.0001, + "loss": 61.6772, + "step": 10070 + }, + { + "epoch": 16.232176147528595, + "grad_norm": 0.828171968460083, + "learning_rate": 0.0001, + "loss": 61.3296, + "step": 10080 + }, + { + "epoch": 16.24829949110697, + "grad_norm": 0.6884972453117371, + "learning_rate": 0.0001, + "loss": 61.5371, + "step": 10090 + }, + { + "epoch": 16.264422834685345, + "grad_norm": 0.6455051302909851, + "learning_rate": 0.0001, + "loss": 60.6548, + "step": 10100 + }, + { + "epoch": 16.280546178263716, + "grad_norm": 0.687982976436615, + "learning_rate": 0.0001, + "loss": 62.0357, + "step": 10110 + }, + { + "epoch": 16.29666952184209, + "grad_norm": 0.811744213104248, + "learning_rate": 0.0001, + "loss": 60.5664, + "step": 10120 + }, + { + "epoch": 16.312792865420466, + "grad_norm": 0.7140675187110901, + "learning_rate": 0.0001, + "loss": 62.9491, + "step": 10130 + }, + { + "epoch": 16.32891620899884, + "grad_norm": 0.9068045020103455, + "learning_rate": 0.0001, + "loss": 61.118, + "step": 10140 + }, + { + "epoch": 16.345039552577216, + "grad_norm": 0.9028809666633606, + "learning_rate": 0.0001, + "loss": 61.4308, + "step": 10150 + }, + { + "epoch": 16.36116289615559, + "grad_norm": 0.7502166628837585, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10160 + }, + { + "epoch": 16.377286239733966, + "grad_norm": 0.807391345500946, + "learning_rate": 0.0001, + "loss": 61.6825, + "step": 10170 + }, + { + "epoch": 16.39340958331234, + "grad_norm": 0.762725830078125, + "learning_rate": 0.0001, + "loss": 62.7879, + "step": 10180 + }, + { + "epoch": 16.409532926890712, + "grad_norm": 0.7973660230636597, + "learning_rate": 0.0001, + "loss": 61.0727, + "step": 10190 + }, + { + "epoch": 16.425656270469087, + "grad_norm": 1.004622459411621, + "learning_rate": 0.0001, + "loss": 59.8048, + "step": 10200 + }, + { + "epoch": 16.441779614047462, + "grad_norm": 0.8243395686149597, + "learning_rate": 0.0001, + "loss": 63.9312, + "step": 10210 + }, + { + "epoch": 16.457902957625837, + "grad_norm": 0.7006281614303589, + "learning_rate": 0.0001, + "loss": 61.3766, + "step": 10220 + }, + { + "epoch": 16.474026301204212, + "grad_norm": 0.8672197461128235, + "learning_rate": 0.0001, + "loss": 61.6659, + "step": 10230 + }, + { + "epoch": 16.490149644782587, + "grad_norm": 0.7836477160453796, + "learning_rate": 0.0001, + "loss": 61.3686, + "step": 10240 + }, + { + "epoch": 16.506272988360962, + "grad_norm": 0.5842607617378235, + "learning_rate": 0.0001, + "loss": 62.616, + "step": 10250 + }, + { + "epoch": 16.522396331939337, + "grad_norm": 0.6777102947235107, + "learning_rate": 0.0001, + "loss": 62.1135, + "step": 10260 + }, + { + "epoch": 16.538519675517712, + "grad_norm": 0.856697678565979, + "learning_rate": 0.0001, + "loss": 61.849, + "step": 10270 + }, + { + "epoch": 16.554643019096083, + "grad_norm": 0.7300705313682556, + "learning_rate": 0.0001, + "loss": 60.7678, + "step": 10280 + }, + { + "epoch": 16.57076636267446, + "grad_norm": 0.799802839756012, + "learning_rate": 0.0001, + "loss": 62.4888, + "step": 10290 + }, + { + "epoch": 16.586889706252833, + "grad_norm": 0.791251540184021, + "learning_rate": 0.0001, + "loss": 61.7589, + "step": 10300 + }, + { + "epoch": 16.60301304983121, + "grad_norm": 0.6498433947563171, + "learning_rate": 0.0001, + "loss": 61.087, + "step": 10310 + }, + { + "epoch": 16.619136393409583, + "grad_norm": 0.702925443649292, + "learning_rate": 0.0001, + "loss": 59.3493, + "step": 10320 + }, + { + "epoch": 16.63525973698796, + "grad_norm": 0.8508023619651794, + "learning_rate": 0.0001, + "loss": 61.2188, + "step": 10330 + }, + { + "epoch": 16.651383080566333, + "grad_norm": 0.8325870633125305, + "learning_rate": 0.0001, + "loss": 62.9164, + "step": 10340 + }, + { + "epoch": 16.667506424144708, + "grad_norm": 0.7140557765960693, + "learning_rate": 0.0001, + "loss": 62.5838, + "step": 10350 + }, + { + "epoch": 16.683629767723083, + "grad_norm": 0.5574952363967896, + "learning_rate": 0.0001, + "loss": 59.767, + "step": 10360 + }, + { + "epoch": 16.699753111301455, + "grad_norm": 0.8022965788841248, + "learning_rate": 0.0001, + "loss": 60.2198, + "step": 10370 + }, + { + "epoch": 16.71587645487983, + "grad_norm": 1.1291698217391968, + "learning_rate": 0.0001, + "loss": 61.4908, + "step": 10380 + }, + { + "epoch": 16.731999798458205, + "grad_norm": 0.8651410937309265, + "learning_rate": 0.0001, + "loss": 60.5228, + "step": 10390 + }, + { + "epoch": 16.74812314203658, + "grad_norm": 0.8426535129547119, + "learning_rate": 0.0001, + "loss": 62.3786, + "step": 10400 + }, + { + "epoch": 16.764246485614954, + "grad_norm": 0.6329453587532043, + "learning_rate": 0.0001, + "loss": 60.9621, + "step": 10410 + }, + { + "epoch": 16.78036982919333, + "grad_norm": 0.6745129227638245, + "learning_rate": 0.0001, + "loss": 59.3405, + "step": 10420 + }, + { + "epoch": 16.796493172771704, + "grad_norm": 0.8483593463897705, + "learning_rate": 0.0001, + "loss": 61.1527, + "step": 10430 + }, + { + "epoch": 16.81261651635008, + "grad_norm": 0.8649617433547974, + "learning_rate": 0.0001, + "loss": 61.051, + "step": 10440 + }, + { + "epoch": 16.828739859928454, + "grad_norm": 0.7309728264808655, + "learning_rate": 0.0001, + "loss": 60.3626, + "step": 10450 + }, + { + "epoch": 16.844863203506826, + "grad_norm": 0.6467884182929993, + "learning_rate": 0.0001, + "loss": 63.8254, + "step": 10460 + }, + { + "epoch": 16.8609865470852, + "grad_norm": 0.7628102898597717, + "learning_rate": 0.0001, + "loss": 59.5604, + "step": 10470 + }, + { + "epoch": 16.877109890663576, + "grad_norm": 0.78266841173172, + "learning_rate": 0.0001, + "loss": 60.9387, + "step": 10480 + }, + { + "epoch": 16.89323323424195, + "grad_norm": 0.6936920285224915, + "learning_rate": 0.0001, + "loss": 62.5492, + "step": 10490 + }, + { + "epoch": 16.909356577820326, + "grad_norm": 0.8343939185142517, + "learning_rate": 0.0001, + "loss": 60.9975, + "step": 10500 + }, + { + "epoch": 16.9254799213987, + "grad_norm": 0.798984169960022, + "learning_rate": 0.0001, + "loss": 59.5835, + "step": 10510 + }, + { + "epoch": 16.941603264977076, + "grad_norm": 1.369275450706482, + "learning_rate": 0.0001, + "loss": 61.749, + "step": 10520 + }, + { + "epoch": 16.95772660855545, + "grad_norm": 0.5993408560752869, + "learning_rate": 0.0001, + "loss": 62.1093, + "step": 10530 + }, + { + "epoch": 16.973849952133826, + "grad_norm": 0.8153544664382935, + "learning_rate": 0.0001, + "loss": 61.857, + "step": 10540 + }, + { + "epoch": 16.989973295712197, + "grad_norm": 0.7042689919471741, + "learning_rate": 0.0001, + "loss": 60.5178, + "step": 10550 + }, + { + "epoch": 17.00483700307351, + "grad_norm": 1.1931812763214111, + "learning_rate": 0.0001, + "loss": 56.4815, + "step": 10560 + }, + { + "epoch": 17.020960346651886, + "grad_norm": 0.9051854610443115, + "learning_rate": 0.0001, + "loss": 60.7729, + "step": 10570 + }, + { + "epoch": 17.03708369023026, + "grad_norm": 1.093518614768982, + "learning_rate": 0.0001, + "loss": 60.904, + "step": 10580 + }, + { + "epoch": 17.053207033808636, + "grad_norm": 0.7545284032821655, + "learning_rate": 0.0001, + "loss": 59.7165, + "step": 10590 + }, + { + "epoch": 17.06933037738701, + "grad_norm": 0.8542641401290894, + "learning_rate": 0.0001, + "loss": 60.8947, + "step": 10600 + }, + { + "epoch": 17.085453720965386, + "grad_norm": 0.7393799424171448, + "learning_rate": 0.0001, + "loss": 62.8384, + "step": 10610 + }, + { + "epoch": 17.10157706454376, + "grad_norm": 0.6756435632705688, + "learning_rate": 0.0001, + "loss": 61.164, + "step": 10620 + }, + { + "epoch": 17.117700408122133, + "grad_norm": 0.725150465965271, + "learning_rate": 0.0001, + "loss": 60.3789, + "step": 10630 + }, + { + "epoch": 17.133823751700508, + "grad_norm": 1.1529417037963867, + "learning_rate": 0.0001, + "loss": 62.2465, + "step": 10640 + }, + { + "epoch": 17.149947095278883, + "grad_norm": 0.8266693353652954, + "learning_rate": 0.0001, + "loss": 61.6755, + "step": 10650 + }, + { + "epoch": 17.166070438857258, + "grad_norm": 0.7444040775299072, + "learning_rate": 0.0001, + "loss": 62.7084, + "step": 10660 + }, + { + "epoch": 17.182193782435633, + "grad_norm": 0.8164665102958679, + "learning_rate": 0.0001, + "loss": 61.6032, + "step": 10670 + }, + { + "epoch": 17.198317126014008, + "grad_norm": 0.8215296268463135, + "learning_rate": 0.0001, + "loss": 61.033, + "step": 10680 + }, + { + "epoch": 17.214440469592382, + "grad_norm": 0.7939956188201904, + "learning_rate": 0.0001, + "loss": 60.5016, + "step": 10690 + }, + { + "epoch": 17.230563813170757, + "grad_norm": 0.9533108472824097, + "learning_rate": 0.0001, + "loss": 62.6995, + "step": 10700 + }, + { + "epoch": 17.246687156749132, + "grad_norm": 0.6602820754051208, + "learning_rate": 0.0001, + "loss": 61.2709, + "step": 10710 + }, + { + "epoch": 17.262810500327504, + "grad_norm": 0.8369201421737671, + "learning_rate": 0.0001, + "loss": 63.1138, + "step": 10720 + }, + { + "epoch": 17.27893384390588, + "grad_norm": 0.7431164979934692, + "learning_rate": 0.0001, + "loss": 62.8588, + "step": 10730 + }, + { + "epoch": 17.295057187484254, + "grad_norm": 0.7957271337509155, + "learning_rate": 0.0001, + "loss": 60.5119, + "step": 10740 + }, + { + "epoch": 17.31118053106263, + "grad_norm": 0.9407948851585388, + "learning_rate": 0.0001, + "loss": 61.1877, + "step": 10750 + }, + { + "epoch": 17.327303874641004, + "grad_norm": 0.6641581654548645, + "learning_rate": 0.0001, + "loss": 60.542, + "step": 10760 + }, + { + "epoch": 17.34342721821938, + "grad_norm": 0.7786923050880432, + "learning_rate": 0.0001, + "loss": 62.1582, + "step": 10770 + }, + { + "epoch": 17.359550561797754, + "grad_norm": 0.8150411248207092, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10780 + }, + { + "epoch": 17.37567390537613, + "grad_norm": 0.8315487504005432, + "learning_rate": 0.0001, + "loss": 60.4382, + "step": 10790 + }, + { + "epoch": 17.391797248954504, + "grad_norm": 0.8305006623268127, + "learning_rate": 0.0001, + "loss": 61.3262, + "step": 10800 + }, + { + "epoch": 17.407920592532875, + "grad_norm": 0.897731363773346, + "learning_rate": 0.0001, + "loss": 60.7809, + "step": 10810 + }, + { + "epoch": 17.42404393611125, + "grad_norm": 0.7461619973182678, + "learning_rate": 0.0001, + "loss": 61.3858, + "step": 10820 + }, + { + "epoch": 17.440167279689625, + "grad_norm": 0.6750472187995911, + "learning_rate": 0.0001, + "loss": 60.2692, + "step": 10830 + }, + { + "epoch": 17.456290623268, + "grad_norm": 0.5412933230400085, + "learning_rate": 0.0001, + "loss": 61.7872, + "step": 10840 + }, + { + "epoch": 17.472413966846375, + "grad_norm": 0.6485120058059692, + "learning_rate": 0.0001, + "loss": 59.5407, + "step": 10850 + }, + { + "epoch": 17.48853731042475, + "grad_norm": 0.818708598613739, + "learning_rate": 0.0001, + "loss": 61.8066, + "step": 10860 + }, + { + "epoch": 17.504660654003125, + "grad_norm": 0.7121092081069946, + "learning_rate": 0.0001, + "loss": 61.8086, + "step": 10870 + }, + { + "epoch": 17.5207839975815, + "grad_norm": 0.7031684517860413, + "learning_rate": 0.0001, + "loss": 60.0706, + "step": 10880 + }, + { + "epoch": 17.536907341159875, + "grad_norm": 0.6259323358535767, + "learning_rate": 0.0001, + "loss": 61.9625, + "step": 10890 + }, + { + "epoch": 17.553030684738246, + "grad_norm": 0.8526529669761658, + "learning_rate": 0.0001, + "loss": 61.3299, + "step": 10900 + }, + { + "epoch": 17.56915402831662, + "grad_norm": 0.6860859990119934, + "learning_rate": 0.0001, + "loss": 61.3193, + "step": 10910 + }, + { + "epoch": 17.585277371894996, + "grad_norm": 0.7715994119644165, + "learning_rate": 0.0001, + "loss": 61.9111, + "step": 10920 + }, + { + "epoch": 17.60140071547337, + "grad_norm": 0.7171186804771423, + "learning_rate": 0.0001, + "loss": 60.3638, + "step": 10930 + }, + { + "epoch": 17.617524059051746, + "grad_norm": 0.7796649932861328, + "learning_rate": 0.0001, + "loss": 61.9437, + "step": 10940 + }, + { + "epoch": 17.63364740263012, + "grad_norm": 0.7930101752281189, + "learning_rate": 0.0001, + "loss": 60.1085, + "step": 10950 + }, + { + "epoch": 17.649770746208496, + "grad_norm": 0.6370466351509094, + "learning_rate": 0.0001, + "loss": 61.8433, + "step": 10960 + }, + { + "epoch": 17.66589408978687, + "grad_norm": 0.7830466628074646, + "learning_rate": 0.0001, + "loss": 61.3403, + "step": 10970 + }, + { + "epoch": 17.682017433365242, + "grad_norm": 0.873456597328186, + "learning_rate": 0.0001, + "loss": 62.4567, + "step": 10980 + }, + { + "epoch": 17.698140776943617, + "grad_norm": 0.7560754418373108, + "learning_rate": 0.0001, + "loss": 60.9808, + "step": 10990 + }, + { + "epoch": 17.714264120521992, + "grad_norm": 0.6832230091094971, + "learning_rate": 0.0001, + "loss": 61.5059, + "step": 11000 + }, + { + "epoch": 17.730387464100367, + "grad_norm": 0.7701742649078369, + "learning_rate": 0.0001, + "loss": 62.4963, + "step": 11010 + }, + { + "epoch": 17.746510807678742, + "grad_norm": 0.7088269591331482, + "learning_rate": 0.0001, + "loss": 62.6467, + "step": 11020 + }, + { + "epoch": 17.762634151257117, + "grad_norm": 0.8424612879753113, + "learning_rate": 0.0001, + "loss": 60.1213, + "step": 11030 + }, + { + "epoch": 17.778757494835492, + "grad_norm": 0.9974421858787537, + "learning_rate": 0.0001, + "loss": 61.6051, + "step": 11040 + }, + { + "epoch": 17.794880838413867, + "grad_norm": 0.701970636844635, + "learning_rate": 0.0001, + "loss": 62.4155, + "step": 11050 + }, + { + "epoch": 17.811004181992242, + "grad_norm": 0.6826842427253723, + "learning_rate": 0.0001, + "loss": 60.2127, + "step": 11060 + }, + { + "epoch": 17.827127525570614, + "grad_norm": 0.7019745707511902, + "learning_rate": 0.0001, + "loss": 60.9281, + "step": 11070 + }, + { + "epoch": 17.84325086914899, + "grad_norm": 0.836280345916748, + "learning_rate": 0.0001, + "loss": 61.1217, + "step": 11080 + }, + { + "epoch": 17.859374212727364, + "grad_norm": 0.6405923366546631, + "learning_rate": 0.0001, + "loss": 61.6393, + "step": 11090 + }, + { + "epoch": 17.87549755630574, + "grad_norm": 0.6809528470039368, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 11100 + }, + { + "epoch": 17.891620899884114, + "grad_norm": 0.6983410120010376, + "learning_rate": 0.0001, + "loss": 59.933, + "step": 11110 + }, + { + "epoch": 17.90774424346249, + "grad_norm": 1.0240110158920288, + "learning_rate": 0.0001, + "loss": 61.1964, + "step": 11120 + }, + { + "epoch": 17.923867587040863, + "grad_norm": 0.860657811164856, + "learning_rate": 0.0001, + "loss": 61.1613, + "step": 11130 + }, + { + "epoch": 17.93999093061924, + "grad_norm": 0.6434107422828674, + "learning_rate": 0.0001, + "loss": 60.2435, + "step": 11140 + }, + { + "epoch": 17.956114274197613, + "grad_norm": 0.7400848269462585, + "learning_rate": 0.0001, + "loss": 62.452, + "step": 11150 + }, + { + "epoch": 17.972237617775985, + "grad_norm": 0.6526095867156982, + "learning_rate": 0.0001, + "loss": 62.3344, + "step": 11160 + }, + { + "epoch": 17.98836096135436, + "grad_norm": 0.6971097588539124, + "learning_rate": 0.0001, + "loss": 60.2415, + "step": 11170 + }, + { + "epoch": 18.003224668715674, + "grad_norm": 0.724881649017334, + "learning_rate": 0.0001, + "loss": 56.0513, + "step": 11180 + }, + { + "epoch": 18.01934801229405, + "grad_norm": 0.653710663318634, + "learning_rate": 0.0001, + "loss": 60.8364, + "step": 11190 + }, + { + "epoch": 18.035471355872424, + "grad_norm": 0.7356674075126648, + "learning_rate": 0.0001, + "loss": 61.6681, + "step": 11200 + }, + { + "epoch": 18.0515946994508, + "grad_norm": 0.6726879477500916, + "learning_rate": 0.0001, + "loss": 60.6682, + "step": 11210 + }, + { + "epoch": 18.067718043029174, + "grad_norm": 0.7367722392082214, + "learning_rate": 0.0001, + "loss": 63.0462, + "step": 11220 + }, + { + "epoch": 18.08384138660755, + "grad_norm": 0.6687121987342834, + "learning_rate": 0.0001, + "loss": 60.8707, + "step": 11230 + }, + { + "epoch": 18.09996473018592, + "grad_norm": 1.0027060508728027, + "learning_rate": 0.0001, + "loss": 62.4703, + "step": 11240 + }, + { + "epoch": 18.116088073764296, + "grad_norm": 1.0447720289230347, + "learning_rate": 0.0001, + "loss": 62.1746, + "step": 11250 + }, + { + "epoch": 18.13221141734267, + "grad_norm": 0.7687053084373474, + "learning_rate": 0.0001, + "loss": 60.1413, + "step": 11260 + }, + { + "epoch": 18.148334760921045, + "grad_norm": 0.7250781059265137, + "learning_rate": 0.0001, + "loss": 60.1143, + "step": 11270 + }, + { + "epoch": 18.16445810449942, + "grad_norm": 0.7356208562850952, + "learning_rate": 0.0001, + "loss": 62.0788, + "step": 11280 + }, + { + "epoch": 18.180581448077795, + "grad_norm": 0.7838225364685059, + "learning_rate": 0.0001, + "loss": 61.7046, + "step": 11290 + }, + { + "epoch": 18.19670479165617, + "grad_norm": 0.7600042819976807, + "learning_rate": 0.0001, + "loss": 61.4646, + "step": 11300 + }, + { + "epoch": 18.212828135234545, + "grad_norm": 0.7339571714401245, + "learning_rate": 0.0001, + "loss": 61.0983, + "step": 11310 + }, + { + "epoch": 18.22895147881292, + "grad_norm": 0.6982674598693848, + "learning_rate": 0.0001, + "loss": 60.2853, + "step": 11320 + }, + { + "epoch": 18.24507482239129, + "grad_norm": 0.7572301030158997, + "learning_rate": 0.0001, + "loss": 61.8967, + "step": 11330 + }, + { + "epoch": 18.261198165969667, + "grad_norm": 0.6206921935081482, + "learning_rate": 0.0001, + "loss": 62.1467, + "step": 11340 + }, + { + "epoch": 18.27732150954804, + "grad_norm": 0.658437967300415, + "learning_rate": 0.0001, + "loss": 61.2117, + "step": 11350 + }, + { + "epoch": 18.293444853126417, + "grad_norm": 0.7880764603614807, + "learning_rate": 0.0001, + "loss": 61.6246, + "step": 11360 + }, + { + "epoch": 18.30956819670479, + "grad_norm": 0.6601800918579102, + "learning_rate": 0.0001, + "loss": 61.4818, + "step": 11370 + }, + { + "epoch": 18.325691540283167, + "grad_norm": 0.6816290616989136, + "learning_rate": 0.0001, + "loss": 60.4834, + "step": 11380 + }, + { + "epoch": 18.34181488386154, + "grad_norm": 0.7667511105537415, + "learning_rate": 0.0001, + "loss": 60.9106, + "step": 11390 + }, + { + "epoch": 18.357938227439917, + "grad_norm": 0.6490039825439453, + "learning_rate": 0.0001, + "loss": 59.6968, + "step": 11400 + }, + { + "epoch": 18.37406157101829, + "grad_norm": 0.6865658760070801, + "learning_rate": 0.0001, + "loss": 61.7898, + "step": 11410 + }, + { + "epoch": 18.390184914596663, + "grad_norm": 0.8605737090110779, + "learning_rate": 0.0001, + "loss": 60.924, + "step": 11420 + }, + { + "epoch": 18.406308258175038, + "grad_norm": 1.0474110841751099, + "learning_rate": 0.0001, + "loss": 61.0381, + "step": 11430 + }, + { + "epoch": 18.422431601753413, + "grad_norm": 0.9241340160369873, + "learning_rate": 0.0001, + "loss": 61.2609, + "step": 11440 + }, + { + "epoch": 18.438554945331788, + "grad_norm": 1.1441621780395508, + "learning_rate": 0.0001, + "loss": 60.5495, + "step": 11450 + }, + { + "epoch": 18.454678288910163, + "grad_norm": 0.7277412414550781, + "learning_rate": 0.0001, + "loss": 60.4654, + "step": 11460 + }, + { + "epoch": 18.470801632488538, + "grad_norm": 0.6668717861175537, + "learning_rate": 0.0001, + "loss": 60.057, + "step": 11470 + }, + { + "epoch": 18.486924976066913, + "grad_norm": 0.7335819005966187, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 11480 + }, + { + "epoch": 18.503048319645288, + "grad_norm": 0.7261807918548584, + "learning_rate": 0.0001, + "loss": 61.1508, + "step": 11490 + }, + { + "epoch": 18.519171663223663, + "grad_norm": 0.7715283036231995, + "learning_rate": 0.0001, + "loss": 61.5492, + "step": 11500 + }, + { + "epoch": 18.535295006802034, + "grad_norm": 0.9603212475776672, + "learning_rate": 0.0001, + "loss": 60.4321, + "step": 11510 + }, + { + "epoch": 18.55141835038041, + "grad_norm": 0.6130046844482422, + "learning_rate": 0.0001, + "loss": 61.2942, + "step": 11520 + }, + { + "epoch": 18.567541693958784, + "grad_norm": 0.8147348165512085, + "learning_rate": 0.0001, + "loss": 61.4313, + "step": 11530 + }, + { + "epoch": 18.58366503753716, + "grad_norm": 0.7909948825836182, + "learning_rate": 0.0001, + "loss": 60.3547, + "step": 11540 + }, + { + "epoch": 18.599788381115534, + "grad_norm": 0.6470443606376648, + "learning_rate": 0.0001, + "loss": 61.4144, + "step": 11550 + }, + { + "epoch": 18.61591172469391, + "grad_norm": 0.7132186889648438, + "learning_rate": 0.0001, + "loss": 61.8102, + "step": 11560 + }, + { + "epoch": 18.632035068272284, + "grad_norm": 0.7054323554039001, + "learning_rate": 0.0001, + "loss": 62.1431, + "step": 11570 + }, + { + "epoch": 18.64815841185066, + "grad_norm": 0.8784761428833008, + "learning_rate": 0.0001, + "loss": 61.6653, + "step": 11580 + }, + { + "epoch": 18.66428175542903, + "grad_norm": 0.7143248319625854, + "learning_rate": 0.0001, + "loss": 60.1275, + "step": 11590 + }, + { + "epoch": 18.680405099007405, + "grad_norm": 7.87012243270874, + "learning_rate": 0.0001, + "loss": 61.3484, + "step": 11600 + }, + { + "epoch": 18.69652844258578, + "grad_norm": 0.7263020873069763, + "learning_rate": 0.0001, + "loss": 59.906, + "step": 11610 + }, + { + "epoch": 18.712651786164155, + "grad_norm": 0.6468373537063599, + "learning_rate": 0.0001, + "loss": 61.8781, + "step": 11620 + }, + { + "epoch": 18.72877512974253, + "grad_norm": 3.34307861328125, + "learning_rate": 0.0001, + "loss": 61.0841, + "step": 11630 + }, + { + "epoch": 18.744898473320905, + "grad_norm": 0.7348340153694153, + "learning_rate": 0.0001, + "loss": 61.3201, + "step": 11640 + }, + { + "epoch": 18.76102181689928, + "grad_norm": 0.7646427154541016, + "learning_rate": 0.0001, + "loss": 60.1858, + "step": 11650 + }, + { + "epoch": 18.777145160477655, + "grad_norm": 0.6659495234489441, + "learning_rate": 0.0001, + "loss": 60.8999, + "step": 11660 + }, + { + "epoch": 18.79326850405603, + "grad_norm": 0.9913583993911743, + "learning_rate": 0.0001, + "loss": 59.0732, + "step": 11670 + }, + { + "epoch": 18.8093918476344, + "grad_norm": 0.7480027079582214, + "learning_rate": 0.0001, + "loss": 58.6846, + "step": 11680 + }, + { + "epoch": 18.825515191212777, + "grad_norm": 0.8000630736351013, + "learning_rate": 0.0001, + "loss": 62.8883, + "step": 11690 + }, + { + "epoch": 18.84163853479115, + "grad_norm": 0.8450509309768677, + "learning_rate": 0.0001, + "loss": 63.4962, + "step": 11700 + }, + { + "epoch": 18.857761878369526, + "grad_norm": 0.8365609645843506, + "learning_rate": 0.0001, + "loss": 62.2701, + "step": 11710 + }, + { + "epoch": 18.8738852219479, + "grad_norm": 0.7197144031524658, + "learning_rate": 0.0001, + "loss": 61.3158, + "step": 11720 + }, + { + "epoch": 18.890008565526276, + "grad_norm": 0.6460379958152771, + "learning_rate": 0.0001, + "loss": 62.6645, + "step": 11730 + }, + { + "epoch": 18.90613190910465, + "grad_norm": 0.8203967213630676, + "learning_rate": 0.0001, + "loss": 62.3257, + "step": 11740 + }, + { + "epoch": 18.922255252683026, + "grad_norm": 0.682134211063385, + "learning_rate": 0.0001, + "loss": 59.9144, + "step": 11750 + }, + { + "epoch": 18.9383785962614, + "grad_norm": 0.7080821394920349, + "learning_rate": 0.0001, + "loss": 61.4325, + "step": 11760 + }, + { + "epoch": 18.954501939839773, + "grad_norm": 0.7963712811470032, + "learning_rate": 0.0001, + "loss": 58.6136, + "step": 11770 + }, + { + "epoch": 18.970625283418148, + "grad_norm": 0.7317046523094177, + "learning_rate": 0.0001, + "loss": 61.3507, + "step": 11780 + }, + { + "epoch": 18.986748626996523, + "grad_norm": 0.6586548686027527, + "learning_rate": 0.0001, + "loss": 61.4898, + "step": 11790 + }, + { + "epoch": 19.001612334357837, + "grad_norm": 0.9010424613952637, + "learning_rate": 0.0001, + "loss": 56.8915, + "step": 11800 + }, + { + "epoch": 19.017735677936212, + "grad_norm": 0.7102377414703369, + "learning_rate": 0.0001, + "loss": 62.2097, + "step": 11810 + }, + { + "epoch": 19.033859021514587, + "grad_norm": 0.7959222793579102, + "learning_rate": 0.0001, + "loss": 61.4484, + "step": 11820 + }, + { + "epoch": 19.049982365092962, + "grad_norm": 0.6483829617500305, + "learning_rate": 0.0001, + "loss": 60.3563, + "step": 11830 + }, + { + "epoch": 19.066105708671337, + "grad_norm": 1.1301988363265991, + "learning_rate": 0.0001, + "loss": 61.3698, + "step": 11840 + }, + { + "epoch": 19.082229052249712, + "grad_norm": 0.7304167151451111, + "learning_rate": 0.0001, + "loss": 59.2096, + "step": 11850 + }, + { + "epoch": 19.098352395828083, + "grad_norm": 0.7757825255393982, + "learning_rate": 0.0001, + "loss": 59.914, + "step": 11860 + }, + { + "epoch": 19.11447573940646, + "grad_norm": 0.7009251713752747, + "learning_rate": 0.0001, + "loss": 61.0768, + "step": 11870 + }, + { + "epoch": 19.130599082984833, + "grad_norm": 0.6015574336051941, + "learning_rate": 0.0001, + "loss": 61.2698, + "step": 11880 + }, + { + "epoch": 19.14672242656321, + "grad_norm": 0.8028033971786499, + "learning_rate": 0.0001, + "loss": 61.422, + "step": 11890 + }, + { + "epoch": 19.162845770141583, + "grad_norm": 0.5753505825996399, + "learning_rate": 0.0001, + "loss": 61.5066, + "step": 11900 + }, + { + "epoch": 19.17896911371996, + "grad_norm": 0.7702978849411011, + "learning_rate": 0.0001, + "loss": 60.0737, + "step": 11910 + }, + { + "epoch": 19.195092457298333, + "grad_norm": 0.6469365358352661, + "learning_rate": 0.0001, + "loss": 59.5091, + "step": 11920 + }, + { + "epoch": 19.211215800876708, + "grad_norm": 0.8400216698646545, + "learning_rate": 0.0001, + "loss": 62.3739, + "step": 11930 + }, + { + "epoch": 19.22733914445508, + "grad_norm": 1.9919443130493164, + "learning_rate": 0.0001, + "loss": 59.7271, + "step": 11940 + }, + { + "epoch": 19.243462488033455, + "grad_norm": 1.0247725248336792, + "learning_rate": 0.0001, + "loss": 59.3804, + "step": 11950 + }, + { + "epoch": 19.25958583161183, + "grad_norm": 1.0717105865478516, + "learning_rate": 0.0001, + "loss": 61.3617, + "step": 11960 + }, + { + "epoch": 19.275709175190205, + "grad_norm": 0.6324111223220825, + "learning_rate": 0.0001, + "loss": 59.6094, + "step": 11970 + }, + { + "epoch": 19.29183251876858, + "grad_norm": 0.6665195226669312, + "learning_rate": 0.0001, + "loss": 61.5879, + "step": 11980 + }, + { + "epoch": 19.307955862346954, + "grad_norm": 0.7933892011642456, + "learning_rate": 0.0001, + "loss": 60.4011, + "step": 11990 + }, + { + "epoch": 19.32407920592533, + "grad_norm": 0.8135427236557007, + "learning_rate": 0.0001, + "loss": 61.2936, + "step": 12000 + }, + { + "epoch": 19.340202549503704, + "grad_norm": 0.7154449820518494, + "learning_rate": 0.0001, + "loss": 62.3996, + "step": 12010 + }, + { + "epoch": 19.35632589308208, + "grad_norm": 0.9926077127456665, + "learning_rate": 0.0001, + "loss": 60.5219, + "step": 12020 + }, + { + "epoch": 19.37244923666045, + "grad_norm": 1.013094425201416, + "learning_rate": 0.0001, + "loss": 59.9494, + "step": 12030 + }, + { + "epoch": 19.388572580238826, + "grad_norm": 0.984582245349884, + "learning_rate": 0.0001, + "loss": 59.7877, + "step": 12040 + }, + { + "epoch": 19.4046959238172, + "grad_norm": 1.256519079208374, + "learning_rate": 0.0001, + "loss": 60.2608, + "step": 12050 + }, + { + "epoch": 19.420819267395576, + "grad_norm": 0.7013972997665405, + "learning_rate": 0.0001, + "loss": 63.4573, + "step": 12060 + }, + { + "epoch": 19.43694261097395, + "grad_norm": 0.9027557373046875, + "learning_rate": 0.0001, + "loss": 61.3252, + "step": 12070 + }, + { + "epoch": 19.453065954552326, + "grad_norm": 0.9057058691978455, + "learning_rate": 0.0001, + "loss": 60.4992, + "step": 12080 + }, + { + "epoch": 19.4691892981307, + "grad_norm": 0.6828131079673767, + "learning_rate": 0.0001, + "loss": 61.5369, + "step": 12090 + }, + { + "epoch": 19.485312641709076, + "grad_norm": 0.617858350276947, + "learning_rate": 0.0001, + "loss": 61.4837, + "step": 12100 + }, + { + "epoch": 19.50143598528745, + "grad_norm": 0.7722935676574707, + "learning_rate": 0.0001, + "loss": 61.0886, + "step": 12110 + }, + { + "epoch": 19.517559328865822, + "grad_norm": 0.7039631009101868, + "learning_rate": 0.0001, + "loss": 59.9055, + "step": 12120 + }, + { + "epoch": 19.533682672444197, + "grad_norm": 0.8353239297866821, + "learning_rate": 0.0001, + "loss": 62.0864, + "step": 12130 + }, + { + "epoch": 19.549806016022572, + "grad_norm": 0.764098048210144, + "learning_rate": 0.0001, + "loss": 58.9591, + "step": 12140 + }, + { + "epoch": 19.565929359600947, + "grad_norm": 0.7842984795570374, + "learning_rate": 0.0001, + "loss": 59.0494, + "step": 12150 + }, + { + "epoch": 19.582052703179322, + "grad_norm": 0.7140147686004639, + "learning_rate": 0.0001, + "loss": 60.8067, + "step": 12160 + }, + { + "epoch": 19.598176046757697, + "grad_norm": 0.8182031512260437, + "learning_rate": 0.0001, + "loss": 59.6013, + "step": 12170 + }, + { + "epoch": 19.614299390336072, + "grad_norm": 0.7428567409515381, + "learning_rate": 0.0001, + "loss": 61.8644, + "step": 12180 + }, + { + "epoch": 19.630422733914447, + "grad_norm": 0.6174675822257996, + "learning_rate": 0.0001, + "loss": 60.2086, + "step": 12190 + }, + { + "epoch": 19.646546077492822, + "grad_norm": 0.6416913866996765, + "learning_rate": 0.0001, + "loss": 61.3452, + "step": 12200 + }, + { + "epoch": 19.662669421071193, + "grad_norm": 1.3036755323410034, + "learning_rate": 0.0001, + "loss": 60.5332, + "step": 12210 + }, + { + "epoch": 19.678792764649568, + "grad_norm": 1.230710506439209, + "learning_rate": 0.0001, + "loss": 62.5671, + "step": 12220 + }, + { + "epoch": 19.694916108227943, + "grad_norm": 0.6914290189743042, + "learning_rate": 0.0001, + "loss": 60.7302, + "step": 12230 + }, + { + "epoch": 19.711039451806318, + "grad_norm": 0.6492108106613159, + "learning_rate": 0.0001, + "loss": 60.6465, + "step": 12240 + }, + { + "epoch": 19.727162795384693, + "grad_norm": 0.7104936838150024, + "learning_rate": 0.0001, + "loss": 61.14, + "step": 12250 + }, + { + "epoch": 19.743286138963068, + "grad_norm": 1.0740787982940674, + "learning_rate": 0.0001, + "loss": 60.1977, + "step": 12260 + }, + { + "epoch": 19.759409482541443, + "grad_norm": 0.5764663815498352, + "learning_rate": 0.0001, + "loss": 59.994, + "step": 12270 + }, + { + "epoch": 19.775532826119818, + "grad_norm": 0.6204820871353149, + "learning_rate": 0.0001, + "loss": 60.3091, + "step": 12280 + }, + { + "epoch": 19.791656169698193, + "grad_norm": 0.702944815158844, + "learning_rate": 0.0001, + "loss": 60.8637, + "step": 12290 + }, + { + "epoch": 19.807779513276564, + "grad_norm": 0.7591354250907898, + "learning_rate": 0.0001, + "loss": 59.54, + "step": 12300 + }, + { + "epoch": 19.82390285685494, + "grad_norm": 0.755224883556366, + "learning_rate": 0.0001, + "loss": 60.0285, + "step": 12310 + }, + { + "epoch": 19.840026200433314, + "grad_norm": 0.7563093900680542, + "learning_rate": 0.0001, + "loss": 60.2669, + "step": 12320 + }, + { + "epoch": 19.85614954401169, + "grad_norm": 0.8339024782180786, + "learning_rate": 0.0001, + "loss": 60.606, + "step": 12330 + }, + { + "epoch": 19.872272887590064, + "grad_norm": 0.7017751932144165, + "learning_rate": 0.0001, + "loss": 59.579, + "step": 12340 + }, + { + "epoch": 19.88839623116844, + "grad_norm": 1.3561909198760986, + "learning_rate": 0.0001, + "loss": 59.7727, + "step": 12350 + }, + { + "epoch": 19.904519574746814, + "grad_norm": 0.727020263671875, + "learning_rate": 0.0001, + "loss": 60.6587, + "step": 12360 + }, + { + "epoch": 19.92064291832519, + "grad_norm": 0.6882230043411255, + "learning_rate": 0.0001, + "loss": 60.3916, + "step": 12370 + }, + { + "epoch": 19.93676626190356, + "grad_norm": 0.6410917639732361, + "learning_rate": 0.0001, + "loss": 61.6649, + "step": 12380 + }, + { + "epoch": 19.952889605481936, + "grad_norm": 0.9747995138168335, + "learning_rate": 0.0001, + "loss": 60.4622, + "step": 12390 + }, + { + "epoch": 19.96901294906031, + "grad_norm": 1.260621190071106, + "learning_rate": 0.0001, + "loss": 60.4228, + "step": 12400 + }, + { + "epoch": 19.985136292638686, + "grad_norm": 0.6525571942329407, + "learning_rate": 0.0001, + "loss": 59.0612, + "step": 12410 + }, + { + "epoch": 20.0, + "grad_norm": 0.7864645719528198, + "learning_rate": 0.0001, + "loss": 57.2347, + "step": 12420 + }, + { + "epoch": 20.016123343578375, + "grad_norm": 0.7479318380355835, + "learning_rate": 0.0001, + "loss": 60.9263, + "step": 12430 + }, + { + "epoch": 20.03224668715675, + "grad_norm": 0.6433089375495911, + "learning_rate": 0.0001, + "loss": 59.8846, + "step": 12440 + }, + { + "epoch": 20.048370030735125, + "grad_norm": 0.7594776153564453, + "learning_rate": 0.0001, + "loss": 61.7793, + "step": 12450 + }, + { + "epoch": 20.0644933743135, + "grad_norm": 0.8710535764694214, + "learning_rate": 0.0001, + "loss": 60.5331, + "step": 12460 + }, + { + "epoch": 20.08061671789187, + "grad_norm": 0.7302114963531494, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 12470 + }, + { + "epoch": 20.096740061470246, + "grad_norm": 0.6039112210273743, + "learning_rate": 0.0001, + "loss": 60.8272, + "step": 12480 + }, + { + "epoch": 20.11286340504862, + "grad_norm": 0.7339144945144653, + "learning_rate": 0.0001, + "loss": 61.4149, + "step": 12490 + }, + { + "epoch": 20.128986748626996, + "grad_norm": 0.7278878092765808, + "learning_rate": 0.0001, + "loss": 62.631, + "step": 12500 + }, + { + "epoch": 20.14511009220537, + "grad_norm": 0.7241142988204956, + "learning_rate": 0.0001, + "loss": 59.9328, + "step": 12510 + }, + { + "epoch": 20.161233435783746, + "grad_norm": 0.6967542767524719, + "learning_rate": 0.0001, + "loss": 58.7463, + "step": 12520 + }, + { + "epoch": 20.17735677936212, + "grad_norm": 0.7326555848121643, + "learning_rate": 0.0001, + "loss": 59.7082, + "step": 12530 + }, + { + "epoch": 20.193480122940496, + "grad_norm": 0.6706270575523376, + "learning_rate": 0.0001, + "loss": 60.5978, + "step": 12540 + }, + { + "epoch": 20.20960346651887, + "grad_norm": 0.8541543483734131, + "learning_rate": 0.0001, + "loss": 58.9632, + "step": 12550 + }, + { + "epoch": 20.225726810097242, + "grad_norm": 0.6599560379981995, + "learning_rate": 0.0001, + "loss": 58.4572, + "step": 12560 + }, + { + "epoch": 20.241850153675617, + "grad_norm": 0.6668654680252075, + "learning_rate": 0.0001, + "loss": 61.8741, + "step": 12570 + }, + { + "epoch": 20.257973497253992, + "grad_norm": 0.6453808546066284, + "learning_rate": 0.0001, + "loss": 60.2538, + "step": 12580 + }, + { + "epoch": 20.274096840832367, + "grad_norm": 0.7429870963096619, + "learning_rate": 0.0001, + "loss": 60.5795, + "step": 12590 + }, + { + "epoch": 20.290220184410742, + "grad_norm": 0.7588251829147339, + "learning_rate": 0.0001, + "loss": 60.6261, + "step": 12600 + }, + { + "epoch": 20.306343527989117, + "grad_norm": 0.6430977582931519, + "learning_rate": 0.0001, + "loss": 61.0344, + "step": 12610 + }, + { + "epoch": 20.322466871567492, + "grad_norm": 0.6879712343215942, + "learning_rate": 0.0001, + "loss": 62.2668, + "step": 12620 + }, + { + "epoch": 20.338590215145867, + "grad_norm": 0.7650917172431946, + "learning_rate": 0.0001, + "loss": 60.3852, + "step": 12630 + }, + { + "epoch": 20.35471355872424, + "grad_norm": 0.5326449871063232, + "learning_rate": 0.0001, + "loss": 60.6138, + "step": 12640 + }, + { + "epoch": 20.370836902302614, + "grad_norm": 0.7018648982048035, + "learning_rate": 0.0001, + "loss": 62.1081, + "step": 12650 + }, + { + "epoch": 20.38696024588099, + "grad_norm": 0.7988570332527161, + "learning_rate": 0.0001, + "loss": 61.2661, + "step": 12660 + }, + { + "epoch": 20.403083589459364, + "grad_norm": 0.6800012588500977, + "learning_rate": 0.0001, + "loss": 59.9279, + "step": 12670 + }, + { + "epoch": 20.41920693303774, + "grad_norm": 0.6380598545074463, + "learning_rate": 0.0001, + "loss": 60.4877, + "step": 12680 + }, + { + "epoch": 20.435330276616114, + "grad_norm": 0.6755974888801575, + "learning_rate": 0.0001, + "loss": 60.9729, + "step": 12690 + }, + { + "epoch": 20.45145362019449, + "grad_norm": 0.8057810664176941, + "learning_rate": 0.0001, + "loss": 59.6747, + "step": 12700 + }, + { + "epoch": 20.467576963772864, + "grad_norm": 0.6942827105522156, + "learning_rate": 0.0001, + "loss": 60.7314, + "step": 12710 + }, + { + "epoch": 20.48370030735124, + "grad_norm": 1.107717752456665, + "learning_rate": 0.0001, + "loss": 59.1397, + "step": 12720 + }, + { + "epoch": 20.49982365092961, + "grad_norm": 2.6167991161346436, + "learning_rate": 0.0001, + "loss": 60.5604, + "step": 12730 + }, + { + "epoch": 20.515946994507985, + "grad_norm": 0.6041843891143799, + "learning_rate": 0.0001, + "loss": 60.6188, + "step": 12740 + }, + { + "epoch": 20.53207033808636, + "grad_norm": 0.5643273591995239, + "learning_rate": 0.0001, + "loss": 59.3172, + "step": 12750 + }, + { + "epoch": 20.548193681664735, + "grad_norm": 0.7615086436271667, + "learning_rate": 0.0001, + "loss": 60.9263, + "step": 12760 + }, + { + "epoch": 20.56431702524311, + "grad_norm": 0.692243754863739, + "learning_rate": 0.0001, + "loss": 58.3092, + "step": 12770 + }, + { + "epoch": 20.580440368821485, + "grad_norm": 0.6447681784629822, + "learning_rate": 0.0001, + "loss": 58.0169, + "step": 12780 + }, + { + "epoch": 20.59656371239986, + "grad_norm": 0.7043688297271729, + "learning_rate": 0.0001, + "loss": 58.8346, + "step": 12790 + }, + { + "epoch": 20.612687055978235, + "grad_norm": 0.8017414808273315, + "learning_rate": 0.0001, + "loss": 60.7451, + "step": 12800 + }, + { + "epoch": 20.62881039955661, + "grad_norm": 0.8983294367790222, + "learning_rate": 0.0001, + "loss": 59.6858, + "step": 12810 + }, + { + "epoch": 20.64493374313498, + "grad_norm": 1.0817819833755493, + "learning_rate": 0.0001, + "loss": 60.8232, + "step": 12820 + }, + { + "epoch": 20.661057086713356, + "grad_norm": 0.5739386081695557, + "learning_rate": 0.0001, + "loss": 60.7852, + "step": 12830 + }, + { + "epoch": 20.67718043029173, + "grad_norm": 1.007978916168213, + "learning_rate": 0.0001, + "loss": 61.7002, + "step": 12840 + }, + { + "epoch": 20.693303773870106, + "grad_norm": 0.7019670009613037, + "learning_rate": 0.0001, + "loss": 59.7211, + "step": 12850 + }, + { + "epoch": 20.70942711744848, + "grad_norm": 0.7891802191734314, + "learning_rate": 0.0001, + "loss": 61.4963, + "step": 12860 + }, + { + "epoch": 20.725550461026856, + "grad_norm": 0.6955318450927734, + "learning_rate": 0.0001, + "loss": 60.8528, + "step": 12870 + }, + { + "epoch": 20.74167380460523, + "grad_norm": 1.9988878965377808, + "learning_rate": 0.0001, + "loss": 58.7444, + "step": 12880 + }, + { + "epoch": 20.757797148183606, + "grad_norm": 0.7859933972358704, + "learning_rate": 0.0001, + "loss": 60.0696, + "step": 12890 + }, + { + "epoch": 20.77392049176198, + "grad_norm": 0.8253535628318787, + "learning_rate": 0.0001, + "loss": 61.3408, + "step": 12900 + }, + { + "epoch": 20.790043835340352, + "grad_norm": 0.6639575362205505, + "learning_rate": 0.0001, + "loss": 58.3044, + "step": 12910 + }, + { + "epoch": 20.806167178918727, + "grad_norm": 0.6754361391067505, + "learning_rate": 0.0001, + "loss": 59.8301, + "step": 12920 + }, + { + "epoch": 20.822290522497102, + "grad_norm": 0.7306725382804871, + "learning_rate": 0.0001, + "loss": 60.0021, + "step": 12930 + }, + { + "epoch": 20.838413866075477, + "grad_norm": 6.799670696258545, + "learning_rate": 0.0001, + "loss": 57.7934, + "step": 12940 + }, + { + "epoch": 20.854537209653852, + "grad_norm": 0.6870545744895935, + "learning_rate": 0.0001, + "loss": 59.2313, + "step": 12950 + }, + { + "epoch": 20.870660553232227, + "grad_norm": 0.6980645060539246, + "learning_rate": 0.0001, + "loss": 60.1903, + "step": 12960 + }, + { + "epoch": 20.886783896810602, + "grad_norm": 0.7006897926330566, + "learning_rate": 0.0001, + "loss": 60.9525, + "step": 12970 + }, + { + "epoch": 20.902907240388977, + "grad_norm": 0.7185907363891602, + "learning_rate": 0.0001, + "loss": 61.8833, + "step": 12980 + }, + { + "epoch": 20.91903058396735, + "grad_norm": 0.6029082536697388, + "learning_rate": 0.0001, + "loss": 60.3764, + "step": 12990 + }, + { + "epoch": 20.935153927545723, + "grad_norm": 0.8502999544143677, + "learning_rate": 0.0001, + "loss": 59.0051, + "step": 13000 + }, + { + "epoch": 20.9512772711241, + "grad_norm": 0.908899188041687, + "learning_rate": 0.0001, + "loss": 59.0677, + "step": 13010 + }, + { + "epoch": 20.967400614702473, + "grad_norm": 0.6983920335769653, + "learning_rate": 0.0001, + "loss": 60.8523, + "step": 13020 + }, + { + "epoch": 20.98352395828085, + "grad_norm": 0.9079043865203857, + "learning_rate": 0.0001, + "loss": 58.8014, + "step": 13030 + }, + { + "epoch": 20.999647301859223, + "grad_norm": 0.7139968872070312, + "learning_rate": 0.0001, + "loss": 59.3019, + "step": 13040 + }, + { + "epoch": 21.014511009220538, + "grad_norm": 0.7472447156906128, + "learning_rate": 0.0001, + "loss": 56.1178, + "step": 13050 + }, + { + "epoch": 21.030634352798913, + "grad_norm": 0.5645695328712463, + "learning_rate": 0.0001, + "loss": 60.5944, + "step": 13060 + }, + { + "epoch": 21.046757696377288, + "grad_norm": 0.6419529318809509, + "learning_rate": 0.0001, + "loss": 59.5792, + "step": 13070 + }, + { + "epoch": 21.06288103995566, + "grad_norm": 0.6508122682571411, + "learning_rate": 0.0001, + "loss": 58.995, + "step": 13080 + }, + { + "epoch": 21.079004383534034, + "grad_norm": 0.6605037450790405, + "learning_rate": 0.0001, + "loss": 60.9783, + "step": 13090 + }, + { + "epoch": 21.09512772711241, + "grad_norm": 0.6636376976966858, + "learning_rate": 0.0001, + "loss": 60.3516, + "step": 13100 + }, + { + "epoch": 21.111251070690784, + "grad_norm": 0.7671985030174255, + "learning_rate": 0.0001, + "loss": 61.8918, + "step": 13110 + }, + { + "epoch": 21.12737441426916, + "grad_norm": 0.859834611415863, + "learning_rate": 0.0001, + "loss": 61.0766, + "step": 13120 + }, + { + "epoch": 21.143497757847534, + "grad_norm": 0.7551945447921753, + "learning_rate": 0.0001, + "loss": 60.6674, + "step": 13130 + }, + { + "epoch": 21.15962110142591, + "grad_norm": 0.6225672960281372, + "learning_rate": 0.0001, + "loss": 60.4107, + "step": 13140 + }, + { + "epoch": 21.175744445004284, + "grad_norm": 0.6100388765335083, + "learning_rate": 0.0001, + "loss": 58.8803, + "step": 13150 + }, + { + "epoch": 21.19186778858266, + "grad_norm": 0.669247567653656, + "learning_rate": 0.0001, + "loss": 60.4534, + "step": 13160 + }, + { + "epoch": 21.20799113216103, + "grad_norm": 0.8899680972099304, + "learning_rate": 0.0001, + "loss": 59.3081, + "step": 13170 + }, + { + "epoch": 21.224114475739405, + "grad_norm": 0.6804965138435364, + "learning_rate": 0.0001, + "loss": 58.1406, + "step": 13180 + }, + { + "epoch": 21.24023781931778, + "grad_norm": 0.7442401051521301, + "learning_rate": 0.0001, + "loss": 61.9136, + "step": 13190 + }, + { + "epoch": 21.256361162896155, + "grad_norm": 0.6071797013282776, + "learning_rate": 0.0001, + "loss": 59.7979, + "step": 13200 + }, + { + "epoch": 21.27248450647453, + "grad_norm": 0.6128804683685303, + "learning_rate": 0.0001, + "loss": 58.802, + "step": 13210 + }, + { + "epoch": 21.288607850052905, + "grad_norm": 0.6430484652519226, + "learning_rate": 0.0001, + "loss": 59.9492, + "step": 13220 + }, + { + "epoch": 21.30473119363128, + "grad_norm": 0.6414241194725037, + "learning_rate": 0.0001, + "loss": 60.699, + "step": 13230 + }, + { + "epoch": 21.320854537209655, + "grad_norm": 0.7222250699996948, + "learning_rate": 0.0001, + "loss": 58.5909, + "step": 13240 + }, + { + "epoch": 21.33697788078803, + "grad_norm": 1.0913405418395996, + "learning_rate": 0.0001, + "loss": 59.9657, + "step": 13250 + }, + { + "epoch": 21.3531012243664, + "grad_norm": 0.8245854377746582, + "learning_rate": 0.0001, + "loss": 60.2276, + "step": 13260 + }, + { + "epoch": 21.369224567944777, + "grad_norm": 0.7204747200012207, + "learning_rate": 0.0001, + "loss": 60.1854, + "step": 13270 + }, + { + "epoch": 21.38534791152315, + "grad_norm": 0.5877585411071777, + "learning_rate": 0.0001, + "loss": 60.8509, + "step": 13280 + }, + { + "epoch": 21.401471255101526, + "grad_norm": 0.906605064868927, + "learning_rate": 0.0001, + "loss": 57.9792, + "step": 13290 + }, + { + "epoch": 21.4175945986799, + "grad_norm": 0.910301923751831, + "learning_rate": 0.0001, + "loss": 58.986, + "step": 13300 + }, + { + "epoch": 21.433717942258276, + "grad_norm": 0.7098278403282166, + "learning_rate": 0.0001, + "loss": 60.6727, + "step": 13310 + }, + { + "epoch": 21.44984128583665, + "grad_norm": 0.842569887638092, + "learning_rate": 0.0001, + "loss": 59.3789, + "step": 13320 + }, + { + "epoch": 21.465964629415026, + "grad_norm": 0.7320014834403992, + "learning_rate": 0.0001, + "loss": 60.7055, + "step": 13330 + }, + { + "epoch": 21.482087972993398, + "grad_norm": 0.7192698121070862, + "learning_rate": 0.0001, + "loss": 59.7494, + "step": 13340 + }, + { + "epoch": 21.498211316571773, + "grad_norm": 0.6317388415336609, + "learning_rate": 0.0001, + "loss": 59.2083, + "step": 13350 + }, + { + "epoch": 21.514334660150148, + "grad_norm": 0.7012689709663391, + "learning_rate": 0.0001, + "loss": 58.6733, + "step": 13360 + }, + { + "epoch": 21.530458003728523, + "grad_norm": 0.8148412704467773, + "learning_rate": 0.0001, + "loss": 60.6708, + "step": 13370 + }, + { + "epoch": 21.546581347306898, + "grad_norm": 0.6740885376930237, + "learning_rate": 0.0001, + "loss": 60.2317, + "step": 13380 + }, + { + "epoch": 21.562704690885273, + "grad_norm": 0.6835291981697083, + "learning_rate": 0.0001, + "loss": 60.1807, + "step": 13390 + }, + { + "epoch": 21.578828034463648, + "grad_norm": 0.9604494571685791, + "learning_rate": 0.0001, + "loss": 60.811, + "step": 13400 + }, + { + "epoch": 21.594951378042023, + "grad_norm": 0.765917956829071, + "learning_rate": 0.0001, + "loss": 60.4539, + "step": 13410 + }, + { + "epoch": 21.611074721620398, + "grad_norm": 0.7830843329429626, + "learning_rate": 0.0001, + "loss": 59.2483, + "step": 13420 + }, + { + "epoch": 21.62719806519877, + "grad_norm": 0.6538785099983215, + "learning_rate": 0.0001, + "loss": 60.4062, + "step": 13430 + }, + { + "epoch": 21.643321408777144, + "grad_norm": 0.7341951727867126, + "learning_rate": 0.0001, + "loss": 59.3654, + "step": 13440 + }, + { + "epoch": 21.65944475235552, + "grad_norm": 0.684467077255249, + "learning_rate": 0.0001, + "loss": 58.9634, + "step": 13450 + }, + { + "epoch": 21.675568095933894, + "grad_norm": 0.6942781805992126, + "learning_rate": 0.0001, + "loss": 60.3677, + "step": 13460 + }, + { + "epoch": 21.69169143951227, + "grad_norm": 0.7780037522315979, + "learning_rate": 0.0001, + "loss": 61.6705, + "step": 13470 + }, + { + "epoch": 21.707814783090644, + "grad_norm": 0.8363171219825745, + "learning_rate": 0.0001, + "loss": 59.7802, + "step": 13480 + }, + { + "epoch": 21.72393812666902, + "grad_norm": 0.7392274737358093, + "learning_rate": 0.0001, + "loss": 59.8264, + "step": 13490 + }, + { + "epoch": 21.740061470247394, + "grad_norm": 0.6051466464996338, + "learning_rate": 0.0001, + "loss": 59.6685, + "step": 13500 + }, + { + "epoch": 21.75618481382577, + "grad_norm": 0.7772730588912964, + "learning_rate": 0.0001, + "loss": 58.261, + "step": 13510 + }, + { + "epoch": 21.77230815740414, + "grad_norm": 0.7481459379196167, + "learning_rate": 0.0001, + "loss": 59.3212, + "step": 13520 + }, + { + "epoch": 21.788431500982515, + "grad_norm": 0.8020729422569275, + "learning_rate": 0.0001, + "loss": 59.0113, + "step": 13530 + }, + { + "epoch": 21.80455484456089, + "grad_norm": 0.8504058122634888, + "learning_rate": 0.0001, + "loss": 59.7243, + "step": 13540 + }, + { + "epoch": 21.820678188139265, + "grad_norm": 0.7256322503089905, + "learning_rate": 0.0001, + "loss": 60.7478, + "step": 13550 + }, + { + "epoch": 21.83680153171764, + "grad_norm": 0.6632773876190186, + "learning_rate": 0.0001, + "loss": 59.2331, + "step": 13560 + }, + { + "epoch": 21.852924875296015, + "grad_norm": 0.6225188970565796, + "learning_rate": 0.0001, + "loss": 59.8263, + "step": 13570 + }, + { + "epoch": 21.86904821887439, + "grad_norm": 0.6302409768104553, + "learning_rate": 0.0001, + "loss": 60.8574, + "step": 13580 + }, + { + "epoch": 21.885171562452765, + "grad_norm": 0.7654368877410889, + "learning_rate": 0.0001, + "loss": 59.8633, + "step": 13590 + }, + { + "epoch": 21.90129490603114, + "grad_norm": 1.227220892906189, + "learning_rate": 0.0001, + "loss": 60.7098, + "step": 13600 + }, + { + "epoch": 21.91741824960951, + "grad_norm": 0.6679421067237854, + "learning_rate": 0.0001, + "loss": 59.0663, + "step": 13610 + }, + { + "epoch": 21.933541593187886, + "grad_norm": 0.8533850312232971, + "learning_rate": 0.0001, + "loss": 60.8597, + "step": 13620 + }, + { + "epoch": 21.94966493676626, + "grad_norm": 0.6417121887207031, + "learning_rate": 0.0001, + "loss": 60.1464, + "step": 13630 + }, + { + "epoch": 21.965788280344636, + "grad_norm": 0.679053544998169, + "learning_rate": 0.0001, + "loss": 60.5066, + "step": 13640 + }, + { + "epoch": 21.98191162392301, + "grad_norm": 0.6914607286453247, + "learning_rate": 0.0001, + "loss": 59.6815, + "step": 13650 + }, + { + "epoch": 21.998034967501386, + "grad_norm": 0.5661413669586182, + "learning_rate": 0.0001, + "loss": 61.6076, + "step": 13660 + }, + { + "epoch": 22.0128986748627, + "grad_norm": 1.4387016296386719, + "learning_rate": 0.0001, + "loss": 55.0652, + "step": 13670 + }, + { + "epoch": 22.029022018441076, + "grad_norm": 0.7774295210838318, + "learning_rate": 0.0001, + "loss": 61.4395, + "step": 13680 + }, + { + "epoch": 22.045145362019447, + "grad_norm": 0.7438349723815918, + "learning_rate": 0.0001, + "loss": 60.998, + "step": 13690 + }, + { + "epoch": 22.061268705597822, + "grad_norm": 0.7506164312362671, + "learning_rate": 0.0001, + "loss": 60.6929, + "step": 13700 + }, + { + "epoch": 22.077392049176197, + "grad_norm": 0.6262985467910767, + "learning_rate": 0.0001, + "loss": 59.4921, + "step": 13710 + }, + { + "epoch": 22.093515392754572, + "grad_norm": 1.826015591621399, + "learning_rate": 0.0001, + "loss": 61.3791, + "step": 13720 + }, + { + "epoch": 22.109638736332947, + "grad_norm": 0.6410622596740723, + "learning_rate": 0.0001, + "loss": 62.5485, + "step": 13730 + }, + { + "epoch": 22.125762079911322, + "grad_norm": 0.7042442560195923, + "learning_rate": 0.0001, + "loss": 60.0654, + "step": 13740 + }, + { + "epoch": 22.141885423489697, + "grad_norm": 0.741142988204956, + "learning_rate": 0.0001, + "loss": 59.9831, + "step": 13750 + }, + { + "epoch": 22.158008767068072, + "grad_norm": 0.836445152759552, + "learning_rate": 0.0001, + "loss": 59.6805, + "step": 13760 + }, + { + "epoch": 22.174132110646447, + "grad_norm": 0.8353415727615356, + "learning_rate": 0.0001, + "loss": 60.1912, + "step": 13770 + }, + { + "epoch": 22.19025545422482, + "grad_norm": 0.6416126489639282, + "learning_rate": 0.0001, + "loss": 59.5518, + "step": 13780 + }, + { + "epoch": 22.206378797803193, + "grad_norm": 0.8702544569969177, + "learning_rate": 0.0001, + "loss": 57.7428, + "step": 13790 + }, + { + "epoch": 22.222502141381568, + "grad_norm": 0.6681593656539917, + "learning_rate": 0.0001, + "loss": 59.9499, + "step": 13800 + }, + { + "epoch": 22.238625484959943, + "grad_norm": 0.6185410022735596, + "learning_rate": 0.0001, + "loss": 59.8062, + "step": 13810 + }, + { + "epoch": 22.254748828538318, + "grad_norm": 0.7695328593254089, + "learning_rate": 0.0001, + "loss": 61.3512, + "step": 13820 + }, + { + "epoch": 22.270872172116693, + "grad_norm": 0.6550581455230713, + "learning_rate": 0.0001, + "loss": 59.3379, + "step": 13830 + }, + { + "epoch": 22.286995515695068, + "grad_norm": 0.7764340043067932, + "learning_rate": 0.0001, + "loss": 59.984, + "step": 13840 + }, + { + "epoch": 22.303118859273443, + "grad_norm": 0.6377913355827332, + "learning_rate": 0.0001, + "loss": 59.1862, + "step": 13850 + }, + { + "epoch": 22.319242202851818, + "grad_norm": 0.7837334275245667, + "learning_rate": 0.0001, + "loss": 60.8087, + "step": 13860 + }, + { + "epoch": 22.33536554643019, + "grad_norm": 0.7966547012329102, + "learning_rate": 0.0001, + "loss": 59.9109, + "step": 13870 + }, + { + "epoch": 22.351488890008564, + "grad_norm": 0.6303540468215942, + "learning_rate": 0.0001, + "loss": 61.0486, + "step": 13880 + }, + { + "epoch": 22.36761223358694, + "grad_norm": 0.6628289818763733, + "learning_rate": 0.0001, + "loss": 61.7723, + "step": 13890 + }, + { + "epoch": 22.383735577165314, + "grad_norm": 1.2156575918197632, + "learning_rate": 0.0001, + "loss": 58.0992, + "step": 13900 + }, + { + "epoch": 22.39985892074369, + "grad_norm": 1.0548591613769531, + "learning_rate": 0.0001, + "loss": 58.9139, + "step": 13910 + }, + { + "epoch": 22.415982264322064, + "grad_norm": 0.6389120817184448, + "learning_rate": 0.0001, + "loss": 59.1494, + "step": 13920 + }, + { + "epoch": 22.43210560790044, + "grad_norm": 0.6384537220001221, + "learning_rate": 0.0001, + "loss": 59.7135, + "step": 13930 + }, + { + "epoch": 22.448228951478814, + "grad_norm": 0.7434266209602356, + "learning_rate": 0.0001, + "loss": 60.7941, + "step": 13940 + }, + { + "epoch": 22.46435229505719, + "grad_norm": 0.7207310199737549, + "learning_rate": 0.0001, + "loss": 57.9466, + "step": 13950 + }, + { + "epoch": 22.48047563863556, + "grad_norm": 0.9680810570716858, + "learning_rate": 0.0001, + "loss": 57.5014, + "step": 13960 + }, + { + "epoch": 22.496598982213936, + "grad_norm": 0.5354718565940857, + "learning_rate": 0.0001, + "loss": 59.8771, + "step": 13970 + }, + { + "epoch": 22.51272232579231, + "grad_norm": 1.1624388694763184, + "learning_rate": 0.0001, + "loss": 60.2305, + "step": 13980 + }, + { + "epoch": 22.528845669370686, + "grad_norm": 0.6650974154472351, + "learning_rate": 0.0001, + "loss": 58.1649, + "step": 13990 + }, + { + "epoch": 22.54496901294906, + "grad_norm": 0.6151170134544373, + "learning_rate": 0.0001, + "loss": 60.4392, + "step": 14000 + }, + { + "epoch": 22.561092356527435, + "grad_norm": 0.6685694456100464, + "learning_rate": 0.0001, + "loss": 60.7055, + "step": 14010 + }, + { + "epoch": 22.57721570010581, + "grad_norm": 0.8092939257621765, + "learning_rate": 0.0001, + "loss": 57.8347, + "step": 14020 + }, + { + "epoch": 22.593339043684185, + "grad_norm": 0.7520709037780762, + "learning_rate": 0.0001, + "loss": 60.359, + "step": 14030 + }, + { + "epoch": 22.609462387262557, + "grad_norm": 0.8913780450820923, + "learning_rate": 0.0001, + "loss": 59.0833, + "step": 14040 + }, + { + "epoch": 22.625585730840932, + "grad_norm": 0.5508324503898621, + "learning_rate": 0.0001, + "loss": 60.8168, + "step": 14050 + }, + { + "epoch": 22.641709074419307, + "grad_norm": 0.7124258279800415, + "learning_rate": 0.0001, + "loss": 60.9433, + "step": 14060 + }, + { + "epoch": 22.65783241799768, + "grad_norm": 0.7286089062690735, + "learning_rate": 0.0001, + "loss": 59.8127, + "step": 14070 + }, + { + "epoch": 22.673955761576057, + "grad_norm": 0.8104858994483948, + "learning_rate": 0.0001, + "loss": 59.764, + "step": 14080 + }, + { + "epoch": 22.69007910515443, + "grad_norm": 0.7618247270584106, + "learning_rate": 0.0001, + "loss": 59.5563, + "step": 14090 + }, + { + "epoch": 22.706202448732807, + "grad_norm": 0.8512730598449707, + "learning_rate": 0.0001, + "loss": 59.3122, + "step": 14100 + }, + { + "epoch": 22.72232579231118, + "grad_norm": 0.6115924715995789, + "learning_rate": 0.0001, + "loss": 61.2874, + "step": 14110 + }, + { + "epoch": 22.738449135889557, + "grad_norm": 0.6883788704872131, + "learning_rate": 0.0001, + "loss": 59.427, + "step": 14120 + }, + { + "epoch": 22.754572479467928, + "grad_norm": 0.6240411400794983, + "learning_rate": 0.0001, + "loss": 58.9808, + "step": 14130 + }, + { + "epoch": 22.770695823046303, + "grad_norm": 0.6807436347007751, + "learning_rate": 0.0001, + "loss": 58.9609, + "step": 14140 + }, + { + "epoch": 22.786819166624678, + "grad_norm": 0.6661691665649414, + "learning_rate": 0.0001, + "loss": 59.8573, + "step": 14150 + }, + { + "epoch": 22.802942510203053, + "grad_norm": 0.7038732171058655, + "learning_rate": 0.0001, + "loss": 59.0114, + "step": 14160 + }, + { + "epoch": 22.819065853781428, + "grad_norm": 0.7736789584159851, + "learning_rate": 0.0001, + "loss": 59.6709, + "step": 14170 + }, + { + "epoch": 22.835189197359803, + "grad_norm": 0.6095895767211914, + "learning_rate": 0.0001, + "loss": 60.4468, + "step": 14180 + }, + { + "epoch": 22.851312540938178, + "grad_norm": 0.6531018018722534, + "learning_rate": 0.0001, + "loss": 61.4419, + "step": 14190 + }, + { + "epoch": 22.867435884516553, + "grad_norm": 0.7522090077400208, + "learning_rate": 0.0001, + "loss": 57.6574, + "step": 14200 + }, + { + "epoch": 22.883559228094928, + "grad_norm": 0.8028358817100525, + "learning_rate": 0.0001, + "loss": 59.0353, + "step": 14210 + }, + { + "epoch": 22.8996825716733, + "grad_norm": 0.7619246244430542, + "learning_rate": 0.0001, + "loss": 59.3586, + "step": 14220 + }, + { + "epoch": 22.915805915251674, + "grad_norm": 0.8312122821807861, + "learning_rate": 0.0001, + "loss": 60.8772, + "step": 14230 + }, + { + "epoch": 22.93192925883005, + "grad_norm": 0.6536790132522583, + "learning_rate": 0.0001, + "loss": 60.5941, + "step": 14240 + }, + { + "epoch": 22.948052602408424, + "grad_norm": 0.6973451972007751, + "learning_rate": 0.0001, + "loss": 59.5332, + "step": 14250 + }, + { + "epoch": 22.9641759459868, + "grad_norm": 0.6776084899902344, + "learning_rate": 0.0001, + "loss": 60.2337, + "step": 14260 + }, + { + "epoch": 22.980299289565174, + "grad_norm": 0.5728079080581665, + "learning_rate": 0.0001, + "loss": 61.0065, + "step": 14270 + }, + { + "epoch": 22.99642263314355, + "grad_norm": 0.798130452632904, + "learning_rate": 0.0001, + "loss": 59.7552, + "step": 14280 + }, + { + "epoch": 23.011286340504864, + "grad_norm": 0.6825193166732788, + "learning_rate": 0.0001, + "loss": 54.8593, + "step": 14290 + }, + { + "epoch": 23.02740968408324, + "grad_norm": 0.794450044631958, + "learning_rate": 0.0001, + "loss": 58.1472, + "step": 14300 + }, + { + "epoch": 23.04353302766161, + "grad_norm": 0.8178662657737732, + "learning_rate": 0.0001, + "loss": 58.8868, + "step": 14310 + }, + { + "epoch": 23.059656371239985, + "grad_norm": 0.6562325358390808, + "learning_rate": 0.0001, + "loss": 59.2406, + "step": 14320 + }, + { + "epoch": 23.07577971481836, + "grad_norm": 0.7282053828239441, + "learning_rate": 0.0001, + "loss": 58.1195, + "step": 14330 + }, + { + "epoch": 23.091903058396735, + "grad_norm": 0.6804936528205872, + "learning_rate": 0.0001, + "loss": 60.3755, + "step": 14340 + }, + { + "epoch": 23.10802640197511, + "grad_norm": 0.5851950645446777, + "learning_rate": 0.0001, + "loss": 59.2353, + "step": 14350 + }, + { + "epoch": 23.124149745553485, + "grad_norm": 0.5784706473350525, + "learning_rate": 0.0001, + "loss": 59.351, + "step": 14360 + }, + { + "epoch": 23.14027308913186, + "grad_norm": 0.7353758215904236, + "learning_rate": 0.0001, + "loss": 60.7708, + "step": 14370 + }, + { + "epoch": 23.156396432710235, + "grad_norm": 0.7761120796203613, + "learning_rate": 0.0001, + "loss": 59.6935, + "step": 14380 + }, + { + "epoch": 23.172519776288606, + "grad_norm": 0.6392366290092468, + "learning_rate": 0.0001, + "loss": 61.0263, + "step": 14390 + }, + { + "epoch": 23.18864311986698, + "grad_norm": 0.5722920894622803, + "learning_rate": 0.0001, + "loss": 59.4247, + "step": 14400 + }, + { + "epoch": 23.204766463445356, + "grad_norm": 0.6938275694847107, + "learning_rate": 0.0001, + "loss": 61.0933, + "step": 14410 + }, + { + "epoch": 23.22088980702373, + "grad_norm": 0.9164982438087463, + "learning_rate": 0.0001, + "loss": 58.0045, + "step": 14420 + }, + { + "epoch": 23.237013150602106, + "grad_norm": 0.6641019582748413, + "learning_rate": 0.0001, + "loss": 62.2162, + "step": 14430 + }, + { + "epoch": 23.25313649418048, + "grad_norm": 0.6775721311569214, + "learning_rate": 0.0001, + "loss": 60.2123, + "step": 14440 + }, + { + "epoch": 23.269259837758856, + "grad_norm": 0.6306013464927673, + "learning_rate": 0.0001, + "loss": 60.0638, + "step": 14450 + }, + { + "epoch": 23.28538318133723, + "grad_norm": 0.7770274877548218, + "learning_rate": 0.0001, + "loss": 60.87, + "step": 14460 + }, + { + "epoch": 23.301506524915606, + "grad_norm": 0.6893128752708435, + "learning_rate": 0.0001, + "loss": 59.2247, + "step": 14470 + }, + { + "epoch": 23.317629868493977, + "grad_norm": 0.7028453946113586, + "learning_rate": 0.0001, + "loss": 60.2405, + "step": 14480 + }, + { + "epoch": 23.333753212072352, + "grad_norm": 0.6184173822402954, + "learning_rate": 0.0001, + "loss": 58.5765, + "step": 14490 + }, + { + "epoch": 23.349876555650727, + "grad_norm": 0.793502151966095, + "learning_rate": 0.0001, + "loss": 57.5978, + "step": 14500 + }, + { + "epoch": 23.365999899229102, + "grad_norm": 0.660885751247406, + "learning_rate": 0.0001, + "loss": 60.5535, + "step": 14510 + }, + { + "epoch": 23.382123242807477, + "grad_norm": 0.9039516448974609, + "learning_rate": 0.0001, + "loss": 58.9432, + "step": 14520 + }, + { + "epoch": 23.398246586385852, + "grad_norm": 0.7348603010177612, + "learning_rate": 0.0001, + "loss": 58.3708, + "step": 14530 + }, + { + "epoch": 23.414369929964227, + "grad_norm": 0.5571108460426331, + "learning_rate": 0.0001, + "loss": 60.2059, + "step": 14540 + }, + { + "epoch": 23.430493273542602, + "grad_norm": 0.77266526222229, + "learning_rate": 0.0001, + "loss": 59.9602, + "step": 14550 + }, + { + "epoch": 23.446616617120977, + "grad_norm": 0.7564616203308105, + "learning_rate": 0.0001, + "loss": 59.0114, + "step": 14560 + }, + { + "epoch": 23.46273996069935, + "grad_norm": 1.0693135261535645, + "learning_rate": 0.0001, + "loss": 60.6075, + "step": 14570 + }, + { + "epoch": 23.478863304277724, + "grad_norm": 0.7800086140632629, + "learning_rate": 0.0001, + "loss": 60.6608, + "step": 14580 + }, + { + "epoch": 23.4949866478561, + "grad_norm": 0.8552407026290894, + "learning_rate": 0.0001, + "loss": 60.2378, + "step": 14590 + }, + { + "epoch": 23.511109991434473, + "grad_norm": 0.7270957231521606, + "learning_rate": 0.0001, + "loss": 60.8936, + "step": 14600 + }, + { + "epoch": 23.52723333501285, + "grad_norm": 0.5652869343757629, + "learning_rate": 0.0001, + "loss": 59.9882, + "step": 14610 + }, + { + "epoch": 23.543356678591223, + "grad_norm": 0.5465036034584045, + "learning_rate": 0.0001, + "loss": 57.4725, + "step": 14620 + }, + { + "epoch": 23.5594800221696, + "grad_norm": 0.6404461860656738, + "learning_rate": 0.0001, + "loss": 60.3962, + "step": 14630 + }, + { + "epoch": 23.575603365747973, + "grad_norm": 0.7860730886459351, + "learning_rate": 0.0001, + "loss": 60.3245, + "step": 14640 + }, + { + "epoch": 23.591726709326345, + "grad_norm": 0.6626171469688416, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 14650 + }, + { + "epoch": 23.60785005290472, + "grad_norm": 1.5377638339996338, + "learning_rate": 0.0001, + "loss": 58.9615, + "step": 14660 + }, + { + "epoch": 23.623973396483095, + "grad_norm": 0.6656123995780945, + "learning_rate": 0.0001, + "loss": 60.7947, + "step": 14670 + }, + { + "epoch": 23.64009674006147, + "grad_norm": 0.5155975818634033, + "learning_rate": 0.0001, + "loss": 57.7742, + "step": 14680 + }, + { + "epoch": 23.656220083639845, + "grad_norm": 0.6278973817825317, + "learning_rate": 0.0001, + "loss": 58.6188, + "step": 14690 + }, + { + "epoch": 23.67234342721822, + "grad_norm": 0.7261143326759338, + "learning_rate": 0.0001, + "loss": 56.8495, + "step": 14700 + }, + { + "epoch": 23.688466770796595, + "grad_norm": 0.8105781078338623, + "learning_rate": 0.0001, + "loss": 57.3139, + "step": 14710 + }, + { + "epoch": 23.70459011437497, + "grad_norm": 0.7012073993682861, + "learning_rate": 0.0001, + "loss": 61.1186, + "step": 14720 + }, + { + "epoch": 23.720713457953345, + "grad_norm": 0.6352536082267761, + "learning_rate": 0.0001, + "loss": 59.0136, + "step": 14730 + }, + { + "epoch": 23.736836801531716, + "grad_norm": 0.7279717326164246, + "learning_rate": 0.0001, + "loss": 58.7408, + "step": 14740 + }, + { + "epoch": 23.75296014511009, + "grad_norm": 0.6714015603065491, + "learning_rate": 0.0001, + "loss": 58.6431, + "step": 14750 + }, + { + "epoch": 23.769083488688466, + "grad_norm": 1.3876261711120605, + "learning_rate": 0.0001, + "loss": 59.7586, + "step": 14760 + }, + { + "epoch": 23.78520683226684, + "grad_norm": 0.7192941904067993, + "learning_rate": 0.0001, + "loss": 58.8487, + "step": 14770 + }, + { + "epoch": 23.801330175845216, + "grad_norm": 0.7333327531814575, + "learning_rate": 0.0001, + "loss": 60.5608, + "step": 14780 + }, + { + "epoch": 23.81745351942359, + "grad_norm": 0.8186061382293701, + "learning_rate": 0.0001, + "loss": 59.4232, + "step": 14790 + }, + { + "epoch": 23.833576863001966, + "grad_norm": 1.037039041519165, + "learning_rate": 0.0001, + "loss": 58.2384, + "step": 14800 + }, + { + "epoch": 23.84970020658034, + "grad_norm": 0.6739209890365601, + "learning_rate": 0.0001, + "loss": 57.7256, + "step": 14810 + }, + { + "epoch": 23.865823550158716, + "grad_norm": 0.6515845656394958, + "learning_rate": 0.0001, + "loss": 61.0039, + "step": 14820 + }, + { + "epoch": 23.881946893737087, + "grad_norm": 0.559317946434021, + "learning_rate": 0.0001, + "loss": 60.4458, + "step": 14830 + }, + { + "epoch": 23.898070237315462, + "grad_norm": 0.663934588432312, + "learning_rate": 0.0001, + "loss": 59.1646, + "step": 14840 + }, + { + "epoch": 23.914193580893837, + "grad_norm": 0.6409069299697876, + "learning_rate": 0.0001, + "loss": 58.9508, + "step": 14850 + }, + { + "epoch": 23.930316924472212, + "grad_norm": 0.7624269127845764, + "learning_rate": 0.0001, + "loss": 60.7256, + "step": 14860 + }, + { + "epoch": 23.946440268050587, + "grad_norm": 0.6949140429496765, + "learning_rate": 0.0001, + "loss": 58.8987, + "step": 14870 + }, + { + "epoch": 23.962563611628962, + "grad_norm": 0.714063823223114, + "learning_rate": 0.0001, + "loss": 59.255, + "step": 14880 + }, + { + "epoch": 23.978686955207337, + "grad_norm": 0.6570878028869629, + "learning_rate": 0.0001, + "loss": 59.2637, + "step": 14890 + }, + { + "epoch": 23.994810298785712, + "grad_norm": 0.7898263335227966, + "learning_rate": 0.0001, + "loss": 61.0218, + "step": 14900 + }, + { + "epoch": 24.009674006147026, + "grad_norm": 0.6673877239227295, + "learning_rate": 0.0001, + "loss": 53.8562, + "step": 14910 + }, + { + "epoch": 24.025797349725398, + "grad_norm": 0.5548285245895386, + "learning_rate": 0.0001, + "loss": 59.4445, + "step": 14920 + }, + { + "epoch": 24.041920693303773, + "grad_norm": 0.6273561716079712, + "learning_rate": 0.0001, + "loss": 59.3292, + "step": 14930 + }, + { + "epoch": 24.058044036882148, + "grad_norm": 0.7050267457962036, + "learning_rate": 0.0001, + "loss": 60.4896, + "step": 14940 + }, + { + "epoch": 24.074167380460523, + "grad_norm": 0.7030066251754761, + "learning_rate": 0.0001, + "loss": 59.1622, + "step": 14950 + }, + { + "epoch": 24.090290724038898, + "grad_norm": 0.8014484643936157, + "learning_rate": 0.0001, + "loss": 59.746, + "step": 14960 + }, + { + "epoch": 24.106414067617273, + "grad_norm": 0.869323194026947, + "learning_rate": 0.0001, + "loss": 60.2156, + "step": 14970 + }, + { + "epoch": 24.122537411195648, + "grad_norm": 0.7157219648361206, + "learning_rate": 0.0001, + "loss": 58.7229, + "step": 14980 + }, + { + "epoch": 24.138660754774023, + "grad_norm": 0.6318342685699463, + "learning_rate": 0.0001, + "loss": 61.1832, + "step": 14990 + }, + { + "epoch": 24.154784098352394, + "grad_norm": 0.8689974546432495, + "learning_rate": 0.0001, + "loss": 58.8083, + "step": 15000 + }, + { + "epoch": 24.17090744193077, + "grad_norm": 1.1719586849212646, + "learning_rate": 0.0001, + "loss": 57.745, + "step": 15010 + }, + { + "epoch": 24.187030785509144, + "grad_norm": 0.676295280456543, + "learning_rate": 0.0001, + "loss": 60.6236, + "step": 15020 + }, + { + "epoch": 24.20315412908752, + "grad_norm": 0.6745544672012329, + "learning_rate": 0.0001, + "loss": 59.494, + "step": 15030 + }, + { + "epoch": 24.219277472665894, + "grad_norm": 0.6221221685409546, + "learning_rate": 0.0001, + "loss": 58.894, + "step": 15040 + }, + { + "epoch": 24.23540081624427, + "grad_norm": 0.7013781070709229, + "learning_rate": 0.0001, + "loss": 59.935, + "step": 15050 + }, + { + "epoch": 24.251524159822644, + "grad_norm": 0.6094849705696106, + "learning_rate": 0.0001, + "loss": 59.8276, + "step": 15060 + }, + { + "epoch": 24.26764750340102, + "grad_norm": 0.6969972848892212, + "learning_rate": 0.0001, + "loss": 59.7568, + "step": 15070 + }, + { + "epoch": 24.283770846979394, + "grad_norm": 0.7223725318908691, + "learning_rate": 0.0001, + "loss": 57.9065, + "step": 15080 + }, + { + "epoch": 24.299894190557765, + "grad_norm": 0.7008336186408997, + "learning_rate": 0.0001, + "loss": 59.5206, + "step": 15090 + }, + { + "epoch": 24.31601753413614, + "grad_norm": 0.6490783095359802, + "learning_rate": 0.0001, + "loss": 59.4457, + "step": 15100 + }, + { + "epoch": 24.332140877714515, + "grad_norm": 0.6096290349960327, + "learning_rate": 0.0001, + "loss": 60.8434, + "step": 15110 + }, + { + "epoch": 24.34826422129289, + "grad_norm": 0.771045446395874, + "learning_rate": 0.0001, + "loss": 60.3167, + "step": 15120 + }, + { + "epoch": 24.364387564871265, + "grad_norm": 0.7178729772567749, + "learning_rate": 0.0001, + "loss": 57.6667, + "step": 15130 + }, + { + "epoch": 24.38051090844964, + "grad_norm": 0.6484465003013611, + "learning_rate": 0.0001, + "loss": 60.1199, + "step": 15140 + }, + { + "epoch": 24.396634252028015, + "grad_norm": 1.0030865669250488, + "learning_rate": 0.0001, + "loss": 57.9288, + "step": 15150 + }, + { + "epoch": 24.41275759560639, + "grad_norm": 0.8297843933105469, + "learning_rate": 0.0001, + "loss": 61.0941, + "step": 15160 + }, + { + "epoch": 24.428880939184765, + "grad_norm": 0.9115694165229797, + "learning_rate": 0.0001, + "loss": 58.9574, + "step": 15170 + }, + { + "epoch": 24.445004282763136, + "grad_norm": 0.7758508920669556, + "learning_rate": 0.0001, + "loss": 58.8852, + "step": 15180 + }, + { + "epoch": 24.46112762634151, + "grad_norm": 0.5539108514785767, + "learning_rate": 0.0001, + "loss": 60.9549, + "step": 15190 + }, + { + "epoch": 24.477250969919886, + "grad_norm": 0.7292829751968384, + "learning_rate": 0.0001, + "loss": 58.0818, + "step": 15200 + }, + { + "epoch": 24.49337431349826, + "grad_norm": 0.6253948211669922, + "learning_rate": 0.0001, + "loss": 59.656, + "step": 15210 + }, + { + "epoch": 24.509497657076636, + "grad_norm": 0.8054394125938416, + "learning_rate": 0.0001, + "loss": 60.934, + "step": 15220 + }, + { + "epoch": 24.52562100065501, + "grad_norm": 0.8183079361915588, + "learning_rate": 0.0001, + "loss": 59.5303, + "step": 15230 + }, + { + "epoch": 24.541744344233386, + "grad_norm": 0.6063419580459595, + "learning_rate": 0.0001, + "loss": 59.2852, + "step": 15240 + }, + { + "epoch": 24.55786768781176, + "grad_norm": 0.6372854113578796, + "learning_rate": 0.0001, + "loss": 58.9647, + "step": 15250 + }, + { + "epoch": 24.573991031390136, + "grad_norm": 0.6270446181297302, + "learning_rate": 0.0001, + "loss": 58.5641, + "step": 15260 + }, + { + "epoch": 24.590114374968508, + "grad_norm": 0.6229680776596069, + "learning_rate": 0.0001, + "loss": 58.4142, + "step": 15270 + }, + { + "epoch": 24.606237718546883, + "grad_norm": 0.6567183136940002, + "learning_rate": 0.0001, + "loss": 60.1162, + "step": 15280 + }, + { + "epoch": 24.622361062125258, + "grad_norm": 0.6975446343421936, + "learning_rate": 0.0001, + "loss": 58.7862, + "step": 15290 + }, + { + "epoch": 24.638484405703633, + "grad_norm": 0.6961022615432739, + "learning_rate": 0.0001, + "loss": 58.3247, + "step": 15300 + }, + { + "epoch": 24.654607749282007, + "grad_norm": 5.25744104385376, + "learning_rate": 0.0001, + "loss": 59.3863, + "step": 15310 + }, + { + "epoch": 24.670731092860382, + "grad_norm": 0.8439975380897522, + "learning_rate": 0.0001, + "loss": 59.7157, + "step": 15320 + }, + { + "epoch": 24.686854436438757, + "grad_norm": 0.596845269203186, + "learning_rate": 0.0001, + "loss": 60.0859, + "step": 15330 + }, + { + "epoch": 24.702977780017132, + "grad_norm": 0.6758428812026978, + "learning_rate": 0.0001, + "loss": 59.9305, + "step": 15340 + }, + { + "epoch": 24.719101123595507, + "grad_norm": 0.7418860197067261, + "learning_rate": 0.0001, + "loss": 59.0929, + "step": 15350 + }, + { + "epoch": 24.73522446717388, + "grad_norm": 0.6856047511100769, + "learning_rate": 0.0001, + "loss": 58.7201, + "step": 15360 + }, + { + "epoch": 24.751347810752254, + "grad_norm": 0.5327897071838379, + "learning_rate": 0.0001, + "loss": 59.5899, + "step": 15370 + }, + { + "epoch": 24.76747115433063, + "grad_norm": 0.7127685546875, + "learning_rate": 0.0001, + "loss": 59.7729, + "step": 15380 + }, + { + "epoch": 24.783594497909004, + "grad_norm": 1.8885949850082397, + "learning_rate": 0.0001, + "loss": 58.3813, + "step": 15390 + }, + { + "epoch": 24.79971784148738, + "grad_norm": 0.7997881174087524, + "learning_rate": 0.0001, + "loss": 61.9402, + "step": 15400 + }, + { + "epoch": 24.815841185065754, + "grad_norm": 0.9729613661766052, + "learning_rate": 0.0001, + "loss": 59.7961, + "step": 15410 + }, + { + "epoch": 24.83196452864413, + "grad_norm": 0.8450629115104675, + "learning_rate": 0.0001, + "loss": 59.5969, + "step": 15420 + }, + { + "epoch": 24.848087872222504, + "grad_norm": 0.720668613910675, + "learning_rate": 0.0001, + "loss": 58.187, + "step": 15430 + }, + { + "epoch": 24.864211215800875, + "grad_norm": 0.638506293296814, + "learning_rate": 0.0001, + "loss": 59.4599, + "step": 15440 + }, + { + "epoch": 24.88033455937925, + "grad_norm": 0.6158926486968994, + "learning_rate": 0.0001, + "loss": 61.3858, + "step": 15450 + }, + { + "epoch": 24.896457902957625, + "grad_norm": 0.8130396604537964, + "learning_rate": 0.0001, + "loss": 59.3795, + "step": 15460 + }, + { + "epoch": 24.912581246536, + "grad_norm": 0.7966095805168152, + "learning_rate": 0.0001, + "loss": 60.3319, + "step": 15470 + }, + { + "epoch": 24.928704590114375, + "grad_norm": 0.6240444779396057, + "learning_rate": 0.0001, + "loss": 59.0407, + "step": 15480 + }, + { + "epoch": 24.94482793369275, + "grad_norm": 0.7439318895339966, + "learning_rate": 0.0001, + "loss": 57.7692, + "step": 15490 + }, + { + "epoch": 24.960951277271125, + "grad_norm": 0.6942011713981628, + "learning_rate": 0.0001, + "loss": 56.8418, + "step": 15500 + }, + { + "epoch": 24.9770746208495, + "grad_norm": 0.6470242738723755, + "learning_rate": 0.0001, + "loss": 59.8361, + "step": 15510 + }, + { + "epoch": 24.993197964427875, + "grad_norm": 0.6935222148895264, + "learning_rate": 0.0001, + "loss": 59.5357, + "step": 15520 + }, + { + "epoch": 25.008061671789186, + "grad_norm": 0.8943103551864624, + "learning_rate": 0.0001, + "loss": 54.8185, + "step": 15530 + }, + { + "epoch": 25.02418501536756, + "grad_norm": 0.7637899518013, + "learning_rate": 0.0001, + "loss": 58.4746, + "step": 15540 + }, + { + "epoch": 25.040308358945936, + "grad_norm": 0.8261918425559998, + "learning_rate": 0.0001, + "loss": 57.3993, + "step": 15550 + }, + { + "epoch": 25.05643170252431, + "grad_norm": 0.6395642161369324, + "learning_rate": 0.0001, + "loss": 58.123, + "step": 15560 + }, + { + "epoch": 25.072555046102686, + "grad_norm": 0.7899443507194519, + "learning_rate": 0.0001, + "loss": 57.1487, + "step": 15570 + }, + { + "epoch": 25.08867838968106, + "grad_norm": 0.6202112436294556, + "learning_rate": 0.0001, + "loss": 59.4995, + "step": 15580 + }, + { + "epoch": 25.104801733259436, + "grad_norm": 0.7683939933776855, + "learning_rate": 0.0001, + "loss": 59.1155, + "step": 15590 + }, + { + "epoch": 25.12092507683781, + "grad_norm": 0.6531215906143188, + "learning_rate": 0.0001, + "loss": 61.9998, + "step": 15600 + }, + { + "epoch": 25.137048420416185, + "grad_norm": 0.6354383826255798, + "learning_rate": 0.0001, + "loss": 59.3861, + "step": 15610 + }, + { + "epoch": 25.153171763994557, + "grad_norm": 0.6507435441017151, + "learning_rate": 0.0001, + "loss": 59.939, + "step": 15620 + }, + { + "epoch": 25.169295107572932, + "grad_norm": 0.8014065623283386, + "learning_rate": 0.0001, + "loss": 59.4927, + "step": 15630 + }, + { + "epoch": 25.185418451151307, + "grad_norm": 0.53805011510849, + "learning_rate": 0.0001, + "loss": 58.526, + "step": 15640 + }, + { + "epoch": 25.201541794729682, + "grad_norm": 4.835650444030762, + "learning_rate": 0.0001, + "loss": 58.6711, + "step": 15650 + }, + { + "epoch": 25.217665138308057, + "grad_norm": 0.723210871219635, + "learning_rate": 0.0001, + "loss": 60.3953, + "step": 15660 + }, + { + "epoch": 25.23378848188643, + "grad_norm": 0.7176834940910339, + "learning_rate": 0.0001, + "loss": 58.8576, + "step": 15670 + }, + { + "epoch": 25.249911825464807, + "grad_norm": 0.8280025124549866, + "learning_rate": 0.0001, + "loss": 58.7146, + "step": 15680 + }, + { + "epoch": 25.26603516904318, + "grad_norm": 0.6296973824501038, + "learning_rate": 0.0001, + "loss": 60.7341, + "step": 15690 + }, + { + "epoch": 25.282158512621557, + "grad_norm": 1.3184518814086914, + "learning_rate": 0.0001, + "loss": 60.6906, + "step": 15700 + }, + { + "epoch": 25.298281856199928, + "grad_norm": 1.0408564805984497, + "learning_rate": 0.0001, + "loss": 57.4703, + "step": 15710 + }, + { + "epoch": 25.314405199778303, + "grad_norm": 0.7140268683433533, + "learning_rate": 0.0001, + "loss": 59.8452, + "step": 15720 + }, + { + "epoch": 25.330528543356678, + "grad_norm": 0.7822073698043823, + "learning_rate": 0.0001, + "loss": 60.4394, + "step": 15730 + }, + { + "epoch": 25.346651886935053, + "grad_norm": 0.6579483151435852, + "learning_rate": 0.0001, + "loss": 59.5358, + "step": 15740 + }, + { + "epoch": 25.362775230513428, + "grad_norm": 0.6936199069023132, + "learning_rate": 0.0001, + "loss": 60.2117, + "step": 15750 + }, + { + "epoch": 25.378898574091803, + "grad_norm": 0.7166867852210999, + "learning_rate": 0.0001, + "loss": 58.6835, + "step": 15760 + }, + { + "epoch": 25.395021917670178, + "grad_norm": 0.7240379452705383, + "learning_rate": 0.0001, + "loss": 59.6293, + "step": 15770 + }, + { + "epoch": 25.411145261248553, + "grad_norm": 0.7465791702270508, + "learning_rate": 0.0001, + "loss": 58.2512, + "step": 15780 + }, + { + "epoch": 25.427268604826924, + "grad_norm": 0.6713439226150513, + "learning_rate": 0.0001, + "loss": 58.4149, + "step": 15790 + }, + { + "epoch": 25.4433919484053, + "grad_norm": 0.587433397769928, + "learning_rate": 0.0001, + "loss": 60.7315, + "step": 15800 + }, + { + "epoch": 25.459515291983674, + "grad_norm": 0.8216325044631958, + "learning_rate": 0.0001, + "loss": 58.5339, + "step": 15810 + }, + { + "epoch": 25.47563863556205, + "grad_norm": 0.8947033286094666, + "learning_rate": 0.0001, + "loss": 58.7762, + "step": 15820 + }, + { + "epoch": 25.491761979140424, + "grad_norm": 0.6659840941429138, + "learning_rate": 0.0001, + "loss": 58.9931, + "step": 15830 + }, + { + "epoch": 25.5078853227188, + "grad_norm": 0.7239252328872681, + "learning_rate": 0.0001, + "loss": 59.7113, + "step": 15840 + }, + { + "epoch": 25.524008666297174, + "grad_norm": 0.7233046889305115, + "learning_rate": 0.0001, + "loss": 58.973, + "step": 15850 + }, + { + "epoch": 25.54013200987555, + "grad_norm": 0.6620002388954163, + "learning_rate": 0.0001, + "loss": 57.1255, + "step": 15860 + }, + { + "epoch": 25.556255353453924, + "grad_norm": 0.6544366478919983, + "learning_rate": 0.0001, + "loss": 58.5897, + "step": 15870 + }, + { + "epoch": 25.572378697032295, + "grad_norm": 0.8955110311508179, + "learning_rate": 0.0001, + "loss": 58.3241, + "step": 15880 + }, + { + "epoch": 25.58850204061067, + "grad_norm": 0.6869794726371765, + "learning_rate": 0.0001, + "loss": 58.1951, + "step": 15890 + }, + { + "epoch": 25.604625384189045, + "grad_norm": 0.6819241642951965, + "learning_rate": 0.0001, + "loss": 58.6505, + "step": 15900 + }, + { + "epoch": 25.62074872776742, + "grad_norm": 0.6019999980926514, + "learning_rate": 0.0001, + "loss": 58.1397, + "step": 15910 + }, + { + "epoch": 25.636872071345795, + "grad_norm": 0.6664146184921265, + "learning_rate": 0.0001, + "loss": 58.6735, + "step": 15920 + }, + { + "epoch": 25.65299541492417, + "grad_norm": 0.7511261701583862, + "learning_rate": 0.0001, + "loss": 58.9189, + "step": 15930 + }, + { + "epoch": 25.669118758502545, + "grad_norm": 0.6886443495750427, + "learning_rate": 0.0001, + "loss": 60.1288, + "step": 15940 + }, + { + "epoch": 25.68524210208092, + "grad_norm": 0.7647057771682739, + "learning_rate": 0.0001, + "loss": 58.5042, + "step": 15950 + }, + { + "epoch": 25.701365445659295, + "grad_norm": 0.6636016368865967, + "learning_rate": 0.0001, + "loss": 56.7644, + "step": 15960 + }, + { + "epoch": 25.717488789237667, + "grad_norm": 0.6909965872764587, + "learning_rate": 0.0001, + "loss": 58.9248, + "step": 15970 + }, + { + "epoch": 25.73361213281604, + "grad_norm": 0.7853394150733948, + "learning_rate": 0.0001, + "loss": 58.7267, + "step": 15980 + }, + { + "epoch": 25.749735476394417, + "grad_norm": 0.5992239713668823, + "learning_rate": 0.0001, + "loss": 58.4463, + "step": 15990 + }, + { + "epoch": 25.76585881997279, + "grad_norm": 0.5730507969856262, + "learning_rate": 0.0001, + "loss": 58.2946, + "step": 16000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 33, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9568102993234493e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ade09d5233289ba4bc5076af39b8652078582dee --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016a4e9b5d71a1947317d3801b00fe5d0604edbef1f981d7b6c92f219131717c +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-16000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2012678b2b54f1337ae97bc43611724c5f342742 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:332fe55107ee4c63d84845e2fcab921abaa89e579ca93df293cbb03ac4c1a7c0 +size 1853729968 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f4727f32e65aa7ec6ffc8f6b7e9af9a44bcd717 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afd7d5020b1b9dac2b6bc61350c8b94dd9fc150b1ed70417cac690b7ac08f486 +size 1853730480 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..79930aaf17cf689c753e8cbdba09ff4bf1098ef0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd57ed647f734cbf9e33755aca0cdc93b9c29be8161e05930c24caaf62951ba1 +size 1853731312 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f23980e4b3714d49605794ecbc1ebaad3b10eb5 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddd9ce8bade1155fdacd74cb82e6a9cbf061f17fb13c2825421a42fb21c095a4 +size 1853731120 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c685896ec5699ca08e0f2367c979d5ff065a7cc6 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e2d7e8faa7a37ac595ee119f866e4465609b3e66cbc8af914f86e70ca172b63 +size 1853731312 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7607883c9ea71c7681ed9af9343f3f3e00e7a0a8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e450787b3101eaf06c8e85def11c649c0ce9ddd7433333b255d8d6b197da3768 +size 1853731120 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f88916ccf1ac1aaa2367a73d5754787163cc9855 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13b9295c484df79bf9721d3430d232ff7b6e1384316018492b27889cbddc13f5 +size 1853731440 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0c5a319a37838f88c74fbe62f84eaad26d5f28a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:383d7cd860f700523fe1f1d6f616e3922e87d6a24d5be1eb6e49a2a2cf7d9a26 +size 1853731056 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/mp_rank_00_model_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..510fe455692ff8ee3e6c7be1bc5d986932499668 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/global_step18000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df4151ea77b5eda68dfb7be097e5c66014d3a24cf0793e3c0c37f57ab0abce15 +size 2471676664 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/latest new file mode 100644 index 0000000000000000000000000000000000000000..b73f60d7a0f5e4a77d445820eafbd0d5e13129a2 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/latest @@ -0,0 +1 @@ +global_step18000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f88d2f46d73723cb1ba1a684cf5278c03903a67d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d05f88d039a0ff0a6f1a70d84710d441ad5fa86ab3b8c7e1819e22f42819bfb +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..13e52f546d1b5f0af0303398d18eba4a0a939882 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f26220b8e21fad33351b729ac877dbaecd0df7fcf236e96502b80b259813c6 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..ebe5b7c4eb8b7d7566fe55de79800ca17596536c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c47c3c97e95f63d9c046b3a606a9c420d3b96669534178b7f1bc9a8f6859322f +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..855484ae7c49a017419c6f000830331818ad7882 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a412365314e83f07ad8ebab9fbe8dfdd0bf79f6263165e8ef61f677b4fc0f6dc +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..536e257d343eb6be38dc68a9ef9ee286f34dc61f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11ba06e33b5230083f68f930c4f90fcbca206503923dbff7b9069919dc264d02 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..bf1cdad0d43cea05aba3652a37cceb77177e0e22 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dad4efd391dc3382a38c07ce95bc0f6edd3bd818a816c4608b1b56c9d77c372 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..a496b4ce2afc6352e2fb70196155eaf2663b8822 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f53b4a6bb820e95fb5c0b5eb98ba47a3dcffe831e3e542811a851abbf02f8bdf +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..d1cd68a04db2677eaa2b2dad7c89e29973d95daf --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0738da00de7894e1f772f96aa897a98f173326fe81bfd43dac83bada3040ea07 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..f49b76fb97aacd44c2197204eb57ab4d7a9fe9b0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fc59cde9135f5d4b8ed0c77b7665c601116e575054ff1f4a5f8c7ca869c7a02 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..21565de482b40f006d37963ebafcdfe26c25d297 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6811f4eff35f13019fe7a0f3e11af3e201050fd538642f614a8e73a804630d2 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..398810563620dcd4e1ef89e6603991ceb38a2c14 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/trainer_state.json @@ -0,0 +1,12634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 28.986748626996523, + "eval_steps": 2000, + "global_step": 18000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.016123343578374565, + "grad_norm": 48.11856460571289, + "learning_rate": 0.0001, + "loss": 299.1088, + "step": 10 + }, + { + "epoch": 0.03224668715674913, + "grad_norm": 9.07711410522461, + "learning_rate": 0.0001, + "loss": 234.0888, + "step": 20 + }, + { + "epoch": 0.0483700307351237, + "grad_norm": 3.5935330390930176, + "learning_rate": 0.0001, + "loss": 231.4619, + "step": 30 + }, + { + "epoch": 0.06449337431349826, + "grad_norm": 3.5784752368927, + "learning_rate": 0.0001, + "loss": 229.7082, + "step": 40 + }, + { + "epoch": 0.08061671789187283, + "grad_norm": 3.5757956504821777, + "learning_rate": 0.0001, + "loss": 230.1062, + "step": 50 + }, + { + "epoch": 0.0967400614702474, + "grad_norm": 3.8763978481292725, + "learning_rate": 0.0001, + "loss": 229.1914, + "step": 60 + }, + { + "epoch": 0.11286340504862195, + "grad_norm": 3.006988525390625, + "learning_rate": 0.0001, + "loss": 229.6482, + "step": 70 + }, + { + "epoch": 0.12898674862699652, + "grad_norm": 2.3527095317840576, + "learning_rate": 0.0001, + "loss": 228.1141, + "step": 80 + }, + { + "epoch": 0.14511009220537108, + "grad_norm": 5.353086471557617, + "learning_rate": 0.0001, + "loss": 228.1448, + "step": 90 + }, + { + "epoch": 0.16123343578374566, + "grad_norm": 2.251854419708252, + "learning_rate": 0.0001, + "loss": 228.3897, + "step": 100 + }, + { + "epoch": 0.17735677936212021, + "grad_norm": 1.5789915323257446, + "learning_rate": 0.0001, + "loss": 227.7662, + "step": 110 + }, + { + "epoch": 0.1934801229404948, + "grad_norm": 3.7646398544311523, + "learning_rate": 0.0001, + "loss": 223.6411, + "step": 120 + }, + { + "epoch": 0.20960346651886935, + "grad_norm": 3.444000720977783, + "learning_rate": 0.0001, + "loss": 221.2073, + "step": 130 + }, + { + "epoch": 0.2257268100972439, + "grad_norm": 2.1871185302734375, + "learning_rate": 0.0001, + "loss": 215.9944, + "step": 140 + }, + { + "epoch": 0.2418501536756185, + "grad_norm": 2.266244411468506, + "learning_rate": 0.0001, + "loss": 212.3066, + "step": 150 + }, + { + "epoch": 0.25797349725399304, + "grad_norm": 2.043832778930664, + "learning_rate": 0.0001, + "loss": 208.1343, + "step": 160 + }, + { + "epoch": 0.2740968408323676, + "grad_norm": 2.2641494274139404, + "learning_rate": 0.0001, + "loss": 203.1485, + "step": 170 + }, + { + "epoch": 0.29022018441074215, + "grad_norm": 2.9727790355682373, + "learning_rate": 0.0001, + "loss": 198.181, + "step": 180 + }, + { + "epoch": 0.30634352798911674, + "grad_norm": 2.751523494720459, + "learning_rate": 0.0001, + "loss": 194.8422, + "step": 190 + }, + { + "epoch": 0.3224668715674913, + "grad_norm": 1.9561562538146973, + "learning_rate": 0.0001, + "loss": 189.9401, + "step": 200 + }, + { + "epoch": 0.33859021514586585, + "grad_norm": 2.3264107704162598, + "learning_rate": 0.0001, + "loss": 186.1309, + "step": 210 + }, + { + "epoch": 0.35471355872424043, + "grad_norm": 5.212426662445068, + "learning_rate": 0.0001, + "loss": 182.2255, + "step": 220 + }, + { + "epoch": 0.370836902302615, + "grad_norm": 2.488032102584839, + "learning_rate": 0.0001, + "loss": 181.0103, + "step": 230 + }, + { + "epoch": 0.3869602458809896, + "grad_norm": 2.6173202991485596, + "learning_rate": 0.0001, + "loss": 177.6162, + "step": 240 + }, + { + "epoch": 0.4030835894593641, + "grad_norm": 2.6245291233062744, + "learning_rate": 0.0001, + "loss": 173.0431, + "step": 250 + }, + { + "epoch": 0.4192069330377387, + "grad_norm": 5.356006145477295, + "learning_rate": 0.0001, + "loss": 172.2371, + "step": 260 + }, + { + "epoch": 0.4353302766161133, + "grad_norm": 2.0453829765319824, + "learning_rate": 0.0001, + "loss": 169.7781, + "step": 270 + }, + { + "epoch": 0.4514536201944878, + "grad_norm": 1.7002843618392944, + "learning_rate": 0.0001, + "loss": 166.9564, + "step": 280 + }, + { + "epoch": 0.4675769637728624, + "grad_norm": 2.181410312652588, + "learning_rate": 0.0001, + "loss": 166.0094, + "step": 290 + }, + { + "epoch": 0.483700307351237, + "grad_norm": 2.2948265075683594, + "learning_rate": 0.0001, + "loss": 162.4388, + "step": 300 + }, + { + "epoch": 0.4998236509296115, + "grad_norm": 3.0272789001464844, + "learning_rate": 0.0001, + "loss": 160.3446, + "step": 310 + }, + { + "epoch": 0.5159469945079861, + "grad_norm": 1.948184847831726, + "learning_rate": 0.0001, + "loss": 158.488, + "step": 320 + }, + { + "epoch": 0.5320703380863606, + "grad_norm": 1.7981538772583008, + "learning_rate": 0.0001, + "loss": 156.2107, + "step": 330 + }, + { + "epoch": 0.5481936816647353, + "grad_norm": 2.049323797225952, + "learning_rate": 0.0001, + "loss": 155.5434, + "step": 340 + }, + { + "epoch": 0.5643170252431098, + "grad_norm": 1.9415442943572998, + "learning_rate": 0.0001, + "loss": 153.8682, + "step": 350 + }, + { + "epoch": 0.5804403688214843, + "grad_norm": 2.2168045043945312, + "learning_rate": 0.0001, + "loss": 150.4087, + "step": 360 + }, + { + "epoch": 0.596563712399859, + "grad_norm": 1.510750651359558, + "learning_rate": 0.0001, + "loss": 151.7655, + "step": 370 + }, + { + "epoch": 0.6126870559782335, + "grad_norm": 2.7469751834869385, + "learning_rate": 0.0001, + "loss": 147.4665, + "step": 380 + }, + { + "epoch": 0.628810399556608, + "grad_norm": 2.0436975955963135, + "learning_rate": 0.0001, + "loss": 147.4405, + "step": 390 + }, + { + "epoch": 0.6449337431349826, + "grad_norm": 1.8219181299209595, + "learning_rate": 0.0001, + "loss": 146.6332, + "step": 400 + }, + { + "epoch": 0.6610570867133572, + "grad_norm": 1.948431134223938, + "learning_rate": 0.0001, + "loss": 143.3298, + "step": 410 + }, + { + "epoch": 0.6771804302917317, + "grad_norm": 1.8887908458709717, + "learning_rate": 0.0001, + "loss": 140.5058, + "step": 420 + }, + { + "epoch": 0.6933037738701063, + "grad_norm": 1.8739320039749146, + "learning_rate": 0.0001, + "loss": 137.6334, + "step": 430 + }, + { + "epoch": 0.7094271174484809, + "grad_norm": 2.057689666748047, + "learning_rate": 0.0001, + "loss": 135.0343, + "step": 440 + }, + { + "epoch": 0.7255504610268555, + "grad_norm": 1.6926478147506714, + "learning_rate": 0.0001, + "loss": 132.8766, + "step": 450 + }, + { + "epoch": 0.74167380460523, + "grad_norm": 1.5039501190185547, + "learning_rate": 0.0001, + "loss": 131.6774, + "step": 460 + }, + { + "epoch": 0.7577971481836046, + "grad_norm": 1.6903547048568726, + "learning_rate": 0.0001, + "loss": 129.3116, + "step": 470 + }, + { + "epoch": 0.7739204917619792, + "grad_norm": 3.7062244415283203, + "learning_rate": 0.0001, + "loss": 127.9314, + "step": 480 + }, + { + "epoch": 0.7900438353403537, + "grad_norm": 2.0586934089660645, + "learning_rate": 0.0001, + "loss": 126.6398, + "step": 490 + }, + { + "epoch": 0.8061671789187282, + "grad_norm": 1.7598215341567993, + "learning_rate": 0.0001, + "loss": 125.1501, + "step": 500 + }, + { + "epoch": 0.8222905224971029, + "grad_norm": 1.5992189645767212, + "learning_rate": 0.0001, + "loss": 124.7039, + "step": 510 + }, + { + "epoch": 0.8384138660754774, + "grad_norm": 2.8735458850860596, + "learning_rate": 0.0001, + "loss": 122.1619, + "step": 520 + }, + { + "epoch": 0.8545372096538519, + "grad_norm": 1.867098093032837, + "learning_rate": 0.0001, + "loss": 122.4506, + "step": 530 + }, + { + "epoch": 0.8706605532322266, + "grad_norm": 2.2088418006896973, + "learning_rate": 0.0001, + "loss": 122.1047, + "step": 540 + }, + { + "epoch": 0.8867838968106011, + "grad_norm": 1.642657995223999, + "learning_rate": 0.0001, + "loss": 120.1141, + "step": 550 + }, + { + "epoch": 0.9029072403889756, + "grad_norm": 1.5567114353179932, + "learning_rate": 0.0001, + "loss": 118.532, + "step": 560 + }, + { + "epoch": 0.9190305839673503, + "grad_norm": 1.667843222618103, + "learning_rate": 0.0001, + "loss": 118.0734, + "step": 570 + }, + { + "epoch": 0.9351539275457248, + "grad_norm": 1.5304442644119263, + "learning_rate": 0.0001, + "loss": 114.4925, + "step": 580 + }, + { + "epoch": 0.9512772711240993, + "grad_norm": 1.6920255422592163, + "learning_rate": 0.0001, + "loss": 114.0729, + "step": 590 + }, + { + "epoch": 0.967400614702474, + "grad_norm": 1.6313318014144897, + "learning_rate": 0.0001, + "loss": 114.2879, + "step": 600 + }, + { + "epoch": 0.9835239582808485, + "grad_norm": 1.9033657312393188, + "learning_rate": 0.0001, + "loss": 110.5299, + "step": 610 + }, + { + "epoch": 0.999647301859223, + "grad_norm": 1.9010754823684692, + "learning_rate": 0.0001, + "loss": 109.1145, + "step": 620 + }, + { + "epoch": 1.0145110092205372, + "grad_norm": 3.5390989780426025, + "learning_rate": 0.0001, + "loss": 101.0258, + "step": 630 + }, + { + "epoch": 1.0306343527989117, + "grad_norm": 2.2925171852111816, + "learning_rate": 0.0001, + "loss": 107.3693, + "step": 640 + }, + { + "epoch": 1.0467576963772862, + "grad_norm": 2.329108238220215, + "learning_rate": 0.0001, + "loss": 104.3778, + "step": 650 + }, + { + "epoch": 1.0628810399556607, + "grad_norm": 1.7325661182403564, + "learning_rate": 0.0001, + "loss": 102.8409, + "step": 660 + }, + { + "epoch": 1.0790043835340353, + "grad_norm": 1.989586353302002, + "learning_rate": 0.0001, + "loss": 104.2577, + "step": 670 + }, + { + "epoch": 1.09512772711241, + "grad_norm": 1.900484323501587, + "learning_rate": 0.0001, + "loss": 102.3051, + "step": 680 + }, + { + "epoch": 1.1112510706907845, + "grad_norm": 2.015859842300415, + "learning_rate": 0.0001, + "loss": 100.893, + "step": 690 + }, + { + "epoch": 1.127374414269159, + "grad_norm": 1.7695508003234863, + "learning_rate": 0.0001, + "loss": 99.215, + "step": 700 + }, + { + "epoch": 1.1434977578475336, + "grad_norm": 2.025923490524292, + "learning_rate": 0.0001, + "loss": 98.8768, + "step": 710 + }, + { + "epoch": 1.1596211014259081, + "grad_norm": 2.946708917617798, + "learning_rate": 0.0001, + "loss": 96.5588, + "step": 720 + }, + { + "epoch": 1.1757444450042827, + "grad_norm": 2.4511258602142334, + "learning_rate": 0.0001, + "loss": 95.0545, + "step": 730 + }, + { + "epoch": 1.1918677885826574, + "grad_norm": 2.3094682693481445, + "learning_rate": 0.0001, + "loss": 97.004, + "step": 740 + }, + { + "epoch": 1.207991132161032, + "grad_norm": 2.1313347816467285, + "learning_rate": 0.0001, + "loss": 94.8819, + "step": 750 + }, + { + "epoch": 1.2241144757394065, + "grad_norm": 1.614182710647583, + "learning_rate": 0.0001, + "loss": 95.5339, + "step": 760 + }, + { + "epoch": 1.240237819317781, + "grad_norm": 2.029953956604004, + "learning_rate": 0.0001, + "loss": 93.7303, + "step": 770 + }, + { + "epoch": 1.2563611628961555, + "grad_norm": 1.7871663570404053, + "learning_rate": 0.0001, + "loss": 93.8354, + "step": 780 + }, + { + "epoch": 1.2724845064745303, + "grad_norm": 1.72455632686615, + "learning_rate": 0.0001, + "loss": 94.7348, + "step": 790 + }, + { + "epoch": 1.2886078500529048, + "grad_norm": 1.9776639938354492, + "learning_rate": 0.0001, + "loss": 93.5957, + "step": 800 + }, + { + "epoch": 1.3047311936312793, + "grad_norm": 1.7187533378601074, + "learning_rate": 0.0001, + "loss": 92.9583, + "step": 810 + }, + { + "epoch": 1.3208545372096538, + "grad_norm": 1.5949162244796753, + "learning_rate": 0.0001, + "loss": 93.3088, + "step": 820 + }, + { + "epoch": 1.3369778807880284, + "grad_norm": 1.4616206884384155, + "learning_rate": 0.0001, + "loss": 93.2125, + "step": 830 + }, + { + "epoch": 1.353101224366403, + "grad_norm": 2.832163095474243, + "learning_rate": 0.0001, + "loss": 90.3301, + "step": 840 + }, + { + "epoch": 1.3692245679447774, + "grad_norm": 2.174501419067383, + "learning_rate": 0.0001, + "loss": 91.6988, + "step": 850 + }, + { + "epoch": 1.3853479115231522, + "grad_norm": 1.2156339883804321, + "learning_rate": 0.0001, + "loss": 92.7419, + "step": 860 + }, + { + "epoch": 1.4014712551015267, + "grad_norm": 1.608261227607727, + "learning_rate": 0.0001, + "loss": 89.8255, + "step": 870 + }, + { + "epoch": 1.4175945986799012, + "grad_norm": 1.6671923398971558, + "learning_rate": 0.0001, + "loss": 90.9683, + "step": 880 + }, + { + "epoch": 1.4337179422582758, + "grad_norm": 1.2346433401107788, + "learning_rate": 0.0001, + "loss": 90.6092, + "step": 890 + }, + { + "epoch": 1.4498412858366505, + "grad_norm": 1.561085820198059, + "learning_rate": 0.0001, + "loss": 88.0842, + "step": 900 + }, + { + "epoch": 1.465964629415025, + "grad_norm": 1.5475201606750488, + "learning_rate": 0.0001, + "loss": 89.8786, + "step": 910 + }, + { + "epoch": 1.4820879729933996, + "grad_norm": 1.6044800281524658, + "learning_rate": 0.0001, + "loss": 88.0364, + "step": 920 + }, + { + "epoch": 1.498211316571774, + "grad_norm": 1.4081908464431763, + "learning_rate": 0.0001, + "loss": 88.8799, + "step": 930 + }, + { + "epoch": 1.5143346601501486, + "grad_norm": 2.341053009033203, + "learning_rate": 0.0001, + "loss": 88.2023, + "step": 940 + }, + { + "epoch": 1.5304580037285231, + "grad_norm": 1.3976799249649048, + "learning_rate": 0.0001, + "loss": 86.6493, + "step": 950 + }, + { + "epoch": 1.5465813473068977, + "grad_norm": 1.2485483884811401, + "learning_rate": 0.0001, + "loss": 88.9224, + "step": 960 + }, + { + "epoch": 1.5627046908852722, + "grad_norm": 1.6824235916137695, + "learning_rate": 0.0001, + "loss": 89.6484, + "step": 970 + }, + { + "epoch": 1.578828034463647, + "grad_norm": 1.64790940284729, + "learning_rate": 0.0001, + "loss": 87.2479, + "step": 980 + }, + { + "epoch": 1.5949513780420215, + "grad_norm": 1.2093443870544434, + "learning_rate": 0.0001, + "loss": 86.8241, + "step": 990 + }, + { + "epoch": 1.611074721620396, + "grad_norm": 1.5306812524795532, + "learning_rate": 0.0001, + "loss": 88.0006, + "step": 1000 + }, + { + "epoch": 1.6271980651987707, + "grad_norm": 1.2465763092041016, + "learning_rate": 0.0001, + "loss": 86.8143, + "step": 1010 + }, + { + "epoch": 1.6433214087771453, + "grad_norm": 1.5224069356918335, + "learning_rate": 0.0001, + "loss": 87.0102, + "step": 1020 + }, + { + "epoch": 1.6594447523555198, + "grad_norm": 1.312179684638977, + "learning_rate": 0.0001, + "loss": 88.7395, + "step": 1030 + }, + { + "epoch": 1.6755680959338943, + "grad_norm": 1.2343158721923828, + "learning_rate": 0.0001, + "loss": 87.2868, + "step": 1040 + }, + { + "epoch": 1.6916914395122689, + "grad_norm": 1.2778265476226807, + "learning_rate": 0.0001, + "loss": 87.043, + "step": 1050 + }, + { + "epoch": 1.7078147830906434, + "grad_norm": 1.2799627780914307, + "learning_rate": 0.0001, + "loss": 87.7843, + "step": 1060 + }, + { + "epoch": 1.723938126669018, + "grad_norm": 1.3688952922821045, + "learning_rate": 0.0001, + "loss": 85.8985, + "step": 1070 + }, + { + "epoch": 1.7400614702473924, + "grad_norm": 1.4317796230316162, + "learning_rate": 0.0001, + "loss": 85.5408, + "step": 1080 + }, + { + "epoch": 1.756184813825767, + "grad_norm": 1.164801001548767, + "learning_rate": 0.0001, + "loss": 85.8484, + "step": 1090 + }, + { + "epoch": 1.7723081574041417, + "grad_norm": 1.3929173946380615, + "learning_rate": 0.0001, + "loss": 86.9031, + "step": 1100 + }, + { + "epoch": 1.7884315009825162, + "grad_norm": 2.116119861602783, + "learning_rate": 0.0001, + "loss": 86.5343, + "step": 1110 + }, + { + "epoch": 1.8045548445608908, + "grad_norm": 1.4117276668548584, + "learning_rate": 0.0001, + "loss": 85.2678, + "step": 1120 + }, + { + "epoch": 1.8206781881392655, + "grad_norm": 1.3213504552841187, + "learning_rate": 0.0001, + "loss": 85.8935, + "step": 1130 + }, + { + "epoch": 1.83680153171764, + "grad_norm": 1.3589770793914795, + "learning_rate": 0.0001, + "loss": 85.032, + "step": 1140 + }, + { + "epoch": 1.8529248752960146, + "grad_norm": 1.3336007595062256, + "learning_rate": 0.0001, + "loss": 84.0993, + "step": 1150 + }, + { + "epoch": 1.869048218874389, + "grad_norm": 1.1081451177597046, + "learning_rate": 0.0001, + "loss": 84.2225, + "step": 1160 + }, + { + "epoch": 1.8851715624527636, + "grad_norm": 1.482038974761963, + "learning_rate": 0.0001, + "loss": 82.1439, + "step": 1170 + }, + { + "epoch": 1.9012949060311382, + "grad_norm": 1.1571478843688965, + "learning_rate": 0.0001, + "loss": 82.5045, + "step": 1180 + }, + { + "epoch": 1.9174182496095127, + "grad_norm": 1.2665468454360962, + "learning_rate": 0.0001, + "loss": 82.8761, + "step": 1190 + }, + { + "epoch": 1.9335415931878872, + "grad_norm": 1.0771509408950806, + "learning_rate": 0.0001, + "loss": 81.5162, + "step": 1200 + }, + { + "epoch": 1.949664936766262, + "grad_norm": 1.2992299795150757, + "learning_rate": 0.0001, + "loss": 82.4038, + "step": 1210 + }, + { + "epoch": 1.9657882803446365, + "grad_norm": 1.2946668863296509, + "learning_rate": 0.0001, + "loss": 83.4837, + "step": 1220 + }, + { + "epoch": 1.981911623923011, + "grad_norm": 1.3505686521530151, + "learning_rate": 0.0001, + "loss": 81.8993, + "step": 1230 + }, + { + "epoch": 1.9980349675013858, + "grad_norm": 1.1247715950012207, + "learning_rate": 0.0001, + "loss": 82.9006, + "step": 1240 + }, + { + "epoch": 2.0128986748627, + "grad_norm": 1.5652127265930176, + "learning_rate": 0.0001, + "loss": 75.8347, + "step": 1250 + }, + { + "epoch": 2.0290220184410743, + "grad_norm": 7.311964988708496, + "learning_rate": 0.0001, + "loss": 80.5435, + "step": 1260 + }, + { + "epoch": 2.045145362019449, + "grad_norm": 1.2229893207550049, + "learning_rate": 0.0001, + "loss": 83.3277, + "step": 1270 + }, + { + "epoch": 2.0612687055978234, + "grad_norm": 1.1387693881988525, + "learning_rate": 0.0001, + "loss": 81.3252, + "step": 1280 + }, + { + "epoch": 2.077392049176198, + "grad_norm": 1.307201623916626, + "learning_rate": 0.0001, + "loss": 82.3311, + "step": 1290 + }, + { + "epoch": 2.0935153927545724, + "grad_norm": 1.1502896547317505, + "learning_rate": 0.0001, + "loss": 81.1682, + "step": 1300 + }, + { + "epoch": 2.109638736332947, + "grad_norm": 0.9291583299636841, + "learning_rate": 0.0001, + "loss": 81.7017, + "step": 1310 + }, + { + "epoch": 2.1257620799113215, + "grad_norm": 1.0334666967391968, + "learning_rate": 0.0001, + "loss": 81.718, + "step": 1320 + }, + { + "epoch": 2.141885423489696, + "grad_norm": 1.1099056005477905, + "learning_rate": 0.0001, + "loss": 81.2232, + "step": 1330 + }, + { + "epoch": 2.1580087670680705, + "grad_norm": 1.289478063583374, + "learning_rate": 0.0001, + "loss": 81.8736, + "step": 1340 + }, + { + "epoch": 2.174132110646445, + "grad_norm": 1.1634794473648071, + "learning_rate": 0.0001, + "loss": 79.9434, + "step": 1350 + }, + { + "epoch": 2.19025545422482, + "grad_norm": 0.9563440680503845, + "learning_rate": 0.0001, + "loss": 80.628, + "step": 1360 + }, + { + "epoch": 2.2063787978031946, + "grad_norm": 1.1268844604492188, + "learning_rate": 0.0001, + "loss": 82.1389, + "step": 1370 + }, + { + "epoch": 2.222502141381569, + "grad_norm": 1.1050443649291992, + "learning_rate": 0.0001, + "loss": 78.9023, + "step": 1380 + }, + { + "epoch": 2.2386254849599436, + "grad_norm": 1.1936142444610596, + "learning_rate": 0.0001, + "loss": 81.2118, + "step": 1390 + }, + { + "epoch": 2.254748828538318, + "grad_norm": 1.7825465202331543, + "learning_rate": 0.0001, + "loss": 80.8804, + "step": 1400 + }, + { + "epoch": 2.2708721721166927, + "grad_norm": 2.437251329421997, + "learning_rate": 0.0001, + "loss": 79.3807, + "step": 1410 + }, + { + "epoch": 2.286995515695067, + "grad_norm": 1.0604854822158813, + "learning_rate": 0.0001, + "loss": 79.5881, + "step": 1420 + }, + { + "epoch": 2.3031188592734417, + "grad_norm": 1.2285529375076294, + "learning_rate": 0.0001, + "loss": 79.5517, + "step": 1430 + }, + { + "epoch": 2.3192422028518163, + "grad_norm": 1.1916660070419312, + "learning_rate": 0.0001, + "loss": 80.5887, + "step": 1440 + }, + { + "epoch": 2.335365546430191, + "grad_norm": 1.224304437637329, + "learning_rate": 0.0001, + "loss": 79.8109, + "step": 1450 + }, + { + "epoch": 2.3514888900085653, + "grad_norm": 0.9228448867797852, + "learning_rate": 0.0001, + "loss": 79.2787, + "step": 1460 + }, + { + "epoch": 2.3676122335869403, + "grad_norm": 1.2085226774215698, + "learning_rate": 0.0001, + "loss": 78.4983, + "step": 1470 + }, + { + "epoch": 2.383735577165315, + "grad_norm": 2.424083948135376, + "learning_rate": 0.0001, + "loss": 79.8657, + "step": 1480 + }, + { + "epoch": 2.3998589207436893, + "grad_norm": 1.0277849435806274, + "learning_rate": 0.0001, + "loss": 80.6915, + "step": 1490 + }, + { + "epoch": 2.415982264322064, + "grad_norm": 1.0226075649261475, + "learning_rate": 0.0001, + "loss": 78.9751, + "step": 1500 + }, + { + "epoch": 2.4321056079004384, + "grad_norm": 0.8860388398170471, + "learning_rate": 0.0001, + "loss": 78.3915, + "step": 1510 + }, + { + "epoch": 2.448228951478813, + "grad_norm": 1.265637755393982, + "learning_rate": 0.0001, + "loss": 78.8737, + "step": 1520 + }, + { + "epoch": 2.4643522950571874, + "grad_norm": 1.2518186569213867, + "learning_rate": 0.0001, + "loss": 76.5169, + "step": 1530 + }, + { + "epoch": 2.480475638635562, + "grad_norm": 1.074277400970459, + "learning_rate": 0.0001, + "loss": 77.9156, + "step": 1540 + }, + { + "epoch": 2.4965989822139365, + "grad_norm": 1.0666711330413818, + "learning_rate": 0.0001, + "loss": 78.4103, + "step": 1550 + }, + { + "epoch": 2.512722325792311, + "grad_norm": 0.9492248892784119, + "learning_rate": 0.0001, + "loss": 76.7393, + "step": 1560 + }, + { + "epoch": 2.5288456693706856, + "grad_norm": 1.127238154411316, + "learning_rate": 0.0001, + "loss": 79.729, + "step": 1570 + }, + { + "epoch": 2.5449690129490605, + "grad_norm": 1.1520992517471313, + "learning_rate": 0.0001, + "loss": 78.5618, + "step": 1580 + }, + { + "epoch": 2.5610923565274346, + "grad_norm": 0.8632035255432129, + "learning_rate": 0.0001, + "loss": 79.032, + "step": 1590 + }, + { + "epoch": 2.5772157001058096, + "grad_norm": 1.0506329536437988, + "learning_rate": 0.0001, + "loss": 77.6204, + "step": 1600 + }, + { + "epoch": 2.593339043684184, + "grad_norm": 1.019895315170288, + "learning_rate": 0.0001, + "loss": 79.2634, + "step": 1610 + }, + { + "epoch": 2.6094623872625586, + "grad_norm": 1.1685571670532227, + "learning_rate": 0.0001, + "loss": 78.8371, + "step": 1620 + }, + { + "epoch": 2.625585730840933, + "grad_norm": 1.147546648979187, + "learning_rate": 0.0001, + "loss": 78.2113, + "step": 1630 + }, + { + "epoch": 2.6417090744193077, + "grad_norm": 1.7336841821670532, + "learning_rate": 0.0001, + "loss": 78.7866, + "step": 1640 + }, + { + "epoch": 2.657832417997682, + "grad_norm": 1.2150614261627197, + "learning_rate": 0.0001, + "loss": 78.5621, + "step": 1650 + }, + { + "epoch": 2.6739557615760567, + "grad_norm": 1.2257503271102905, + "learning_rate": 0.0001, + "loss": 77.5835, + "step": 1660 + }, + { + "epoch": 2.6900791051544313, + "grad_norm": 1.0375311374664307, + "learning_rate": 0.0001, + "loss": 77.7509, + "step": 1670 + }, + { + "epoch": 2.706202448732806, + "grad_norm": 1.2551004886627197, + "learning_rate": 0.0001, + "loss": 77.8615, + "step": 1680 + }, + { + "epoch": 2.7223257923111808, + "grad_norm": 1.0464223623275757, + "learning_rate": 0.0001, + "loss": 77.2837, + "step": 1690 + }, + { + "epoch": 2.738449135889555, + "grad_norm": 1.0254806280136108, + "learning_rate": 0.0001, + "loss": 76.9616, + "step": 1700 + }, + { + "epoch": 2.75457247946793, + "grad_norm": 1.1665682792663574, + "learning_rate": 0.0001, + "loss": 77.9108, + "step": 1710 + }, + { + "epoch": 2.7706958230463044, + "grad_norm": 1.1349453926086426, + "learning_rate": 0.0001, + "loss": 78.7636, + "step": 1720 + }, + { + "epoch": 2.786819166624679, + "grad_norm": 1.4961057901382446, + "learning_rate": 0.0001, + "loss": 76.3862, + "step": 1730 + }, + { + "epoch": 2.8029425102030534, + "grad_norm": 1.10721755027771, + "learning_rate": 0.0001, + "loss": 77.4105, + "step": 1740 + }, + { + "epoch": 2.819065853781428, + "grad_norm": 0.9878576397895813, + "learning_rate": 0.0001, + "loss": 78.4855, + "step": 1750 + }, + { + "epoch": 2.8351891973598025, + "grad_norm": 1.1722687482833862, + "learning_rate": 0.0001, + "loss": 75.1589, + "step": 1760 + }, + { + "epoch": 2.851312540938177, + "grad_norm": 1.6624362468719482, + "learning_rate": 0.0001, + "loss": 77.1647, + "step": 1770 + }, + { + "epoch": 2.8674358845165515, + "grad_norm": 0.9542295932769775, + "learning_rate": 0.0001, + "loss": 77.1656, + "step": 1780 + }, + { + "epoch": 2.883559228094926, + "grad_norm": 2.328542947769165, + "learning_rate": 0.0001, + "loss": 76.4686, + "step": 1790 + }, + { + "epoch": 2.899682571673301, + "grad_norm": 1.2532029151916504, + "learning_rate": 0.0001, + "loss": 76.1727, + "step": 1800 + }, + { + "epoch": 2.915805915251675, + "grad_norm": 0.9604716897010803, + "learning_rate": 0.0001, + "loss": 74.3132, + "step": 1810 + }, + { + "epoch": 2.93192925883005, + "grad_norm": 0.9182595610618591, + "learning_rate": 0.0001, + "loss": 77.8689, + "step": 1820 + }, + { + "epoch": 2.9480526024084246, + "grad_norm": 0.9623042941093445, + "learning_rate": 0.0001, + "loss": 77.7531, + "step": 1830 + }, + { + "epoch": 2.964175945986799, + "grad_norm": 1.0500119924545288, + "learning_rate": 0.0001, + "loss": 77.6007, + "step": 1840 + }, + { + "epoch": 2.9802992895651736, + "grad_norm": 1.0874103307724, + "learning_rate": 0.0001, + "loss": 77.042, + "step": 1850 + }, + { + "epoch": 2.996422633143548, + "grad_norm": 0.9955475330352783, + "learning_rate": 0.0001, + "loss": 75.468, + "step": 1860 + }, + { + "epoch": 3.011286340504862, + "grad_norm": 1.4281760454177856, + "learning_rate": 0.0001, + "loss": 69.5972, + "step": 1870 + }, + { + "epoch": 3.0274096840832367, + "grad_norm": 1.0292123556137085, + "learning_rate": 0.0001, + "loss": 76.0186, + "step": 1880 + }, + { + "epoch": 3.0435330276616113, + "grad_norm": 0.9535816311836243, + "learning_rate": 0.0001, + "loss": 76.0979, + "step": 1890 + }, + { + "epoch": 3.059656371239986, + "grad_norm": 0.9099674820899963, + "learning_rate": 0.0001, + "loss": 75.9641, + "step": 1900 + }, + { + "epoch": 3.0757797148183603, + "grad_norm": 0.9568380117416382, + "learning_rate": 0.0001, + "loss": 75.0656, + "step": 1910 + }, + { + "epoch": 3.091903058396735, + "grad_norm": 0.9701060652732849, + "learning_rate": 0.0001, + "loss": 75.1422, + "step": 1920 + }, + { + "epoch": 3.1080264019751094, + "grad_norm": 1.3879098892211914, + "learning_rate": 0.0001, + "loss": 76.0731, + "step": 1930 + }, + { + "epoch": 3.1241497455534843, + "grad_norm": 0.9782084226608276, + "learning_rate": 0.0001, + "loss": 75.9666, + "step": 1940 + }, + { + "epoch": 3.140273089131859, + "grad_norm": 1.0856964588165283, + "learning_rate": 0.0001, + "loss": 75.0217, + "step": 1950 + }, + { + "epoch": 3.1563964327102334, + "grad_norm": 0.8506444692611694, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 1960 + }, + { + "epoch": 3.172519776288608, + "grad_norm": 1.044980525970459, + "learning_rate": 0.0001, + "loss": 75.3471, + "step": 1970 + }, + { + "epoch": 3.1886431198669825, + "grad_norm": 0.8432291746139526, + "learning_rate": 0.0001, + "loss": 75.3563, + "step": 1980 + }, + { + "epoch": 3.204766463445357, + "grad_norm": 0.9651834964752197, + "learning_rate": 0.0001, + "loss": 76.3991, + "step": 1990 + }, + { + "epoch": 3.2208898070237315, + "grad_norm": 0.957943320274353, + "learning_rate": 0.0001, + "loss": 77.2613, + "step": 2000 + }, + { + "epoch": 3.237013150602106, + "grad_norm": 0.9495745301246643, + "learning_rate": 0.0001, + "loss": 73.3801, + "step": 2010 + }, + { + "epoch": 3.2531364941804806, + "grad_norm": 1.182983160018921, + "learning_rate": 0.0001, + "loss": 74.5695, + "step": 2020 + }, + { + "epoch": 3.269259837758855, + "grad_norm": 0.828452467918396, + "learning_rate": 0.0001, + "loss": 75.4313, + "step": 2030 + }, + { + "epoch": 3.2853831813372296, + "grad_norm": 0.8340104818344116, + "learning_rate": 0.0001, + "loss": 75.1086, + "step": 2040 + }, + { + "epoch": 3.301506524915604, + "grad_norm": 1.1700762510299683, + "learning_rate": 0.0001, + "loss": 75.7496, + "step": 2050 + }, + { + "epoch": 3.317629868493979, + "grad_norm": 0.8709110617637634, + "learning_rate": 0.0001, + "loss": 73.8403, + "step": 2060 + }, + { + "epoch": 3.3337532120723536, + "grad_norm": 0.8928954601287842, + "learning_rate": 0.0001, + "loss": 75.6134, + "step": 2070 + }, + { + "epoch": 3.349876555650728, + "grad_norm": 0.9787353277206421, + "learning_rate": 0.0001, + "loss": 75.8116, + "step": 2080 + }, + { + "epoch": 3.3659998992291027, + "grad_norm": 0.9515123963356018, + "learning_rate": 0.0001, + "loss": 75.5024, + "step": 2090 + }, + { + "epoch": 3.3821232428074772, + "grad_norm": 0.9529262185096741, + "learning_rate": 0.0001, + "loss": 74.1553, + "step": 2100 + }, + { + "epoch": 3.3982465863858518, + "grad_norm": 0.8934710621833801, + "learning_rate": 0.0001, + "loss": 74.1501, + "step": 2110 + }, + { + "epoch": 3.4143699299642263, + "grad_norm": 1.089447021484375, + "learning_rate": 0.0001, + "loss": 74.9123, + "step": 2120 + }, + { + "epoch": 3.430493273542601, + "grad_norm": 1.2569622993469238, + "learning_rate": 0.0001, + "loss": 74.3962, + "step": 2130 + }, + { + "epoch": 3.4466166171209753, + "grad_norm": 1.020094394683838, + "learning_rate": 0.0001, + "loss": 74.6617, + "step": 2140 + }, + { + "epoch": 3.46273996069935, + "grad_norm": 1.1579667329788208, + "learning_rate": 0.0001, + "loss": 74.2745, + "step": 2150 + }, + { + "epoch": 3.4788633042777244, + "grad_norm": 1.448935627937317, + "learning_rate": 0.0001, + "loss": 71.5285, + "step": 2160 + }, + { + "epoch": 3.4949866478560994, + "grad_norm": 1.0030494928359985, + "learning_rate": 0.0001, + "loss": 74.6666, + "step": 2170 + }, + { + "epoch": 3.511109991434474, + "grad_norm": 0.9010888934135437, + "learning_rate": 0.0001, + "loss": 74.9355, + "step": 2180 + }, + { + "epoch": 3.5272333350128484, + "grad_norm": 1.1595124006271362, + "learning_rate": 0.0001, + "loss": 74.5049, + "step": 2190 + }, + { + "epoch": 3.543356678591223, + "grad_norm": 1.066405177116394, + "learning_rate": 0.0001, + "loss": 74.6336, + "step": 2200 + }, + { + "epoch": 3.5594800221695975, + "grad_norm": 1.2301596403121948, + "learning_rate": 0.0001, + "loss": 74.6664, + "step": 2210 + }, + { + "epoch": 3.575603365747972, + "grad_norm": 1.0073511600494385, + "learning_rate": 0.0001, + "loss": 73.1405, + "step": 2220 + }, + { + "epoch": 3.5917267093263465, + "grad_norm": 0.9913995862007141, + "learning_rate": 0.0001, + "loss": 75.7443, + "step": 2230 + }, + { + "epoch": 3.607850052904721, + "grad_norm": 0.9555648565292358, + "learning_rate": 0.0001, + "loss": 74.8036, + "step": 2240 + }, + { + "epoch": 3.6239733964830956, + "grad_norm": 0.9350649118423462, + "learning_rate": 0.0001, + "loss": 73.5783, + "step": 2250 + }, + { + "epoch": 3.64009674006147, + "grad_norm": 0.935840368270874, + "learning_rate": 0.0001, + "loss": 75.1694, + "step": 2260 + }, + { + "epoch": 3.6562200836398446, + "grad_norm": 0.9303803443908691, + "learning_rate": 0.0001, + "loss": 72.8282, + "step": 2270 + }, + { + "epoch": 3.6723434272182196, + "grad_norm": 0.937164843082428, + "learning_rate": 0.0001, + "loss": 70.3305, + "step": 2280 + }, + { + "epoch": 3.6884667707965937, + "grad_norm": 1.027329444885254, + "learning_rate": 0.0001, + "loss": 73.748, + "step": 2290 + }, + { + "epoch": 3.7045901143749687, + "grad_norm": 1.0915215015411377, + "learning_rate": 0.0001, + "loss": 72.2265, + "step": 2300 + }, + { + "epoch": 3.720713457953343, + "grad_norm": 1.1031261682510376, + "learning_rate": 0.0001, + "loss": 71.2795, + "step": 2310 + }, + { + "epoch": 3.7368368015317177, + "grad_norm": 1.1466140747070312, + "learning_rate": 0.0001, + "loss": 72.0116, + "step": 2320 + }, + { + "epoch": 3.7529601451100922, + "grad_norm": 1.0441662073135376, + "learning_rate": 0.0001, + "loss": 73.3735, + "step": 2330 + }, + { + "epoch": 3.7690834886884668, + "grad_norm": 0.8610432744026184, + "learning_rate": 0.0001, + "loss": 73.3433, + "step": 2340 + }, + { + "epoch": 3.7852068322668413, + "grad_norm": 0.885720431804657, + "learning_rate": 0.0001, + "loss": 73.575, + "step": 2350 + }, + { + "epoch": 3.801330175845216, + "grad_norm": 1.0023950338363647, + "learning_rate": 0.0001, + "loss": 73.4166, + "step": 2360 + }, + { + "epoch": 3.8174535194235903, + "grad_norm": 1.3820304870605469, + "learning_rate": 0.0001, + "loss": 72.9599, + "step": 2370 + }, + { + "epoch": 3.833576863001965, + "grad_norm": 0.9645710587501526, + "learning_rate": 0.0001, + "loss": 72.3922, + "step": 2380 + }, + { + "epoch": 3.84970020658034, + "grad_norm": 1.273369312286377, + "learning_rate": 0.0001, + "loss": 72.0158, + "step": 2390 + }, + { + "epoch": 3.865823550158714, + "grad_norm": 0.859863817691803, + "learning_rate": 0.0001, + "loss": 74.6642, + "step": 2400 + }, + { + "epoch": 3.881946893737089, + "grad_norm": 0.9396682381629944, + "learning_rate": 0.0001, + "loss": 71.9003, + "step": 2410 + }, + { + "epoch": 3.8980702373154634, + "grad_norm": 0.9976151585578918, + "learning_rate": 0.0001, + "loss": 73.0716, + "step": 2420 + }, + { + "epoch": 3.914193580893838, + "grad_norm": 1.0730767250061035, + "learning_rate": 0.0001, + "loss": 72.7847, + "step": 2430 + }, + { + "epoch": 3.9303169244722125, + "grad_norm": 1.002018690109253, + "learning_rate": 0.0001, + "loss": 71.7309, + "step": 2440 + }, + { + "epoch": 3.946440268050587, + "grad_norm": 0.9102753400802612, + "learning_rate": 0.0001, + "loss": 71.616, + "step": 2450 + }, + { + "epoch": 3.9625636116289615, + "grad_norm": 0.9827467799186707, + "learning_rate": 0.0001, + "loss": 72.22, + "step": 2460 + }, + { + "epoch": 3.978686955207336, + "grad_norm": 1.2781893014907837, + "learning_rate": 0.0001, + "loss": 73.2048, + "step": 2470 + }, + { + "epoch": 3.9948102987857106, + "grad_norm": 0.9042410850524902, + "learning_rate": 0.0001, + "loss": 72.9843, + "step": 2480 + }, + { + "epoch": 4.009674006147025, + "grad_norm": 0.8662117123603821, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 2490 + }, + { + "epoch": 4.0257973497254, + "grad_norm": 0.9448756575584412, + "learning_rate": 0.0001, + "loss": 73.36, + "step": 2500 + }, + { + "epoch": 4.041920693303774, + "grad_norm": 0.9054566621780396, + "learning_rate": 0.0001, + "loss": 73.94, + "step": 2510 + }, + { + "epoch": 4.058044036882149, + "grad_norm": 0.957398533821106, + "learning_rate": 0.0001, + "loss": 73.2019, + "step": 2520 + }, + { + "epoch": 4.074167380460523, + "grad_norm": 0.9444408416748047, + "learning_rate": 0.0001, + "loss": 70.6553, + "step": 2530 + }, + { + "epoch": 4.090290724038898, + "grad_norm": 0.8716063499450684, + "learning_rate": 0.0001, + "loss": 70.1048, + "step": 2540 + }, + { + "epoch": 4.106414067617272, + "grad_norm": 0.949249267578125, + "learning_rate": 0.0001, + "loss": 72.4453, + "step": 2550 + }, + { + "epoch": 4.122537411195647, + "grad_norm": 0.9843282699584961, + "learning_rate": 0.0001, + "loss": 72.1725, + "step": 2560 + }, + { + "epoch": 4.138660754774021, + "grad_norm": 0.8977480530738831, + "learning_rate": 0.0001, + "loss": 72.5272, + "step": 2570 + }, + { + "epoch": 4.154784098352396, + "grad_norm": 0.9957827925682068, + "learning_rate": 0.0001, + "loss": 72.289, + "step": 2580 + }, + { + "epoch": 4.170907441930771, + "grad_norm": 1.1358791589736938, + "learning_rate": 0.0001, + "loss": 72.1949, + "step": 2590 + }, + { + "epoch": 4.187030785509145, + "grad_norm": 0.9953081607818604, + "learning_rate": 0.0001, + "loss": 72.6004, + "step": 2600 + }, + { + "epoch": 4.20315412908752, + "grad_norm": 1.019061803817749, + "learning_rate": 0.0001, + "loss": 72.3772, + "step": 2610 + }, + { + "epoch": 4.219277472665894, + "grad_norm": 0.9062562584877014, + "learning_rate": 0.0001, + "loss": 72.0566, + "step": 2620 + }, + { + "epoch": 4.235400816244269, + "grad_norm": 1.0877611637115479, + "learning_rate": 0.0001, + "loss": 72.854, + "step": 2630 + }, + { + "epoch": 4.251524159822643, + "grad_norm": 1.2419648170471191, + "learning_rate": 0.0001, + "loss": 72.6498, + "step": 2640 + }, + { + "epoch": 4.267647503401018, + "grad_norm": 0.9507225751876831, + "learning_rate": 0.0001, + "loss": 68.9583, + "step": 2650 + }, + { + "epoch": 4.283770846979392, + "grad_norm": 1.121191382408142, + "learning_rate": 0.0001, + "loss": 72.8464, + "step": 2660 + }, + { + "epoch": 4.299894190557767, + "grad_norm": 1.0543361902236938, + "learning_rate": 0.0001, + "loss": 71.9882, + "step": 2670 + }, + { + "epoch": 4.316017534136141, + "grad_norm": 0.8622955083847046, + "learning_rate": 0.0001, + "loss": 71.7373, + "step": 2680 + }, + { + "epoch": 4.332140877714516, + "grad_norm": 0.911253809928894, + "learning_rate": 0.0001, + "loss": 71.1968, + "step": 2690 + }, + { + "epoch": 4.34826422129289, + "grad_norm": 4.405472755432129, + "learning_rate": 0.0001, + "loss": 70.4128, + "step": 2700 + }, + { + "epoch": 4.364387564871265, + "grad_norm": 0.9142985939979553, + "learning_rate": 0.0001, + "loss": 69.7548, + "step": 2710 + }, + { + "epoch": 4.38051090844964, + "grad_norm": 1.1022522449493408, + "learning_rate": 0.0001, + "loss": 71.8282, + "step": 2720 + }, + { + "epoch": 4.396634252028014, + "grad_norm": 1.018386721611023, + "learning_rate": 0.0001, + "loss": 71.0853, + "step": 2730 + }, + { + "epoch": 4.412757595606389, + "grad_norm": 1.0483211278915405, + "learning_rate": 0.0001, + "loss": 72.1478, + "step": 2740 + }, + { + "epoch": 4.428880939184763, + "grad_norm": 1.026861548423767, + "learning_rate": 0.0001, + "loss": 71.8947, + "step": 2750 + }, + { + "epoch": 4.445004282763138, + "grad_norm": 1.1858552694320679, + "learning_rate": 0.0001, + "loss": 71.7016, + "step": 2760 + }, + { + "epoch": 4.461127626341512, + "grad_norm": 1.1212221384048462, + "learning_rate": 0.0001, + "loss": 71.4454, + "step": 2770 + }, + { + "epoch": 4.477250969919887, + "grad_norm": 0.8062102794647217, + "learning_rate": 0.0001, + "loss": 73.2672, + "step": 2780 + }, + { + "epoch": 4.493374313498261, + "grad_norm": 1.104483962059021, + "learning_rate": 0.0001, + "loss": 70.548, + "step": 2790 + }, + { + "epoch": 4.509497657076636, + "grad_norm": 0.942057728767395, + "learning_rate": 0.0001, + "loss": 72.3584, + "step": 2800 + }, + { + "epoch": 4.525621000655011, + "grad_norm": 0.9028333425521851, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 2810 + }, + { + "epoch": 4.541744344233385, + "grad_norm": 0.9096795916557312, + "learning_rate": 0.0001, + "loss": 70.6596, + "step": 2820 + }, + { + "epoch": 4.55786768781176, + "grad_norm": 0.9038829207420349, + "learning_rate": 0.0001, + "loss": 72.6148, + "step": 2830 + }, + { + "epoch": 4.573991031390134, + "grad_norm": 1.8511613607406616, + "learning_rate": 0.0001, + "loss": 71.7205, + "step": 2840 + }, + { + "epoch": 4.590114374968509, + "grad_norm": 1.3492343425750732, + "learning_rate": 0.0001, + "loss": 72.1492, + "step": 2850 + }, + { + "epoch": 4.6062377185468835, + "grad_norm": 1.1469528675079346, + "learning_rate": 0.0001, + "loss": 70.5892, + "step": 2860 + }, + { + "epoch": 4.622361062125258, + "grad_norm": 1.1374379396438599, + "learning_rate": 0.0001, + "loss": 70.061, + "step": 2870 + }, + { + "epoch": 4.6384844057036325, + "grad_norm": 0.9597565531730652, + "learning_rate": 0.0001, + "loss": 71.0869, + "step": 2880 + }, + { + "epoch": 4.6546077492820075, + "grad_norm": 0.8356521129608154, + "learning_rate": 0.0001, + "loss": 72.1891, + "step": 2890 + }, + { + "epoch": 4.670731092860382, + "grad_norm": 0.8882647752761841, + "learning_rate": 0.0001, + "loss": 71.3845, + "step": 2900 + }, + { + "epoch": 4.6868544364387565, + "grad_norm": 1.1904003620147705, + "learning_rate": 0.0001, + "loss": 72.3086, + "step": 2910 + }, + { + "epoch": 4.702977780017131, + "grad_norm": 0.8010085821151733, + "learning_rate": 0.0001, + "loss": 71.3051, + "step": 2920 + }, + { + "epoch": 4.719101123595506, + "grad_norm": 0.7572135329246521, + "learning_rate": 0.0001, + "loss": 72.5605, + "step": 2930 + }, + { + "epoch": 4.735224467173881, + "grad_norm": 1.0866942405700684, + "learning_rate": 0.0001, + "loss": 70.0935, + "step": 2940 + }, + { + "epoch": 4.751347810752255, + "grad_norm": 0.9358721971511841, + "learning_rate": 0.0001, + "loss": 71.0391, + "step": 2950 + }, + { + "epoch": 4.76747115433063, + "grad_norm": 1.0276161432266235, + "learning_rate": 0.0001, + "loss": 69.8462, + "step": 2960 + }, + { + "epoch": 4.783594497909004, + "grad_norm": 0.9573063850402832, + "learning_rate": 0.0001, + "loss": 69.7089, + "step": 2970 + }, + { + "epoch": 4.799717841487379, + "grad_norm": 1.0638892650604248, + "learning_rate": 0.0001, + "loss": 71.8376, + "step": 2980 + }, + { + "epoch": 4.815841185065753, + "grad_norm": 0.9894681572914124, + "learning_rate": 0.0001, + "loss": 70.1764, + "step": 2990 + }, + { + "epoch": 4.831964528644128, + "grad_norm": 1.1265769004821777, + "learning_rate": 0.0001, + "loss": 71.7918, + "step": 3000 + }, + { + "epoch": 4.848087872222502, + "grad_norm": 2.2792513370513916, + "learning_rate": 0.0001, + "loss": 70.4341, + "step": 3010 + }, + { + "epoch": 4.864211215800877, + "grad_norm": 0.8469329476356506, + "learning_rate": 0.0001, + "loss": 72.1137, + "step": 3020 + }, + { + "epoch": 4.880334559379252, + "grad_norm": 0.9333013296127319, + "learning_rate": 0.0001, + "loss": 71.0789, + "step": 3030 + }, + { + "epoch": 4.896457902957626, + "grad_norm": 0.7888994216918945, + "learning_rate": 0.0001, + "loss": 70.8197, + "step": 3040 + }, + { + "epoch": 4.912581246536, + "grad_norm": 0.9194874167442322, + "learning_rate": 0.0001, + "loss": 72.071, + "step": 3050 + }, + { + "epoch": 4.928704590114375, + "grad_norm": 1.1252421140670776, + "learning_rate": 0.0001, + "loss": 71.6943, + "step": 3060 + }, + { + "epoch": 4.94482793369275, + "grad_norm": 1.1801071166992188, + "learning_rate": 0.0001, + "loss": 70.8937, + "step": 3070 + }, + { + "epoch": 4.960951277271124, + "grad_norm": 0.9162415266036987, + "learning_rate": 0.0001, + "loss": 70.024, + "step": 3080 + }, + { + "epoch": 4.977074620849499, + "grad_norm": 0.8833363652229309, + "learning_rate": 0.0001, + "loss": 71.3835, + "step": 3090 + }, + { + "epoch": 4.993197964427873, + "grad_norm": 0.9985883831977844, + "learning_rate": 0.0001, + "loss": 71.635, + "step": 3100 + }, + { + "epoch": 5.0080616717891875, + "grad_norm": 1.097298264503479, + "learning_rate": 0.0001, + "loss": 65.3542, + "step": 3110 + }, + { + "epoch": 5.024185015367562, + "grad_norm": 0.9703903198242188, + "learning_rate": 0.0001, + "loss": 69.8587, + "step": 3120 + }, + { + "epoch": 5.0403083589459365, + "grad_norm": 0.8894070386886597, + "learning_rate": 0.0001, + "loss": 71.2868, + "step": 3130 + }, + { + "epoch": 5.056431702524311, + "grad_norm": 0.6984127163887024, + "learning_rate": 0.0001, + "loss": 69.0425, + "step": 3140 + }, + { + "epoch": 5.072555046102686, + "grad_norm": 0.8607762455940247, + "learning_rate": 0.0001, + "loss": 69.5354, + "step": 3150 + }, + { + "epoch": 5.08867838968106, + "grad_norm": 1.1215914487838745, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 3160 + }, + { + "epoch": 5.104801733259435, + "grad_norm": 0.9114409685134888, + "learning_rate": 0.0001, + "loss": 68.4186, + "step": 3170 + }, + { + "epoch": 5.12092507683781, + "grad_norm": 1.010400652885437, + "learning_rate": 0.0001, + "loss": 68.3965, + "step": 3180 + }, + { + "epoch": 5.137048420416184, + "grad_norm": 0.8741958141326904, + "learning_rate": 0.0001, + "loss": 70.6728, + "step": 3190 + }, + { + "epoch": 5.153171763994559, + "grad_norm": 0.8760207891464233, + "learning_rate": 0.0001, + "loss": 72.0704, + "step": 3200 + }, + { + "epoch": 5.169295107572933, + "grad_norm": 1.328247308731079, + "learning_rate": 0.0001, + "loss": 69.9082, + "step": 3210 + }, + { + "epoch": 5.185418451151308, + "grad_norm": 0.8263765573501587, + "learning_rate": 0.0001, + "loss": 70.3539, + "step": 3220 + }, + { + "epoch": 5.201541794729682, + "grad_norm": 0.9358834028244019, + "learning_rate": 0.0001, + "loss": 70.4221, + "step": 3230 + }, + { + "epoch": 5.217665138308057, + "grad_norm": 1.0769531726837158, + "learning_rate": 0.0001, + "loss": 68.8167, + "step": 3240 + }, + { + "epoch": 5.233788481886431, + "grad_norm": 1.1438790559768677, + "learning_rate": 0.0001, + "loss": 68.9288, + "step": 3250 + }, + { + "epoch": 5.249911825464806, + "grad_norm": 0.9534218311309814, + "learning_rate": 0.0001, + "loss": 70.3277, + "step": 3260 + }, + { + "epoch": 5.26603516904318, + "grad_norm": 0.88532954454422, + "learning_rate": 0.0001, + "loss": 67.5781, + "step": 3270 + }, + { + "epoch": 5.282158512621555, + "grad_norm": 0.7986133694648743, + "learning_rate": 0.0001, + "loss": 69.759, + "step": 3280 + }, + { + "epoch": 5.29828185619993, + "grad_norm": 0.9100566506385803, + "learning_rate": 0.0001, + "loss": 68.6368, + "step": 3290 + }, + { + "epoch": 5.314405199778304, + "grad_norm": 0.8861745595932007, + "learning_rate": 0.0001, + "loss": 70.5756, + "step": 3300 + }, + { + "epoch": 5.330528543356679, + "grad_norm": 1.0729680061340332, + "learning_rate": 0.0001, + "loss": 70.3178, + "step": 3310 + }, + { + "epoch": 5.346651886935053, + "grad_norm": 1.0288280248641968, + "learning_rate": 0.0001, + "loss": 69.422, + "step": 3320 + }, + { + "epoch": 5.362775230513428, + "grad_norm": 0.8335140347480774, + "learning_rate": 0.0001, + "loss": 69.8281, + "step": 3330 + }, + { + "epoch": 5.378898574091802, + "grad_norm": 0.9789717197418213, + "learning_rate": 0.0001, + "loss": 71.1532, + "step": 3340 + }, + { + "epoch": 5.395021917670177, + "grad_norm": 0.9884677529335022, + "learning_rate": 0.0001, + "loss": 69.5673, + "step": 3350 + }, + { + "epoch": 5.411145261248551, + "grad_norm": 1.050154447555542, + "learning_rate": 0.0001, + "loss": 70.8015, + "step": 3360 + }, + { + "epoch": 5.427268604826926, + "grad_norm": 0.9645342230796814, + "learning_rate": 0.0001, + "loss": 69.1615, + "step": 3370 + }, + { + "epoch": 5.4433919484053, + "grad_norm": 0.8383159041404724, + "learning_rate": 0.0001, + "loss": 70.6299, + "step": 3380 + }, + { + "epoch": 5.459515291983675, + "grad_norm": 0.9250556826591492, + "learning_rate": 0.0001, + "loss": 69.231, + "step": 3390 + }, + { + "epoch": 5.475638635562049, + "grad_norm": 0.8108687400817871, + "learning_rate": 0.0001, + "loss": 70.5858, + "step": 3400 + }, + { + "epoch": 5.491761979140424, + "grad_norm": 0.8748818039894104, + "learning_rate": 0.0001, + "loss": 69.9001, + "step": 3410 + }, + { + "epoch": 5.507885322718799, + "grad_norm": 0.7411381006240845, + "learning_rate": 0.0001, + "loss": 68.3248, + "step": 3420 + }, + { + "epoch": 5.524008666297173, + "grad_norm": 1.0753326416015625, + "learning_rate": 0.0001, + "loss": 69.5831, + "step": 3430 + }, + { + "epoch": 5.540132009875548, + "grad_norm": 0.875254213809967, + "learning_rate": 0.0001, + "loss": 69.7516, + "step": 3440 + }, + { + "epoch": 5.556255353453922, + "grad_norm": 0.7637733817100525, + "learning_rate": 0.0001, + "loss": 68.0078, + "step": 3450 + }, + { + "epoch": 5.572378697032297, + "grad_norm": 0.968203604221344, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 3460 + }, + { + "epoch": 5.588502040610671, + "grad_norm": 0.8903141617774963, + "learning_rate": 0.0001, + "loss": 68.5835, + "step": 3470 + }, + { + "epoch": 5.604625384189046, + "grad_norm": 0.9536195397377014, + "learning_rate": 0.0001, + "loss": 71.0259, + "step": 3480 + }, + { + "epoch": 5.62074872776742, + "grad_norm": 0.8951714038848877, + "learning_rate": 0.0001, + "loss": 69.4422, + "step": 3490 + }, + { + "epoch": 5.636872071345795, + "grad_norm": 0.9639522433280945, + "learning_rate": 0.0001, + "loss": 66.5173, + "step": 3500 + }, + { + "epoch": 5.65299541492417, + "grad_norm": 1.0022730827331543, + "learning_rate": 0.0001, + "loss": 68.3443, + "step": 3510 + }, + { + "epoch": 5.669118758502544, + "grad_norm": 1.0654487609863281, + "learning_rate": 0.0001, + "loss": 69.0835, + "step": 3520 + }, + { + "epoch": 5.685242102080919, + "grad_norm": 0.9934629797935486, + "learning_rate": 0.0001, + "loss": 69.2278, + "step": 3530 + }, + { + "epoch": 5.7013654456592935, + "grad_norm": 0.8773566484451294, + "learning_rate": 0.0001, + "loss": 68.9339, + "step": 3540 + }, + { + "epoch": 5.7174887892376685, + "grad_norm": 0.7756998538970947, + "learning_rate": 0.0001, + "loss": 69.2683, + "step": 3550 + }, + { + "epoch": 5.7336121328160425, + "grad_norm": 0.8819359540939331, + "learning_rate": 0.0001, + "loss": 68.5071, + "step": 3560 + }, + { + "epoch": 5.7497354763944175, + "grad_norm": 1.1069825887680054, + "learning_rate": 0.0001, + "loss": 68.5236, + "step": 3570 + }, + { + "epoch": 5.765858819972792, + "grad_norm": 0.8006312847137451, + "learning_rate": 0.0001, + "loss": 68.7192, + "step": 3580 + }, + { + "epoch": 5.781982163551167, + "grad_norm": 1.0278719663619995, + "learning_rate": 0.0001, + "loss": 69.2545, + "step": 3590 + }, + { + "epoch": 5.798105507129541, + "grad_norm": 0.9205041527748108, + "learning_rate": 0.0001, + "loss": 69.7698, + "step": 3600 + }, + { + "epoch": 5.814228850707916, + "grad_norm": 1.0769038200378418, + "learning_rate": 0.0001, + "loss": 67.6568, + "step": 3610 + }, + { + "epoch": 5.83035219428629, + "grad_norm": 1.0565663576126099, + "learning_rate": 0.0001, + "loss": 68.9166, + "step": 3620 + }, + { + "epoch": 5.846475537864665, + "grad_norm": 0.9008663296699524, + "learning_rate": 0.0001, + "loss": 69.5837, + "step": 3630 + }, + { + "epoch": 5.86259888144304, + "grad_norm": 0.8369411826133728, + "learning_rate": 0.0001, + "loss": 68.5661, + "step": 3640 + }, + { + "epoch": 5.878722225021414, + "grad_norm": 1.0472379922866821, + "learning_rate": 0.0001, + "loss": 68.3122, + "step": 3650 + }, + { + "epoch": 5.894845568599789, + "grad_norm": 0.965277373790741, + "learning_rate": 0.0001, + "loss": 69.8062, + "step": 3660 + }, + { + "epoch": 5.910968912178163, + "grad_norm": 0.8936495780944824, + "learning_rate": 0.0001, + "loss": 67.1016, + "step": 3670 + }, + { + "epoch": 5.927092255756538, + "grad_norm": 0.8559162020683289, + "learning_rate": 0.0001, + "loss": 69.9986, + "step": 3680 + }, + { + "epoch": 5.943215599334912, + "grad_norm": 1.4815773963928223, + "learning_rate": 0.0001, + "loss": 66.7648, + "step": 3690 + }, + { + "epoch": 5.959338942913287, + "grad_norm": 0.9034469723701477, + "learning_rate": 0.0001, + "loss": 68.0767, + "step": 3700 + }, + { + "epoch": 5.975462286491661, + "grad_norm": 1.838799238204956, + "learning_rate": 0.0001, + "loss": 66.5848, + "step": 3710 + }, + { + "epoch": 5.991585630070036, + "grad_norm": 0.9274787902832031, + "learning_rate": 0.0001, + "loss": 70.0396, + "step": 3720 + }, + { + "epoch": 6.0064493374313495, + "grad_norm": 1.016974687576294, + "learning_rate": 0.0001, + "loss": 62.2268, + "step": 3730 + }, + { + "epoch": 6.022572681009724, + "grad_norm": 1.0522642135620117, + "learning_rate": 0.0001, + "loss": 69.3381, + "step": 3740 + }, + { + "epoch": 6.0386960245880985, + "grad_norm": 1.1157186031341553, + "learning_rate": 0.0001, + "loss": 68.8969, + "step": 3750 + }, + { + "epoch": 6.0548193681664735, + "grad_norm": 0.9913288950920105, + "learning_rate": 0.0001, + "loss": 67.3634, + "step": 3760 + }, + { + "epoch": 6.0709427117448485, + "grad_norm": 0.8823490142822266, + "learning_rate": 0.0001, + "loss": 70.978, + "step": 3770 + }, + { + "epoch": 6.0870660553232225, + "grad_norm": 0.853837788105011, + "learning_rate": 0.0001, + "loss": 68.9924, + "step": 3780 + }, + { + "epoch": 6.1031893989015975, + "grad_norm": 0.7937766909599304, + "learning_rate": 0.0001, + "loss": 68.1994, + "step": 3790 + }, + { + "epoch": 6.119312742479972, + "grad_norm": 1.7638057470321655, + "learning_rate": 0.0001, + "loss": 68.5078, + "step": 3800 + }, + { + "epoch": 6.135436086058347, + "grad_norm": 1.0542887449264526, + "learning_rate": 0.0001, + "loss": 68.9362, + "step": 3810 + }, + { + "epoch": 6.151559429636721, + "grad_norm": 1.06019926071167, + "learning_rate": 0.0001, + "loss": 67.2493, + "step": 3820 + }, + { + "epoch": 6.167682773215096, + "grad_norm": 0.9765111804008484, + "learning_rate": 0.0001, + "loss": 70.4797, + "step": 3830 + }, + { + "epoch": 6.18380611679347, + "grad_norm": 1.006372332572937, + "learning_rate": 0.0001, + "loss": 69.4233, + "step": 3840 + }, + { + "epoch": 6.199929460371845, + "grad_norm": 0.8274241089820862, + "learning_rate": 0.0001, + "loss": 67.9907, + "step": 3850 + }, + { + "epoch": 6.216052803950219, + "grad_norm": 0.9905765056610107, + "learning_rate": 0.0001, + "loss": 69.6988, + "step": 3860 + }, + { + "epoch": 6.232176147528594, + "grad_norm": 1.0065921545028687, + "learning_rate": 0.0001, + "loss": 68.7678, + "step": 3870 + }, + { + "epoch": 6.248299491106969, + "grad_norm": 0.8845255970954895, + "learning_rate": 0.0001, + "loss": 68.3039, + "step": 3880 + }, + { + "epoch": 6.264422834685343, + "grad_norm": 0.9030968546867371, + "learning_rate": 0.0001, + "loss": 68.671, + "step": 3890 + }, + { + "epoch": 6.280546178263718, + "grad_norm": 0.8650258183479309, + "learning_rate": 0.0001, + "loss": 68.5309, + "step": 3900 + }, + { + "epoch": 6.296669521842092, + "grad_norm": 0.8961272835731506, + "learning_rate": 0.0001, + "loss": 67.5611, + "step": 3910 + }, + { + "epoch": 6.312792865420467, + "grad_norm": 1.1702407598495483, + "learning_rate": 0.0001, + "loss": 68.2768, + "step": 3920 + }, + { + "epoch": 6.328916208998841, + "grad_norm": 0.8598962426185608, + "learning_rate": 0.0001, + "loss": 69.1961, + "step": 3930 + }, + { + "epoch": 6.345039552577216, + "grad_norm": 0.8714122176170349, + "learning_rate": 0.0001, + "loss": 66.8628, + "step": 3940 + }, + { + "epoch": 6.36116289615559, + "grad_norm": 0.7757895588874817, + "learning_rate": 0.0001, + "loss": 68.6908, + "step": 3950 + }, + { + "epoch": 6.377286239733965, + "grad_norm": 0.9189490675926208, + "learning_rate": 0.0001, + "loss": 67.8685, + "step": 3960 + }, + { + "epoch": 6.393409583312339, + "grad_norm": 0.8161299228668213, + "learning_rate": 0.0001, + "loss": 66.816, + "step": 3970 + }, + { + "epoch": 6.409532926890714, + "grad_norm": 0.8559791445732117, + "learning_rate": 0.0001, + "loss": 68.4302, + "step": 3980 + }, + { + "epoch": 6.425656270469089, + "grad_norm": 0.8087984919548035, + "learning_rate": 0.0001, + "loss": 66.2646, + "step": 3990 + }, + { + "epoch": 6.441779614047463, + "grad_norm": 0.7733110785484314, + "learning_rate": 0.0001, + "loss": 69.2769, + "step": 4000 + }, + { + "epoch": 6.457902957625838, + "grad_norm": 0.9335984587669373, + "learning_rate": 0.0001, + "loss": 67.1199, + "step": 4010 + }, + { + "epoch": 6.474026301204212, + "grad_norm": 0.9546418786048889, + "learning_rate": 0.0001, + "loss": 68.8706, + "step": 4020 + }, + { + "epoch": 6.490149644782587, + "grad_norm": 0.9670593738555908, + "learning_rate": 0.0001, + "loss": 68.2329, + "step": 4030 + }, + { + "epoch": 6.506272988360961, + "grad_norm": 0.8428588509559631, + "learning_rate": 0.0001, + "loss": 68.0105, + "step": 4040 + }, + { + "epoch": 6.522396331939336, + "grad_norm": 0.9755622744560242, + "learning_rate": 0.0001, + "loss": 68.656, + "step": 4050 + }, + { + "epoch": 6.53851967551771, + "grad_norm": 0.8972346186637878, + "learning_rate": 0.0001, + "loss": 69.3665, + "step": 4060 + }, + { + "epoch": 6.554643019096085, + "grad_norm": 0.9716668128967285, + "learning_rate": 0.0001, + "loss": 68.1053, + "step": 4070 + }, + { + "epoch": 6.570766362674459, + "grad_norm": 0.8147910833358765, + "learning_rate": 0.0001, + "loss": 67.9778, + "step": 4080 + }, + { + "epoch": 6.586889706252834, + "grad_norm": 0.8304234147071838, + "learning_rate": 0.0001, + "loss": 67.7069, + "step": 4090 + }, + { + "epoch": 6.603013049831208, + "grad_norm": 0.9356856346130371, + "learning_rate": 0.0001, + "loss": 68.1942, + "step": 4100 + }, + { + "epoch": 6.619136393409583, + "grad_norm": 1.017134189605713, + "learning_rate": 0.0001, + "loss": 67.4211, + "step": 4110 + }, + { + "epoch": 6.635259736987958, + "grad_norm": 1.142558217048645, + "learning_rate": 0.0001, + "loss": 68.4413, + "step": 4120 + }, + { + "epoch": 6.651383080566332, + "grad_norm": 0.9071153998374939, + "learning_rate": 0.0001, + "loss": 69.3666, + "step": 4130 + }, + { + "epoch": 6.667506424144707, + "grad_norm": 0.9620397686958313, + "learning_rate": 0.0001, + "loss": 68.208, + "step": 4140 + }, + { + "epoch": 6.683629767723081, + "grad_norm": 0.8282128572463989, + "learning_rate": 0.0001, + "loss": 67.1702, + "step": 4150 + }, + { + "epoch": 6.699753111301456, + "grad_norm": 0.8838199973106384, + "learning_rate": 0.0001, + "loss": 69.0074, + "step": 4160 + }, + { + "epoch": 6.71587645487983, + "grad_norm": 0.8864356279373169, + "learning_rate": 0.0001, + "loss": 67.8052, + "step": 4170 + }, + { + "epoch": 6.731999798458205, + "grad_norm": 0.9373385906219482, + "learning_rate": 0.0001, + "loss": 67.446, + "step": 4180 + }, + { + "epoch": 6.7481231420365795, + "grad_norm": 0.8308956027030945, + "learning_rate": 0.0001, + "loss": 68.716, + "step": 4190 + }, + { + "epoch": 6.7642464856149545, + "grad_norm": 0.8031372427940369, + "learning_rate": 0.0001, + "loss": 68.0482, + "step": 4200 + }, + { + "epoch": 6.780369829193329, + "grad_norm": 0.9258586764335632, + "learning_rate": 0.0001, + "loss": 68.5607, + "step": 4210 + }, + { + "epoch": 6.7964931727717035, + "grad_norm": 0.8147327899932861, + "learning_rate": 0.0001, + "loss": 67.7265, + "step": 4220 + }, + { + "epoch": 6.8126165163500785, + "grad_norm": 0.8717535734176636, + "learning_rate": 0.0001, + "loss": 67.8097, + "step": 4230 + }, + { + "epoch": 6.828739859928453, + "grad_norm": 0.9876209497451782, + "learning_rate": 0.0001, + "loss": 67.9068, + "step": 4240 + }, + { + "epoch": 6.8448632035068275, + "grad_norm": 0.718376874923706, + "learning_rate": 0.0001, + "loss": 68.1762, + "step": 4250 + }, + { + "epoch": 6.860986547085202, + "grad_norm": 0.8264320492744446, + "learning_rate": 0.0001, + "loss": 67.8391, + "step": 4260 + }, + { + "epoch": 6.877109890663577, + "grad_norm": 0.7478179335594177, + "learning_rate": 0.0001, + "loss": 67.6635, + "step": 4270 + }, + { + "epoch": 6.893233234241951, + "grad_norm": 0.8394957184791565, + "learning_rate": 0.0001, + "loss": 68.6073, + "step": 4280 + }, + { + "epoch": 6.909356577820326, + "grad_norm": 0.8534071445465088, + "learning_rate": 0.0001, + "loss": 69.6206, + "step": 4290 + }, + { + "epoch": 6.9254799213987, + "grad_norm": 0.9137582778930664, + "learning_rate": 0.0001, + "loss": 67.4362, + "step": 4300 + }, + { + "epoch": 6.941603264977075, + "grad_norm": 0.8869971036911011, + "learning_rate": 0.0001, + "loss": 67.6811, + "step": 4310 + }, + { + "epoch": 6.957726608555449, + "grad_norm": 0.889025866985321, + "learning_rate": 0.0001, + "loss": 65.3354, + "step": 4320 + }, + { + "epoch": 6.973849952133824, + "grad_norm": 0.7968549728393555, + "learning_rate": 0.0001, + "loss": 66.192, + "step": 4330 + }, + { + "epoch": 6.989973295712199, + "grad_norm": 1.0077433586120605, + "learning_rate": 0.0001, + "loss": 67.5216, + "step": 4340 + }, + { + "epoch": 7.004837003073512, + "grad_norm": 0.7900649309158325, + "learning_rate": 0.0001, + "loss": 62.2421, + "step": 4350 + }, + { + "epoch": 7.020960346651887, + "grad_norm": 1.0160410404205322, + "learning_rate": 0.0001, + "loss": 68.0978, + "step": 4360 + }, + { + "epoch": 7.037083690230261, + "grad_norm": 0.841387927532196, + "learning_rate": 0.0001, + "loss": 68.1008, + "step": 4370 + }, + { + "epoch": 7.053207033808636, + "grad_norm": 0.87884122133255, + "learning_rate": 0.0001, + "loss": 66.4667, + "step": 4380 + }, + { + "epoch": 7.06933037738701, + "grad_norm": 0.8580564856529236, + "learning_rate": 0.0001, + "loss": 67.2002, + "step": 4390 + }, + { + "epoch": 7.085453720965385, + "grad_norm": 0.7560063600540161, + "learning_rate": 0.0001, + "loss": 67.6743, + "step": 4400 + }, + { + "epoch": 7.1015770645437595, + "grad_norm": 0.855478048324585, + "learning_rate": 0.0001, + "loss": 67.822, + "step": 4410 + }, + { + "epoch": 7.1177004081221344, + "grad_norm": 0.7994911670684814, + "learning_rate": 0.0001, + "loss": 66.0094, + "step": 4420 + }, + { + "epoch": 7.1338237517005085, + "grad_norm": 0.8728616833686829, + "learning_rate": 0.0001, + "loss": 66.405, + "step": 4430 + }, + { + "epoch": 7.1499470952788835, + "grad_norm": 0.8434081673622131, + "learning_rate": 0.0001, + "loss": 67.0535, + "step": 4440 + }, + { + "epoch": 7.166070438857258, + "grad_norm": 0.8543094396591187, + "learning_rate": 0.0001, + "loss": 66.2417, + "step": 4450 + }, + { + "epoch": 7.182193782435633, + "grad_norm": 0.7209319472312927, + "learning_rate": 0.0001, + "loss": 67.3958, + "step": 4460 + }, + { + "epoch": 7.1983171260140075, + "grad_norm": 0.6735154390335083, + "learning_rate": 0.0001, + "loss": 67.4086, + "step": 4470 + }, + { + "epoch": 7.214440469592382, + "grad_norm": 0.9126104116439819, + "learning_rate": 0.0001, + "loss": 68.4191, + "step": 4480 + }, + { + "epoch": 7.230563813170757, + "grad_norm": 0.9334300756454468, + "learning_rate": 0.0001, + "loss": 68.3247, + "step": 4490 + }, + { + "epoch": 7.246687156749131, + "grad_norm": 1.5956038236618042, + "learning_rate": 0.0001, + "loss": 68.2779, + "step": 4500 + }, + { + "epoch": 7.262810500327506, + "grad_norm": 1.5012775659561157, + "learning_rate": 0.0001, + "loss": 68.3533, + "step": 4510 + }, + { + "epoch": 7.27893384390588, + "grad_norm": 1.1376632452011108, + "learning_rate": 0.0001, + "loss": 65.3627, + "step": 4520 + }, + { + "epoch": 7.295057187484255, + "grad_norm": 0.8597291111946106, + "learning_rate": 0.0001, + "loss": 67.8653, + "step": 4530 + }, + { + "epoch": 7.311180531062629, + "grad_norm": 0.7645048499107361, + "learning_rate": 0.0001, + "loss": 65.4592, + "step": 4540 + }, + { + "epoch": 7.327303874641004, + "grad_norm": 0.7717318534851074, + "learning_rate": 0.0001, + "loss": 66.902, + "step": 4550 + }, + { + "epoch": 7.343427218219378, + "grad_norm": 0.8884950876235962, + "learning_rate": 0.0001, + "loss": 67.1858, + "step": 4560 + }, + { + "epoch": 7.359550561797753, + "grad_norm": 0.7826898694038391, + "learning_rate": 0.0001, + "loss": 68.4673, + "step": 4570 + }, + { + "epoch": 7.375673905376128, + "grad_norm": 0.7484182119369507, + "learning_rate": 0.0001, + "loss": 67.4859, + "step": 4580 + }, + { + "epoch": 7.391797248954502, + "grad_norm": 0.918194591999054, + "learning_rate": 0.0001, + "loss": 67.7393, + "step": 4590 + }, + { + "epoch": 7.407920592532877, + "grad_norm": 1.0696896314620972, + "learning_rate": 0.0001, + "loss": 67.0101, + "step": 4600 + }, + { + "epoch": 7.424043936111251, + "grad_norm": 0.8190692067146301, + "learning_rate": 0.0001, + "loss": 68.5358, + "step": 4610 + }, + { + "epoch": 7.440167279689626, + "grad_norm": 0.9063783884048462, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 4620 + }, + { + "epoch": 7.456290623268, + "grad_norm": 0.7588100433349609, + "learning_rate": 0.0001, + "loss": 68.0066, + "step": 4630 + }, + { + "epoch": 7.472413966846375, + "grad_norm": 0.897281289100647, + "learning_rate": 0.0001, + "loss": 67.2499, + "step": 4640 + }, + { + "epoch": 7.488537310424749, + "grad_norm": 0.9505569338798523, + "learning_rate": 0.0001, + "loss": 67.5452, + "step": 4650 + }, + { + "epoch": 7.504660654003124, + "grad_norm": 0.8409159183502197, + "learning_rate": 0.0001, + "loss": 66.6293, + "step": 4660 + }, + { + "epoch": 7.520783997581498, + "grad_norm": 0.9878574013710022, + "learning_rate": 0.0001, + "loss": 65.3379, + "step": 4670 + }, + { + "epoch": 7.536907341159873, + "grad_norm": 1.0410685539245605, + "learning_rate": 0.0001, + "loss": 65.8028, + "step": 4680 + }, + { + "epoch": 7.553030684738248, + "grad_norm": 0.8417725563049316, + "learning_rate": 0.0001, + "loss": 65.4283, + "step": 4690 + }, + { + "epoch": 7.569154028316622, + "grad_norm": 0.7660863399505615, + "learning_rate": 0.0001, + "loss": 66.6765, + "step": 4700 + }, + { + "epoch": 7.585277371894997, + "grad_norm": 0.8074233531951904, + "learning_rate": 0.0001, + "loss": 67.1082, + "step": 4710 + }, + { + "epoch": 7.601400715473371, + "grad_norm": 0.7653326988220215, + "learning_rate": 0.0001, + "loss": 66.028, + "step": 4720 + }, + { + "epoch": 7.617524059051746, + "grad_norm": 1.1764041185379028, + "learning_rate": 0.0001, + "loss": 65.2587, + "step": 4730 + }, + { + "epoch": 7.63364740263012, + "grad_norm": 1.7604950666427612, + "learning_rate": 0.0001, + "loss": 66.0543, + "step": 4740 + }, + { + "epoch": 7.649770746208495, + "grad_norm": 0.941438615322113, + "learning_rate": 0.0001, + "loss": 65.6954, + "step": 4750 + }, + { + "epoch": 7.665894089786869, + "grad_norm": 0.740069568157196, + "learning_rate": 0.0001, + "loss": 67.9701, + "step": 4760 + }, + { + "epoch": 7.682017433365244, + "grad_norm": 0.9867332577705383, + "learning_rate": 0.0001, + "loss": 66.1445, + "step": 4770 + }, + { + "epoch": 7.698140776943618, + "grad_norm": 0.8156166672706604, + "learning_rate": 0.0001, + "loss": 67.2647, + "step": 4780 + }, + { + "epoch": 7.714264120521993, + "grad_norm": 0.7689579129219055, + "learning_rate": 0.0001, + "loss": 67.0961, + "step": 4790 + }, + { + "epoch": 7.730387464100367, + "grad_norm": 0.8159700036048889, + "learning_rate": 0.0001, + "loss": 66.4974, + "step": 4800 + }, + { + "epoch": 7.746510807678742, + "grad_norm": 0.8554222583770752, + "learning_rate": 0.0001, + "loss": 64.5057, + "step": 4810 + }, + { + "epoch": 7.762634151257117, + "grad_norm": 0.856629490852356, + "learning_rate": 0.0001, + "loss": 67.7819, + "step": 4820 + }, + { + "epoch": 7.778757494835491, + "grad_norm": 1.157179832458496, + "learning_rate": 0.0001, + "loss": 66.0122, + "step": 4830 + }, + { + "epoch": 7.794880838413866, + "grad_norm": 0.9597711563110352, + "learning_rate": 0.0001, + "loss": 66.8248, + "step": 4840 + }, + { + "epoch": 7.8110041819922404, + "grad_norm": 0.9175744652748108, + "learning_rate": 0.0001, + "loss": 65.7998, + "step": 4850 + }, + { + "epoch": 7.827127525570615, + "grad_norm": 0.8467298150062561, + "learning_rate": 0.0001, + "loss": 67.5363, + "step": 4860 + }, + { + "epoch": 7.8432508691489895, + "grad_norm": 0.863922655582428, + "learning_rate": 0.0001, + "loss": 67.7219, + "step": 4870 + }, + { + "epoch": 7.8593742127273645, + "grad_norm": 0.9896290302276611, + "learning_rate": 0.0001, + "loss": 66.7098, + "step": 4880 + }, + { + "epoch": 7.8754975563057386, + "grad_norm": 0.8828520774841309, + "learning_rate": 0.0001, + "loss": 66.0192, + "step": 4890 + }, + { + "epoch": 7.8916208998841135, + "grad_norm": 0.9093656539916992, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 4900 + }, + { + "epoch": 7.9077442434624885, + "grad_norm": 0.8665091395378113, + "learning_rate": 0.0001, + "loss": 66.5851, + "step": 4910 + }, + { + "epoch": 7.923867587040863, + "grad_norm": 0.8027398586273193, + "learning_rate": 0.0001, + "loss": 64.8607, + "step": 4920 + }, + { + "epoch": 7.9399909306192376, + "grad_norm": 0.7808436751365662, + "learning_rate": 0.0001, + "loss": 65.052, + "step": 4930 + }, + { + "epoch": 7.956114274197612, + "grad_norm": 0.7246070504188538, + "learning_rate": 0.0001, + "loss": 64.789, + "step": 4940 + }, + { + "epoch": 7.972237617775987, + "grad_norm": 1.0405027866363525, + "learning_rate": 0.0001, + "loss": 66.7735, + "step": 4950 + }, + { + "epoch": 7.988360961354361, + "grad_norm": 0.812921404838562, + "learning_rate": 0.0001, + "loss": 64.1286, + "step": 4960 + }, + { + "epoch": 8.003224668715674, + "grad_norm": 0.884215772151947, + "learning_rate": 0.0001, + "loss": 61.0812, + "step": 4970 + }, + { + "epoch": 8.01934801229405, + "grad_norm": 0.8519641757011414, + "learning_rate": 0.0001, + "loss": 66.2784, + "step": 4980 + }, + { + "epoch": 8.035471355872424, + "grad_norm": 0.8331199884414673, + "learning_rate": 0.0001, + "loss": 67.1405, + "step": 4990 + }, + { + "epoch": 8.0515946994508, + "grad_norm": 0.8568746447563171, + "learning_rate": 0.0001, + "loss": 66.7835, + "step": 5000 + }, + { + "epoch": 8.067718043029172, + "grad_norm": 1.170482873916626, + "learning_rate": 0.0001, + "loss": 64.2269, + "step": 5010 + }, + { + "epoch": 8.083841386607547, + "grad_norm": 0.9171113967895508, + "learning_rate": 0.0001, + "loss": 66.5567, + "step": 5020 + }, + { + "epoch": 8.099964730185922, + "grad_norm": 0.8988714814186096, + "learning_rate": 0.0001, + "loss": 66.0471, + "step": 5030 + }, + { + "epoch": 8.116088073764297, + "grad_norm": 0.7651671171188354, + "learning_rate": 0.0001, + "loss": 65.7185, + "step": 5040 + }, + { + "epoch": 8.132211417342672, + "grad_norm": 0.6890843510627747, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 5050 + }, + { + "epoch": 8.148334760921045, + "grad_norm": 0.7683818340301514, + "learning_rate": 0.0001, + "loss": 65.895, + "step": 5060 + }, + { + "epoch": 8.16445810449942, + "grad_norm": 0.9398573637008667, + "learning_rate": 0.0001, + "loss": 64.674, + "step": 5070 + }, + { + "epoch": 8.180581448077795, + "grad_norm": 0.7708531022071838, + "learning_rate": 0.0001, + "loss": 65.0637, + "step": 5080 + }, + { + "epoch": 8.19670479165617, + "grad_norm": 0.8840665221214294, + "learning_rate": 0.0001, + "loss": 65.6449, + "step": 5090 + }, + { + "epoch": 8.212828135234544, + "grad_norm": 0.8390524983406067, + "learning_rate": 0.0001, + "loss": 65.4738, + "step": 5100 + }, + { + "epoch": 8.228951478812919, + "grad_norm": 1.7559128999710083, + "learning_rate": 0.0001, + "loss": 68.911, + "step": 5110 + }, + { + "epoch": 8.245074822391294, + "grad_norm": 0.8293817043304443, + "learning_rate": 0.0001, + "loss": 65.2601, + "step": 5120 + }, + { + "epoch": 8.261198165969668, + "grad_norm": 0.8947123289108276, + "learning_rate": 0.0001, + "loss": 65.8611, + "step": 5130 + }, + { + "epoch": 8.277321509548042, + "grad_norm": 0.8526618480682373, + "learning_rate": 0.0001, + "loss": 65.559, + "step": 5140 + }, + { + "epoch": 8.293444853126417, + "grad_norm": 1.7423408031463623, + "learning_rate": 0.0001, + "loss": 64.5007, + "step": 5150 + }, + { + "epoch": 8.309568196704792, + "grad_norm": 1.0512502193450928, + "learning_rate": 0.0001, + "loss": 66.2593, + "step": 5160 + }, + { + "epoch": 8.325691540283167, + "grad_norm": 0.8129869103431702, + "learning_rate": 0.0001, + "loss": 64.7989, + "step": 5170 + }, + { + "epoch": 8.341814883861542, + "grad_norm": 0.852199912071228, + "learning_rate": 0.0001, + "loss": 65.9627, + "step": 5180 + }, + { + "epoch": 8.357938227439915, + "grad_norm": 0.8932474851608276, + "learning_rate": 0.0001, + "loss": 64.1035, + "step": 5190 + }, + { + "epoch": 8.37406157101829, + "grad_norm": 1.2631597518920898, + "learning_rate": 0.0001, + "loss": 65.3406, + "step": 5200 + }, + { + "epoch": 8.390184914596665, + "grad_norm": 0.9062619209289551, + "learning_rate": 0.0001, + "loss": 65.1841, + "step": 5210 + }, + { + "epoch": 8.40630825817504, + "grad_norm": 0.9153940081596375, + "learning_rate": 0.0001, + "loss": 66.3852, + "step": 5220 + }, + { + "epoch": 8.422431601753413, + "grad_norm": 0.8056164979934692, + "learning_rate": 0.0001, + "loss": 66.511, + "step": 5230 + }, + { + "epoch": 8.438554945331788, + "grad_norm": 0.7506036162376404, + "learning_rate": 0.0001, + "loss": 65.275, + "step": 5240 + }, + { + "epoch": 8.454678288910163, + "grad_norm": 0.7994081377983093, + "learning_rate": 0.0001, + "loss": 64.9946, + "step": 5250 + }, + { + "epoch": 8.470801632488538, + "grad_norm": 0.9887873530387878, + "learning_rate": 0.0001, + "loss": 67.2057, + "step": 5260 + }, + { + "epoch": 8.486924976066913, + "grad_norm": 0.8495044112205505, + "learning_rate": 0.0001, + "loss": 66.2481, + "step": 5270 + }, + { + "epoch": 8.503048319645286, + "grad_norm": 0.7353820204734802, + "learning_rate": 0.0001, + "loss": 65.6163, + "step": 5280 + }, + { + "epoch": 8.519171663223661, + "grad_norm": 0.795216977596283, + "learning_rate": 0.0001, + "loss": 64.808, + "step": 5290 + }, + { + "epoch": 8.535295006802036, + "grad_norm": 0.8121851682662964, + "learning_rate": 0.0001, + "loss": 65.9755, + "step": 5300 + }, + { + "epoch": 8.55141835038041, + "grad_norm": 0.8164243698120117, + "learning_rate": 0.0001, + "loss": 66.0583, + "step": 5310 + }, + { + "epoch": 8.567541693958784, + "grad_norm": 0.9893137216567993, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 5320 + }, + { + "epoch": 8.583665037537159, + "grad_norm": 1.3616434335708618, + "learning_rate": 0.0001, + "loss": 64.1444, + "step": 5330 + }, + { + "epoch": 8.599788381115534, + "grad_norm": 0.7002740502357483, + "learning_rate": 0.0001, + "loss": 65.0906, + "step": 5340 + }, + { + "epoch": 8.615911724693909, + "grad_norm": 0.838424563407898, + "learning_rate": 0.0001, + "loss": 65.4578, + "step": 5350 + }, + { + "epoch": 8.632035068272282, + "grad_norm": 0.988902747631073, + "learning_rate": 0.0001, + "loss": 64.3405, + "step": 5360 + }, + { + "epoch": 8.648158411850657, + "grad_norm": 0.7941252589225769, + "learning_rate": 0.0001, + "loss": 67.0146, + "step": 5370 + }, + { + "epoch": 8.664281755429032, + "grad_norm": 0.7727184295654297, + "learning_rate": 0.0001, + "loss": 64.8011, + "step": 5380 + }, + { + "epoch": 8.680405099007407, + "grad_norm": 0.9514367580413818, + "learning_rate": 0.0001, + "loss": 65.8165, + "step": 5390 + }, + { + "epoch": 8.69652844258578, + "grad_norm": 0.870544970035553, + "learning_rate": 0.0001, + "loss": 65.803, + "step": 5400 + }, + { + "epoch": 8.712651786164155, + "grad_norm": 0.7348082065582275, + "learning_rate": 0.0001, + "loss": 65.8409, + "step": 5410 + }, + { + "epoch": 8.72877512974253, + "grad_norm": 0.7543814778327942, + "learning_rate": 0.0001, + "loss": 65.5958, + "step": 5420 + }, + { + "epoch": 8.744898473320905, + "grad_norm": 0.7520368099212646, + "learning_rate": 0.0001, + "loss": 65.3323, + "step": 5430 + }, + { + "epoch": 8.76102181689928, + "grad_norm": 0.7610031366348267, + "learning_rate": 0.0001, + "loss": 65.4386, + "step": 5440 + }, + { + "epoch": 8.777145160477653, + "grad_norm": 0.8854930400848389, + "learning_rate": 0.0001, + "loss": 66.181, + "step": 5450 + }, + { + "epoch": 8.793268504056028, + "grad_norm": 0.7659342885017395, + "learning_rate": 0.0001, + "loss": 65.2586, + "step": 5460 + }, + { + "epoch": 8.809391847634403, + "grad_norm": 0.8592572212219238, + "learning_rate": 0.0001, + "loss": 66.3306, + "step": 5470 + }, + { + "epoch": 8.825515191212778, + "grad_norm": 0.8169172406196594, + "learning_rate": 0.0001, + "loss": 65.8283, + "step": 5480 + }, + { + "epoch": 8.841638534791151, + "grad_norm": 0.7380614876747131, + "learning_rate": 0.0001, + "loss": 65.8711, + "step": 5490 + }, + { + "epoch": 8.857761878369526, + "grad_norm": 0.9993331432342529, + "learning_rate": 0.0001, + "loss": 64.9378, + "step": 5500 + }, + { + "epoch": 8.873885221947901, + "grad_norm": 0.6252862811088562, + "learning_rate": 0.0001, + "loss": 64.1655, + "step": 5510 + }, + { + "epoch": 8.890008565526276, + "grad_norm": 0.8441852331161499, + "learning_rate": 0.0001, + "loss": 64.9795, + "step": 5520 + }, + { + "epoch": 8.906131909104651, + "grad_norm": 0.7075499892234802, + "learning_rate": 0.0001, + "loss": 65.5843, + "step": 5530 + }, + { + "epoch": 8.922255252683025, + "grad_norm": 0.7374144196510315, + "learning_rate": 0.0001, + "loss": 67.5064, + "step": 5540 + }, + { + "epoch": 8.9383785962614, + "grad_norm": 0.8073632121086121, + "learning_rate": 0.0001, + "loss": 67.0182, + "step": 5550 + }, + { + "epoch": 8.954501939839774, + "grad_norm": 0.7576195001602173, + "learning_rate": 0.0001, + "loss": 65.9139, + "step": 5560 + }, + { + "epoch": 8.97062528341815, + "grad_norm": 1.6479626893997192, + "learning_rate": 0.0001, + "loss": 66.3517, + "step": 5570 + }, + { + "epoch": 8.986748626996523, + "grad_norm": 0.846513569355011, + "learning_rate": 0.0001, + "loss": 64.2908, + "step": 5580 + }, + { + "epoch": 9.001612334357837, + "grad_norm": 0.9551037549972534, + "learning_rate": 0.0001, + "loss": 61.7312, + "step": 5590 + }, + { + "epoch": 9.017735677936212, + "grad_norm": 0.8250304460525513, + "learning_rate": 0.0001, + "loss": 66.3351, + "step": 5600 + }, + { + "epoch": 9.033859021514587, + "grad_norm": 0.7932960391044617, + "learning_rate": 0.0001, + "loss": 66.7753, + "step": 5610 + }, + { + "epoch": 9.04998236509296, + "grad_norm": 0.9839881658554077, + "learning_rate": 0.0001, + "loss": 64.5146, + "step": 5620 + }, + { + "epoch": 9.066105708671335, + "grad_norm": 0.7337850332260132, + "learning_rate": 0.0001, + "loss": 65.0544, + "step": 5630 + }, + { + "epoch": 9.08222905224971, + "grad_norm": 0.7496634721755981, + "learning_rate": 0.0001, + "loss": 64.1451, + "step": 5640 + }, + { + "epoch": 9.098352395828085, + "grad_norm": 0.9457297921180725, + "learning_rate": 0.0001, + "loss": 65.3036, + "step": 5650 + }, + { + "epoch": 9.11447573940646, + "grad_norm": 0.8315348625183105, + "learning_rate": 0.0001, + "loss": 66.0585, + "step": 5660 + }, + { + "epoch": 9.130599082984833, + "grad_norm": 0.9666195511817932, + "learning_rate": 0.0001, + "loss": 64.6802, + "step": 5670 + }, + { + "epoch": 9.146722426563208, + "grad_norm": 0.7416576147079468, + "learning_rate": 0.0001, + "loss": 65.1557, + "step": 5680 + }, + { + "epoch": 9.162845770141583, + "grad_norm": 0.7723799347877502, + "learning_rate": 0.0001, + "loss": 66.2871, + "step": 5690 + }, + { + "epoch": 9.178969113719958, + "grad_norm": 0.7461305856704712, + "learning_rate": 0.0001, + "loss": 65.5795, + "step": 5700 + }, + { + "epoch": 9.195092457298331, + "grad_norm": 0.6663411259651184, + "learning_rate": 0.0001, + "loss": 66.4094, + "step": 5710 + }, + { + "epoch": 9.211215800876706, + "grad_norm": 0.9340134859085083, + "learning_rate": 0.0001, + "loss": 63.5753, + "step": 5720 + }, + { + "epoch": 9.227339144455081, + "grad_norm": 0.7643963694572449, + "learning_rate": 0.0001, + "loss": 65.3359, + "step": 5730 + }, + { + "epoch": 9.243462488033456, + "grad_norm": 0.6931570768356323, + "learning_rate": 0.0001, + "loss": 65.0755, + "step": 5740 + }, + { + "epoch": 9.259585831611831, + "grad_norm": 0.9096988439559937, + "learning_rate": 0.0001, + "loss": 64.7983, + "step": 5750 + }, + { + "epoch": 9.275709175190205, + "grad_norm": 0.8360128402709961, + "learning_rate": 0.0001, + "loss": 64.3604, + "step": 5760 + }, + { + "epoch": 9.29183251876858, + "grad_norm": 0.9302591681480408, + "learning_rate": 0.0001, + "loss": 66.2754, + "step": 5770 + }, + { + "epoch": 9.307955862346954, + "grad_norm": 0.7000735998153687, + "learning_rate": 0.0001, + "loss": 65.2241, + "step": 5780 + }, + { + "epoch": 9.32407920592533, + "grad_norm": 0.7321175932884216, + "learning_rate": 0.0001, + "loss": 65.3558, + "step": 5790 + }, + { + "epoch": 9.340202549503703, + "grad_norm": 0.7948715090751648, + "learning_rate": 0.0001, + "loss": 64.2793, + "step": 5800 + }, + { + "epoch": 9.356325893082078, + "grad_norm": 0.7676129341125488, + "learning_rate": 0.0001, + "loss": 66.4778, + "step": 5810 + }, + { + "epoch": 9.372449236660453, + "grad_norm": 0.977236270904541, + "learning_rate": 0.0001, + "loss": 64.5587, + "step": 5820 + }, + { + "epoch": 9.388572580238828, + "grad_norm": 0.86808842420578, + "learning_rate": 0.0001, + "loss": 65.8026, + "step": 5830 + }, + { + "epoch": 9.4046959238172, + "grad_norm": 0.7455324530601501, + "learning_rate": 0.0001, + "loss": 65.066, + "step": 5840 + }, + { + "epoch": 9.420819267395576, + "grad_norm": 0.6784212589263916, + "learning_rate": 0.0001, + "loss": 65.4167, + "step": 5850 + }, + { + "epoch": 9.43694261097395, + "grad_norm": 1.033731460571289, + "learning_rate": 0.0001, + "loss": 66.4502, + "step": 5860 + }, + { + "epoch": 9.453065954552326, + "grad_norm": 0.7946107983589172, + "learning_rate": 0.0001, + "loss": 63.7567, + "step": 5870 + }, + { + "epoch": 9.4691892981307, + "grad_norm": 0.7826364636421204, + "learning_rate": 0.0001, + "loss": 65.4484, + "step": 5880 + }, + { + "epoch": 9.485312641709074, + "grad_norm": 0.7605016827583313, + "learning_rate": 0.0001, + "loss": 66.8459, + "step": 5890 + }, + { + "epoch": 9.501435985287449, + "grad_norm": 0.7383126020431519, + "learning_rate": 0.0001, + "loss": 64.6873, + "step": 5900 + }, + { + "epoch": 9.517559328865824, + "grad_norm": 0.8355686664581299, + "learning_rate": 0.0001, + "loss": 63.5145, + "step": 5910 + }, + { + "epoch": 9.533682672444199, + "grad_norm": 0.8827091455459595, + "learning_rate": 0.0001, + "loss": 67.2207, + "step": 5920 + }, + { + "epoch": 9.549806016022572, + "grad_norm": 0.6988142728805542, + "learning_rate": 0.0001, + "loss": 64.2289, + "step": 5930 + }, + { + "epoch": 9.565929359600947, + "grad_norm": 0.7609000205993652, + "learning_rate": 0.0001, + "loss": 63.9324, + "step": 5940 + }, + { + "epoch": 9.582052703179322, + "grad_norm": 0.8053853511810303, + "learning_rate": 0.0001, + "loss": 62.5564, + "step": 5950 + }, + { + "epoch": 9.598176046757697, + "grad_norm": 0.7684671878814697, + "learning_rate": 0.0001, + "loss": 66.2669, + "step": 5960 + }, + { + "epoch": 9.614299390336072, + "grad_norm": 0.8411065340042114, + "learning_rate": 0.0001, + "loss": 64.5087, + "step": 5970 + }, + { + "epoch": 9.630422733914445, + "grad_norm": 0.963498592376709, + "learning_rate": 0.0001, + "loss": 68.1736, + "step": 5980 + }, + { + "epoch": 9.64654607749282, + "grad_norm": 0.8715981841087341, + "learning_rate": 0.0001, + "loss": 64.1686, + "step": 5990 + }, + { + "epoch": 9.662669421071195, + "grad_norm": 0.8217800855636597, + "learning_rate": 0.0001, + "loss": 64.2211, + "step": 6000 + }, + { + "epoch": 9.67879276464957, + "grad_norm": 0.7824742197990417, + "learning_rate": 0.0001, + "loss": 65.9052, + "step": 6010 + }, + { + "epoch": 9.694916108227943, + "grad_norm": 0.7440009117126465, + "learning_rate": 0.0001, + "loss": 63.3749, + "step": 6020 + }, + { + "epoch": 9.711039451806318, + "grad_norm": 0.8759479522705078, + "learning_rate": 0.0001, + "loss": 63.4488, + "step": 6030 + }, + { + "epoch": 9.727162795384693, + "grad_norm": 3.662144184112549, + "learning_rate": 0.0001, + "loss": 62.8787, + "step": 6040 + }, + { + "epoch": 9.743286138963068, + "grad_norm": 1.007149577140808, + "learning_rate": 0.0001, + "loss": 66.0053, + "step": 6050 + }, + { + "epoch": 9.759409482541441, + "grad_norm": 0.8369242548942566, + "learning_rate": 0.0001, + "loss": 67.233, + "step": 6060 + }, + { + "epoch": 9.775532826119816, + "grad_norm": 0.8160434365272522, + "learning_rate": 0.0001, + "loss": 65.2089, + "step": 6070 + }, + { + "epoch": 9.791656169698191, + "grad_norm": 0.8482319712638855, + "learning_rate": 0.0001, + "loss": 65.0718, + "step": 6080 + }, + { + "epoch": 9.807779513276566, + "grad_norm": 0.8800129890441895, + "learning_rate": 0.0001, + "loss": 65.145, + "step": 6090 + }, + { + "epoch": 9.82390285685494, + "grad_norm": 0.8911405801773071, + "learning_rate": 0.0001, + "loss": 65.7952, + "step": 6100 + }, + { + "epoch": 9.840026200433314, + "grad_norm": 0.8108358979225159, + "learning_rate": 0.0001, + "loss": 65.1928, + "step": 6110 + }, + { + "epoch": 9.85614954401169, + "grad_norm": 0.8919743895530701, + "learning_rate": 0.0001, + "loss": 65.8129, + "step": 6120 + }, + { + "epoch": 9.872272887590064, + "grad_norm": 0.73152756690979, + "learning_rate": 0.0001, + "loss": 65.7141, + "step": 6130 + }, + { + "epoch": 9.88839623116844, + "grad_norm": 0.8583903312683105, + "learning_rate": 0.0001, + "loss": 64.8231, + "step": 6140 + }, + { + "epoch": 9.904519574746812, + "grad_norm": 0.798543393611908, + "learning_rate": 0.0001, + "loss": 64.5073, + "step": 6150 + }, + { + "epoch": 9.920642918325187, + "grad_norm": 0.8657358288764954, + "learning_rate": 0.0001, + "loss": 64.1207, + "step": 6160 + }, + { + "epoch": 9.936766261903562, + "grad_norm": 0.9324677586555481, + "learning_rate": 0.0001, + "loss": 64.7129, + "step": 6170 + }, + { + "epoch": 9.952889605481937, + "grad_norm": 0.7371154427528381, + "learning_rate": 0.0001, + "loss": 64.0251, + "step": 6180 + }, + { + "epoch": 9.96901294906031, + "grad_norm": 0.8273237347602844, + "learning_rate": 0.0001, + "loss": 65.8633, + "step": 6190 + }, + { + "epoch": 9.985136292638686, + "grad_norm": 0.7219365239143372, + "learning_rate": 0.0001, + "loss": 63.4029, + "step": 6200 + }, + { + "epoch": 10.0, + "grad_norm": 0.76242995262146, + "learning_rate": 0.0001, + "loss": 60.9994, + "step": 6210 + }, + { + "epoch": 10.016123343578375, + "grad_norm": 0.8677287697792053, + "learning_rate": 0.0001, + "loss": 65.9687, + "step": 6220 + }, + { + "epoch": 10.03224668715675, + "grad_norm": 0.7073521018028259, + "learning_rate": 0.0001, + "loss": 63.0083, + "step": 6230 + }, + { + "epoch": 10.048370030735123, + "grad_norm": 0.794535756111145, + "learning_rate": 0.0001, + "loss": 63.0616, + "step": 6240 + }, + { + "epoch": 10.064493374313498, + "grad_norm": 0.6974631547927856, + "learning_rate": 0.0001, + "loss": 64.6579, + "step": 6250 + }, + { + "epoch": 10.080616717891873, + "grad_norm": 0.8279150128364563, + "learning_rate": 0.0001, + "loss": 65.1069, + "step": 6260 + }, + { + "epoch": 10.096740061470248, + "grad_norm": 0.8682932257652283, + "learning_rate": 0.0001, + "loss": 64.5019, + "step": 6270 + }, + { + "epoch": 10.112863405048621, + "grad_norm": 1.0158190727233887, + "learning_rate": 0.0001, + "loss": 65.8854, + "step": 6280 + }, + { + "epoch": 10.128986748626996, + "grad_norm": 0.6911011934280396, + "learning_rate": 0.0001, + "loss": 62.4879, + "step": 6290 + }, + { + "epoch": 10.145110092205371, + "grad_norm": 0.7394022941589355, + "learning_rate": 0.0001, + "loss": 64.2604, + "step": 6300 + }, + { + "epoch": 10.161233435783746, + "grad_norm": 0.7941621541976929, + "learning_rate": 0.0001, + "loss": 64.5133, + "step": 6310 + }, + { + "epoch": 10.17735677936212, + "grad_norm": 1.0238994359970093, + "learning_rate": 0.0001, + "loss": 63.8594, + "step": 6320 + }, + { + "epoch": 10.193480122940494, + "grad_norm": 0.7332146763801575, + "learning_rate": 0.0001, + "loss": 66.5434, + "step": 6330 + }, + { + "epoch": 10.20960346651887, + "grad_norm": 0.9375078082084656, + "learning_rate": 0.0001, + "loss": 63.4793, + "step": 6340 + }, + { + "epoch": 10.225726810097244, + "grad_norm": 0.715931236743927, + "learning_rate": 0.0001, + "loss": 65.8812, + "step": 6350 + }, + { + "epoch": 10.24185015367562, + "grad_norm": 1.291536808013916, + "learning_rate": 0.0001, + "loss": 64.9125, + "step": 6360 + }, + { + "epoch": 10.257973497253992, + "grad_norm": 0.8942781686782837, + "learning_rate": 0.0001, + "loss": 67.1706, + "step": 6370 + }, + { + "epoch": 10.274096840832367, + "grad_norm": 0.8272124528884888, + "learning_rate": 0.0001, + "loss": 62.1721, + "step": 6380 + }, + { + "epoch": 10.290220184410742, + "grad_norm": 0.8191152215003967, + "learning_rate": 0.0001, + "loss": 64.8931, + "step": 6390 + }, + { + "epoch": 10.306343527989117, + "grad_norm": 1.7500241994857788, + "learning_rate": 0.0001, + "loss": 62.9693, + "step": 6400 + }, + { + "epoch": 10.32246687156749, + "grad_norm": 1.0159430503845215, + "learning_rate": 0.0001, + "loss": 63.0889, + "step": 6410 + }, + { + "epoch": 10.338590215145866, + "grad_norm": 0.7300348281860352, + "learning_rate": 0.0001, + "loss": 65.1742, + "step": 6420 + }, + { + "epoch": 10.35471355872424, + "grad_norm": 1.3132387399673462, + "learning_rate": 0.0001, + "loss": 64.8867, + "step": 6430 + }, + { + "epoch": 10.370836902302615, + "grad_norm": 0.7502451539039612, + "learning_rate": 0.0001, + "loss": 66.5094, + "step": 6440 + }, + { + "epoch": 10.38696024588099, + "grad_norm": 0.9408686757087708, + "learning_rate": 0.0001, + "loss": 65.1515, + "step": 6450 + }, + { + "epoch": 10.403083589459364, + "grad_norm": 0.769598126411438, + "learning_rate": 0.0001, + "loss": 65.1478, + "step": 6460 + }, + { + "epoch": 10.419206933037739, + "grad_norm": 0.9086162447929382, + "learning_rate": 0.0001, + "loss": 63.5263, + "step": 6470 + }, + { + "epoch": 10.435330276616114, + "grad_norm": 0.7899322509765625, + "learning_rate": 0.0001, + "loss": 63.3939, + "step": 6480 + }, + { + "epoch": 10.451453620194489, + "grad_norm": 0.9135434031486511, + "learning_rate": 0.0001, + "loss": 63.867, + "step": 6490 + }, + { + "epoch": 10.467576963772862, + "grad_norm": 0.9312737584114075, + "learning_rate": 0.0001, + "loss": 63.6939, + "step": 6500 + }, + { + "epoch": 10.483700307351237, + "grad_norm": 0.861764132976532, + "learning_rate": 0.0001, + "loss": 63.4523, + "step": 6510 + }, + { + "epoch": 10.499823650929612, + "grad_norm": 0.7056366801261902, + "learning_rate": 0.0001, + "loss": 63.2889, + "step": 6520 + }, + { + "epoch": 10.515946994507987, + "grad_norm": 0.7007710337638855, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 6530 + }, + { + "epoch": 10.53207033808636, + "grad_norm": 0.7091413736343384, + "learning_rate": 0.0001, + "loss": 63.4185, + "step": 6540 + }, + { + "epoch": 10.548193681664735, + "grad_norm": 0.7147234678268433, + "learning_rate": 0.0001, + "loss": 65.0726, + "step": 6550 + }, + { + "epoch": 10.56431702524311, + "grad_norm": 0.7954673767089844, + "learning_rate": 0.0001, + "loss": 65.0334, + "step": 6560 + }, + { + "epoch": 10.580440368821485, + "grad_norm": 0.725352942943573, + "learning_rate": 0.0001, + "loss": 62.6663, + "step": 6570 + }, + { + "epoch": 10.59656371239986, + "grad_norm": 0.9563508629798889, + "learning_rate": 0.0001, + "loss": 64.0705, + "step": 6580 + }, + { + "epoch": 10.612687055978233, + "grad_norm": 0.7778734564781189, + "learning_rate": 0.0001, + "loss": 63.7619, + "step": 6590 + }, + { + "epoch": 10.628810399556608, + "grad_norm": 0.8414369821548462, + "learning_rate": 0.0001, + "loss": 63.9238, + "step": 6600 + }, + { + "epoch": 10.644933743134983, + "grad_norm": 0.8070562481880188, + "learning_rate": 0.0001, + "loss": 63.9591, + "step": 6610 + }, + { + "epoch": 10.661057086713358, + "grad_norm": 0.6493734121322632, + "learning_rate": 0.0001, + "loss": 63.7237, + "step": 6620 + }, + { + "epoch": 10.677180430291731, + "grad_norm": 0.8263372778892517, + "learning_rate": 0.0001, + "loss": 64.2262, + "step": 6630 + }, + { + "epoch": 10.693303773870106, + "grad_norm": 1.0090227127075195, + "learning_rate": 0.0001, + "loss": 63.5469, + "step": 6640 + }, + { + "epoch": 10.709427117448481, + "grad_norm": 0.998319685459137, + "learning_rate": 0.0001, + "loss": 62.9113, + "step": 6650 + }, + { + "epoch": 10.725550461026856, + "grad_norm": 1.006489634513855, + "learning_rate": 0.0001, + "loss": 66.1458, + "step": 6660 + }, + { + "epoch": 10.741673804605231, + "grad_norm": 0.9575757384300232, + "learning_rate": 0.0001, + "loss": 63.0913, + "step": 6670 + }, + { + "epoch": 10.757797148183604, + "grad_norm": 0.6832238435745239, + "learning_rate": 0.0001, + "loss": 66.7501, + "step": 6680 + }, + { + "epoch": 10.773920491761979, + "grad_norm": 0.8075717687606812, + "learning_rate": 0.0001, + "loss": 64.2839, + "step": 6690 + }, + { + "epoch": 10.790043835340354, + "grad_norm": 0.9402624368667603, + "learning_rate": 0.0001, + "loss": 64.9296, + "step": 6700 + }, + { + "epoch": 10.806167178918729, + "grad_norm": 0.7681158185005188, + "learning_rate": 0.0001, + "loss": 63.5686, + "step": 6710 + }, + { + "epoch": 10.822290522497102, + "grad_norm": 0.8451285362243652, + "learning_rate": 0.0001, + "loss": 64.5404, + "step": 6720 + }, + { + "epoch": 10.838413866075477, + "grad_norm": 0.7278769016265869, + "learning_rate": 0.0001, + "loss": 64.1341, + "step": 6730 + }, + { + "epoch": 10.854537209653852, + "grad_norm": 0.8524075746536255, + "learning_rate": 0.0001, + "loss": 64.0483, + "step": 6740 + }, + { + "epoch": 10.870660553232227, + "grad_norm": 0.703598141670227, + "learning_rate": 0.0001, + "loss": 64.0072, + "step": 6750 + }, + { + "epoch": 10.8867838968106, + "grad_norm": 0.7740384936332703, + "learning_rate": 0.0001, + "loss": 63.4171, + "step": 6760 + }, + { + "epoch": 10.902907240388975, + "grad_norm": 0.8302356004714966, + "learning_rate": 0.0001, + "loss": 65.5159, + "step": 6770 + }, + { + "epoch": 10.91903058396735, + "grad_norm": 1.0289733409881592, + "learning_rate": 0.0001, + "loss": 64.9785, + "step": 6780 + }, + { + "epoch": 10.935153927545725, + "grad_norm": 0.6721987128257751, + "learning_rate": 0.0001, + "loss": 63.965, + "step": 6790 + }, + { + "epoch": 10.951277271124098, + "grad_norm": 0.7614948749542236, + "learning_rate": 0.0001, + "loss": 64.1375, + "step": 6800 + }, + { + "epoch": 10.967400614702473, + "grad_norm": 0.7342216372489929, + "learning_rate": 0.0001, + "loss": 64.4342, + "step": 6810 + }, + { + "epoch": 10.983523958280848, + "grad_norm": 0.8808079957962036, + "learning_rate": 0.0001, + "loss": 65.2864, + "step": 6820 + }, + { + "epoch": 10.999647301859223, + "grad_norm": 0.818477988243103, + "learning_rate": 0.0001, + "loss": 63.9931, + "step": 6830 + }, + { + "epoch": 11.014511009220538, + "grad_norm": 0.7867474555969238, + "learning_rate": 0.0001, + "loss": 59.2426, + "step": 6840 + }, + { + "epoch": 11.030634352798911, + "grad_norm": 0.6163678765296936, + "learning_rate": 0.0001, + "loss": 65.0149, + "step": 6850 + }, + { + "epoch": 11.046757696377286, + "grad_norm": 0.8334609866142273, + "learning_rate": 0.0001, + "loss": 63.5017, + "step": 6860 + }, + { + "epoch": 11.062881039955661, + "grad_norm": 0.7335264086723328, + "learning_rate": 0.0001, + "loss": 64.6307, + "step": 6870 + }, + { + "epoch": 11.079004383534036, + "grad_norm": 0.7723846435546875, + "learning_rate": 0.0001, + "loss": 64.316, + "step": 6880 + }, + { + "epoch": 11.09512772711241, + "grad_norm": 0.7972145080566406, + "learning_rate": 0.0001, + "loss": 63.1469, + "step": 6890 + }, + { + "epoch": 11.111251070690784, + "grad_norm": 0.7696133852005005, + "learning_rate": 0.0001, + "loss": 64.0338, + "step": 6900 + }, + { + "epoch": 11.127374414269159, + "grad_norm": 0.8972133994102478, + "learning_rate": 0.0001, + "loss": 65.5727, + "step": 6910 + }, + { + "epoch": 11.143497757847534, + "grad_norm": 0.6776295900344849, + "learning_rate": 0.0001, + "loss": 66.176, + "step": 6920 + }, + { + "epoch": 11.159621101425909, + "grad_norm": 0.7345874905586243, + "learning_rate": 0.0001, + "loss": 64.0949, + "step": 6930 + }, + { + "epoch": 11.175744445004282, + "grad_norm": 0.7988374829292297, + "learning_rate": 0.0001, + "loss": 64.7721, + "step": 6940 + }, + { + "epoch": 11.191867788582657, + "grad_norm": 0.7552387714385986, + "learning_rate": 0.0001, + "loss": 63.1053, + "step": 6950 + }, + { + "epoch": 11.207991132161032, + "grad_norm": 0.6388214230537415, + "learning_rate": 0.0001, + "loss": 63.7871, + "step": 6960 + }, + { + "epoch": 11.224114475739407, + "grad_norm": 0.6837960481643677, + "learning_rate": 0.0001, + "loss": 63.906, + "step": 6970 + }, + { + "epoch": 11.24023781931778, + "grad_norm": 0.7050679326057434, + "learning_rate": 0.0001, + "loss": 64.2803, + "step": 6980 + }, + { + "epoch": 11.256361162896155, + "grad_norm": 0.8619478940963745, + "learning_rate": 0.0001, + "loss": 62.8786, + "step": 6990 + }, + { + "epoch": 11.27248450647453, + "grad_norm": 0.9759813547134399, + "learning_rate": 0.0001, + "loss": 65.2299, + "step": 7000 + }, + { + "epoch": 11.288607850052905, + "grad_norm": 0.7584514021873474, + "learning_rate": 0.0001, + "loss": 61.4178, + "step": 7010 + }, + { + "epoch": 11.304731193631278, + "grad_norm": 0.8467874526977539, + "learning_rate": 0.0001, + "loss": 65.6229, + "step": 7020 + }, + { + "epoch": 11.320854537209653, + "grad_norm": 0.7121654152870178, + "learning_rate": 0.0001, + "loss": 63.9061, + "step": 7030 + }, + { + "epoch": 11.336977880788028, + "grad_norm": 0.8885557055473328, + "learning_rate": 0.0001, + "loss": 65.1436, + "step": 7040 + }, + { + "epoch": 11.353101224366403, + "grad_norm": 0.6320177912712097, + "learning_rate": 0.0001, + "loss": 62.8535, + "step": 7050 + }, + { + "epoch": 11.369224567944778, + "grad_norm": 0.9287629723548889, + "learning_rate": 0.0001, + "loss": 62.7407, + "step": 7060 + }, + { + "epoch": 11.385347911523152, + "grad_norm": 0.7705608606338501, + "learning_rate": 0.0001, + "loss": 63.469, + "step": 7070 + }, + { + "epoch": 11.401471255101526, + "grad_norm": 0.6687246561050415, + "learning_rate": 0.0001, + "loss": 62.722, + "step": 7080 + }, + { + "epoch": 11.417594598679901, + "grad_norm": 0.7774648666381836, + "learning_rate": 0.0001, + "loss": 63.1145, + "step": 7090 + }, + { + "epoch": 11.433717942258276, + "grad_norm": 0.7808363437652588, + "learning_rate": 0.0001, + "loss": 63.9342, + "step": 7100 + }, + { + "epoch": 11.44984128583665, + "grad_norm": 0.8033562302589417, + "learning_rate": 0.0001, + "loss": 65.7876, + "step": 7110 + }, + { + "epoch": 11.465964629415025, + "grad_norm": 0.8143486976623535, + "learning_rate": 0.0001, + "loss": 64.8782, + "step": 7120 + }, + { + "epoch": 11.4820879729934, + "grad_norm": 0.7855561971664429, + "learning_rate": 0.0001, + "loss": 63.4944, + "step": 7130 + }, + { + "epoch": 11.498211316571775, + "grad_norm": 0.7525351047515869, + "learning_rate": 0.0001, + "loss": 62.4088, + "step": 7140 + }, + { + "epoch": 11.514334660150148, + "grad_norm": 0.7193494439125061, + "learning_rate": 0.0001, + "loss": 64.2386, + "step": 7150 + }, + { + "epoch": 11.530458003728523, + "grad_norm": 0.892281174659729, + "learning_rate": 0.0001, + "loss": 63.8075, + "step": 7160 + }, + { + "epoch": 11.546581347306898, + "grad_norm": 0.720287561416626, + "learning_rate": 0.0001, + "loss": 63.3259, + "step": 7170 + }, + { + "epoch": 11.562704690885273, + "grad_norm": 1.4404689073562622, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 7180 + }, + { + "epoch": 11.578828034463648, + "grad_norm": 0.9082726240158081, + "learning_rate": 0.0001, + "loss": 64.6877, + "step": 7190 + }, + { + "epoch": 11.59495137804202, + "grad_norm": 0.7137670516967773, + "learning_rate": 0.0001, + "loss": 63.4832, + "step": 7200 + }, + { + "epoch": 11.611074721620396, + "grad_norm": 0.6848503947257996, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 7210 + }, + { + "epoch": 11.62719806519877, + "grad_norm": 0.7483687996864319, + "learning_rate": 0.0001, + "loss": 63.294, + "step": 7220 + }, + { + "epoch": 11.643321408777146, + "grad_norm": 0.935321569442749, + "learning_rate": 0.0001, + "loss": 62.8669, + "step": 7230 + }, + { + "epoch": 11.659444752355519, + "grad_norm": 0.8359326124191284, + "learning_rate": 0.0001, + "loss": 64.4482, + "step": 7240 + }, + { + "epoch": 11.675568095933894, + "grad_norm": 0.7838515043258667, + "learning_rate": 0.0001, + "loss": 64.1975, + "step": 7250 + }, + { + "epoch": 11.691691439512269, + "grad_norm": 0.9128443598747253, + "learning_rate": 0.0001, + "loss": 63.1822, + "step": 7260 + }, + { + "epoch": 11.707814783090644, + "grad_norm": 1.0545419454574585, + "learning_rate": 0.0001, + "loss": 63.3112, + "step": 7270 + }, + { + "epoch": 11.723938126669019, + "grad_norm": 0.8572486639022827, + "learning_rate": 0.0001, + "loss": 64.9605, + "step": 7280 + }, + { + "epoch": 11.740061470247392, + "grad_norm": 1.0894116163253784, + "learning_rate": 0.0001, + "loss": 64.3671, + "step": 7290 + }, + { + "epoch": 11.756184813825767, + "grad_norm": 0.7500355839729309, + "learning_rate": 0.0001, + "loss": 62.6381, + "step": 7300 + }, + { + "epoch": 11.772308157404142, + "grad_norm": 0.7315185070037842, + "learning_rate": 0.0001, + "loss": 66.1438, + "step": 7310 + }, + { + "epoch": 11.788431500982517, + "grad_norm": 0.8123629093170166, + "learning_rate": 0.0001, + "loss": 62.6012, + "step": 7320 + }, + { + "epoch": 11.80455484456089, + "grad_norm": 0.8072763085365295, + "learning_rate": 0.0001, + "loss": 63.4424, + "step": 7330 + }, + { + "epoch": 11.820678188139265, + "grad_norm": 1.1077874898910522, + "learning_rate": 0.0001, + "loss": 63.2812, + "step": 7340 + }, + { + "epoch": 11.83680153171764, + "grad_norm": 0.7116726636886597, + "learning_rate": 0.0001, + "loss": 63.6077, + "step": 7350 + }, + { + "epoch": 11.852924875296015, + "grad_norm": 0.8817790746688843, + "learning_rate": 0.0001, + "loss": 63.1938, + "step": 7360 + }, + { + "epoch": 11.86904821887439, + "grad_norm": 1.021730661392212, + "learning_rate": 0.0001, + "loss": 65.2215, + "step": 7370 + }, + { + "epoch": 11.885171562452763, + "grad_norm": 0.9871250987052917, + "learning_rate": 0.0001, + "loss": 64.6468, + "step": 7380 + }, + { + "epoch": 11.901294906031138, + "grad_norm": 1.3591430187225342, + "learning_rate": 0.0001, + "loss": 61.5546, + "step": 7390 + }, + { + "epoch": 11.917418249609513, + "grad_norm": 0.8114227056503296, + "learning_rate": 0.0001, + "loss": 61.8215, + "step": 7400 + }, + { + "epoch": 11.933541593187888, + "grad_norm": 0.7151288986206055, + "learning_rate": 0.0001, + "loss": 61.4856, + "step": 7410 + }, + { + "epoch": 11.949664936766261, + "grad_norm": 0.7494287490844727, + "learning_rate": 0.0001, + "loss": 63.6451, + "step": 7420 + }, + { + "epoch": 11.965788280344636, + "grad_norm": 0.7377745509147644, + "learning_rate": 0.0001, + "loss": 63.002, + "step": 7430 + }, + { + "epoch": 11.981911623923011, + "grad_norm": 0.6534659266471863, + "learning_rate": 0.0001, + "loss": 63.6404, + "step": 7440 + }, + { + "epoch": 11.998034967501386, + "grad_norm": 0.7548736333847046, + "learning_rate": 0.0001, + "loss": 64.1986, + "step": 7450 + }, + { + "epoch": 12.012898674862699, + "grad_norm": 0.7227553129196167, + "learning_rate": 0.0001, + "loss": 56.8327, + "step": 7460 + }, + { + "epoch": 12.029022018441074, + "grad_norm": 0.6348695158958435, + "learning_rate": 0.0001, + "loss": 62.4472, + "step": 7470 + }, + { + "epoch": 12.045145362019449, + "grad_norm": 0.8249921202659607, + "learning_rate": 0.0001, + "loss": 65.1877, + "step": 7480 + }, + { + "epoch": 12.061268705597824, + "grad_norm": 0.7152777314186096, + "learning_rate": 0.0001, + "loss": 64.2609, + "step": 7490 + }, + { + "epoch": 12.077392049176197, + "grad_norm": 0.7399184703826904, + "learning_rate": 0.0001, + "loss": 63.3997, + "step": 7500 + }, + { + "epoch": 12.093515392754572, + "grad_norm": 0.6972354650497437, + "learning_rate": 0.0001, + "loss": 61.3573, + "step": 7510 + }, + { + "epoch": 12.109638736332947, + "grad_norm": 0.7666720151901245, + "learning_rate": 0.0001, + "loss": 63.8438, + "step": 7520 + }, + { + "epoch": 12.125762079911322, + "grad_norm": 0.7309302687644958, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 7530 + }, + { + "epoch": 12.141885423489697, + "grad_norm": 1.3805177211761475, + "learning_rate": 0.0001, + "loss": 64.68, + "step": 7540 + }, + { + "epoch": 12.15800876706807, + "grad_norm": 1.0315922498703003, + "learning_rate": 0.0001, + "loss": 63.6372, + "step": 7550 + }, + { + "epoch": 12.174132110646445, + "grad_norm": 1.2012155055999756, + "learning_rate": 0.0001, + "loss": 62.5765, + "step": 7560 + }, + { + "epoch": 12.19025545422482, + "grad_norm": 0.7769846320152283, + "learning_rate": 0.0001, + "loss": 63.9737, + "step": 7570 + }, + { + "epoch": 12.206378797803195, + "grad_norm": 0.6587379574775696, + "learning_rate": 0.0001, + "loss": 63.382, + "step": 7580 + }, + { + "epoch": 12.222502141381568, + "grad_norm": 0.8309435248374939, + "learning_rate": 0.0001, + "loss": 64.4027, + "step": 7590 + }, + { + "epoch": 12.238625484959943, + "grad_norm": 0.8221393823623657, + "learning_rate": 0.0001, + "loss": 62.8561, + "step": 7600 + }, + { + "epoch": 12.254748828538318, + "grad_norm": 0.8629311323165894, + "learning_rate": 0.0001, + "loss": 64.3673, + "step": 7610 + }, + { + "epoch": 12.270872172116693, + "grad_norm": 0.698113739490509, + "learning_rate": 0.0001, + "loss": 63.6153, + "step": 7620 + }, + { + "epoch": 12.286995515695068, + "grad_norm": 0.72906094789505, + "learning_rate": 0.0001, + "loss": 64.8439, + "step": 7630 + }, + { + "epoch": 12.303118859273441, + "grad_norm": 0.7402384877204895, + "learning_rate": 0.0001, + "loss": 64.2103, + "step": 7640 + }, + { + "epoch": 12.319242202851816, + "grad_norm": 0.9972009062767029, + "learning_rate": 0.0001, + "loss": 63.803, + "step": 7650 + }, + { + "epoch": 12.335365546430191, + "grad_norm": 0.7852274775505066, + "learning_rate": 0.0001, + "loss": 64.8144, + "step": 7660 + }, + { + "epoch": 12.351488890008566, + "grad_norm": 0.7433444857597351, + "learning_rate": 0.0001, + "loss": 63.0123, + "step": 7670 + }, + { + "epoch": 12.36761223358694, + "grad_norm": 0.8548759818077087, + "learning_rate": 0.0001, + "loss": 63.2215, + "step": 7680 + }, + { + "epoch": 12.383735577165314, + "grad_norm": 0.6759565472602844, + "learning_rate": 0.0001, + "loss": 62.1246, + "step": 7690 + }, + { + "epoch": 12.39985892074369, + "grad_norm": 0.9079536199569702, + "learning_rate": 0.0001, + "loss": 64.1692, + "step": 7700 + }, + { + "epoch": 12.415982264322064, + "grad_norm": 0.6890071034431458, + "learning_rate": 0.0001, + "loss": 62.5602, + "step": 7710 + }, + { + "epoch": 12.432105607900438, + "grad_norm": 0.8096348643302917, + "learning_rate": 0.0001, + "loss": 61.0103, + "step": 7720 + }, + { + "epoch": 12.448228951478812, + "grad_norm": 0.7359873056411743, + "learning_rate": 0.0001, + "loss": 62.1914, + "step": 7730 + }, + { + "epoch": 12.464352295057187, + "grad_norm": 1.0701091289520264, + "learning_rate": 0.0001, + "loss": 61.7295, + "step": 7740 + }, + { + "epoch": 12.480475638635562, + "grad_norm": 0.791928768157959, + "learning_rate": 0.0001, + "loss": 62.6812, + "step": 7750 + }, + { + "epoch": 12.496598982213937, + "grad_norm": 0.7719446420669556, + "learning_rate": 0.0001, + "loss": 61.9114, + "step": 7760 + }, + { + "epoch": 12.51272232579231, + "grad_norm": 0.8720338344573975, + "learning_rate": 0.0001, + "loss": 64.8107, + "step": 7770 + }, + { + "epoch": 12.528845669370686, + "grad_norm": 0.9999920129776001, + "learning_rate": 0.0001, + "loss": 61.198, + "step": 7780 + }, + { + "epoch": 12.54496901294906, + "grad_norm": 0.710754930973053, + "learning_rate": 0.0001, + "loss": 64.1462, + "step": 7790 + }, + { + "epoch": 12.561092356527435, + "grad_norm": 0.7401363849639893, + "learning_rate": 0.0001, + "loss": 61.8988, + "step": 7800 + }, + { + "epoch": 12.577215700105809, + "grad_norm": 0.9183729290962219, + "learning_rate": 0.0001, + "loss": 62.6392, + "step": 7810 + }, + { + "epoch": 12.593339043684184, + "grad_norm": 0.7341399788856506, + "learning_rate": 0.0001, + "loss": 62.7068, + "step": 7820 + }, + { + "epoch": 12.609462387262559, + "grad_norm": 0.8066722750663757, + "learning_rate": 0.0001, + "loss": 64.0625, + "step": 7830 + }, + { + "epoch": 12.625585730840934, + "grad_norm": 0.6088835597038269, + "learning_rate": 0.0001, + "loss": 64.3208, + "step": 7840 + }, + { + "epoch": 12.641709074419307, + "grad_norm": 0.6969302892684937, + "learning_rate": 0.0001, + "loss": 62.5817, + "step": 7850 + }, + { + "epoch": 12.657832417997682, + "grad_norm": 0.82635098695755, + "learning_rate": 0.0001, + "loss": 62.0061, + "step": 7860 + }, + { + "epoch": 12.673955761576057, + "grad_norm": 0.7016922235488892, + "learning_rate": 0.0001, + "loss": 62.5166, + "step": 7870 + }, + { + "epoch": 12.690079105154432, + "grad_norm": 0.8035777807235718, + "learning_rate": 0.0001, + "loss": 63.6543, + "step": 7880 + }, + { + "epoch": 12.706202448732807, + "grad_norm": 0.8573809862136841, + "learning_rate": 0.0001, + "loss": 62.5258, + "step": 7890 + }, + { + "epoch": 12.72232579231118, + "grad_norm": 0.9497472643852234, + "learning_rate": 0.0001, + "loss": 63.1133, + "step": 7900 + }, + { + "epoch": 12.738449135889555, + "grad_norm": 0.7494457960128784, + "learning_rate": 0.0001, + "loss": 63.8157, + "step": 7910 + }, + { + "epoch": 12.75457247946793, + "grad_norm": 0.7657570242881775, + "learning_rate": 0.0001, + "loss": 62.6255, + "step": 7920 + }, + { + "epoch": 12.770695823046305, + "grad_norm": 0.7848380208015442, + "learning_rate": 0.0001, + "loss": 62.9095, + "step": 7930 + }, + { + "epoch": 12.786819166624678, + "grad_norm": 0.8386877775192261, + "learning_rate": 0.0001, + "loss": 63.2254, + "step": 7940 + }, + { + "epoch": 12.802942510203053, + "grad_norm": 0.6774776577949524, + "learning_rate": 0.0001, + "loss": 64.3929, + "step": 7950 + }, + { + "epoch": 12.819065853781428, + "grad_norm": 0.798547625541687, + "learning_rate": 0.0001, + "loss": 62.9884, + "step": 7960 + }, + { + "epoch": 12.835189197359803, + "grad_norm": 0.9340888261795044, + "learning_rate": 0.0001, + "loss": 62.6197, + "step": 7970 + }, + { + "epoch": 12.851312540938178, + "grad_norm": 0.8211432695388794, + "learning_rate": 0.0001, + "loss": 62.7756, + "step": 7980 + }, + { + "epoch": 12.867435884516551, + "grad_norm": 0.7612001299858093, + "learning_rate": 0.0001, + "loss": 62.0792, + "step": 7990 + }, + { + "epoch": 12.883559228094926, + "grad_norm": 0.7743386626243591, + "learning_rate": 0.0001, + "loss": 63.9203, + "step": 8000 + }, + { + "epoch": 12.899682571673301, + "grad_norm": 0.7419672608375549, + "learning_rate": 0.0001, + "loss": 62.1705, + "step": 8010 + }, + { + "epoch": 12.915805915251676, + "grad_norm": 0.6556256413459778, + "learning_rate": 0.0001, + "loss": 64.5863, + "step": 8020 + }, + { + "epoch": 12.93192925883005, + "grad_norm": 1.136175274848938, + "learning_rate": 0.0001, + "loss": 63.2619, + "step": 8030 + }, + { + "epoch": 12.948052602408424, + "grad_norm": 0.8631499409675598, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8040 + }, + { + "epoch": 12.9641759459868, + "grad_norm": 0.8386172652244568, + "learning_rate": 0.0001, + "loss": 63.0598, + "step": 8050 + }, + { + "epoch": 12.980299289565174, + "grad_norm": 0.8078306913375854, + "learning_rate": 0.0001, + "loss": 62.6877, + "step": 8060 + }, + { + "epoch": 12.996422633143549, + "grad_norm": 0.8486189842224121, + "learning_rate": 0.0001, + "loss": 62.8643, + "step": 8070 + }, + { + "epoch": 13.011286340504862, + "grad_norm": 0.8717893958091736, + "learning_rate": 0.0001, + "loss": 57.6947, + "step": 8080 + }, + { + "epoch": 13.027409684083237, + "grad_norm": 1.2023789882659912, + "learning_rate": 0.0001, + "loss": 60.4728, + "step": 8090 + }, + { + "epoch": 13.043533027661612, + "grad_norm": 0.6879557967185974, + "learning_rate": 0.0001, + "loss": 61.7643, + "step": 8100 + }, + { + "epoch": 13.059656371239987, + "grad_norm": 1.0918916463851929, + "learning_rate": 0.0001, + "loss": 65.3682, + "step": 8110 + }, + { + "epoch": 13.07577971481836, + "grad_norm": 0.85753333568573, + "learning_rate": 0.0001, + "loss": 63.0815, + "step": 8120 + }, + { + "epoch": 13.091903058396735, + "grad_norm": 0.6568583846092224, + "learning_rate": 0.0001, + "loss": 64.7167, + "step": 8130 + }, + { + "epoch": 13.10802640197511, + "grad_norm": 0.6178144812583923, + "learning_rate": 0.0001, + "loss": 62.4294, + "step": 8140 + }, + { + "epoch": 13.124149745553485, + "grad_norm": 0.7930784821510315, + "learning_rate": 0.0001, + "loss": 64.542, + "step": 8150 + }, + { + "epoch": 13.140273089131858, + "grad_norm": 0.7675037980079651, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8160 + }, + { + "epoch": 13.156396432710233, + "grad_norm": 0.786997377872467, + "learning_rate": 0.0001, + "loss": 64.1495, + "step": 8170 + }, + { + "epoch": 13.172519776288608, + "grad_norm": 0.7752560377120972, + "learning_rate": 0.0001, + "loss": 61.6483, + "step": 8180 + }, + { + "epoch": 13.188643119866983, + "grad_norm": 0.8455203771591187, + "learning_rate": 0.0001, + "loss": 63.3551, + "step": 8190 + }, + { + "epoch": 13.204766463445356, + "grad_norm": 0.7223097681999207, + "learning_rate": 0.0001, + "loss": 63.1473, + "step": 8200 + }, + { + "epoch": 13.220889807023731, + "grad_norm": 0.7688442468643188, + "learning_rate": 0.0001, + "loss": 62.6723, + "step": 8210 + }, + { + "epoch": 13.237013150602106, + "grad_norm": 0.7730395793914795, + "learning_rate": 0.0001, + "loss": 64.3424, + "step": 8220 + }, + { + "epoch": 13.253136494180481, + "grad_norm": 0.8479727506637573, + "learning_rate": 0.0001, + "loss": 61.4336, + "step": 8230 + }, + { + "epoch": 13.269259837758856, + "grad_norm": 0.9070560932159424, + "learning_rate": 0.0001, + "loss": 63.1221, + "step": 8240 + }, + { + "epoch": 13.28538318133723, + "grad_norm": 0.8924600481987, + "learning_rate": 0.0001, + "loss": 63.9172, + "step": 8250 + }, + { + "epoch": 13.301506524915604, + "grad_norm": 1.0688443183898926, + "learning_rate": 0.0001, + "loss": 63.7944, + "step": 8260 + }, + { + "epoch": 13.31762986849398, + "grad_norm": 0.6866253018379211, + "learning_rate": 0.0001, + "loss": 65.5847, + "step": 8270 + }, + { + "epoch": 13.333753212072354, + "grad_norm": 0.711171567440033, + "learning_rate": 0.0001, + "loss": 62.698, + "step": 8280 + }, + { + "epoch": 13.349876555650727, + "grad_norm": 0.8128448724746704, + "learning_rate": 0.0001, + "loss": 61.5992, + "step": 8290 + }, + { + "epoch": 13.365999899229102, + "grad_norm": 0.7483755350112915, + "learning_rate": 0.0001, + "loss": 63.782, + "step": 8300 + }, + { + "epoch": 13.382123242807477, + "grad_norm": 0.7872599363327026, + "learning_rate": 0.0001, + "loss": 65.4922, + "step": 8310 + }, + { + "epoch": 13.398246586385852, + "grad_norm": 0.6861543655395508, + "learning_rate": 0.0001, + "loss": 62.6946, + "step": 8320 + }, + { + "epoch": 13.414369929964227, + "grad_norm": 0.7192599773406982, + "learning_rate": 0.0001, + "loss": 62.894, + "step": 8330 + }, + { + "epoch": 13.4304932735426, + "grad_norm": 0.7665286064147949, + "learning_rate": 0.0001, + "loss": 62.6253, + "step": 8340 + }, + { + "epoch": 13.446616617120975, + "grad_norm": 0.7909309267997742, + "learning_rate": 0.0001, + "loss": 62.5355, + "step": 8350 + }, + { + "epoch": 13.46273996069935, + "grad_norm": 0.7339541912078857, + "learning_rate": 0.0001, + "loss": 62.8514, + "step": 8360 + }, + { + "epoch": 13.478863304277725, + "grad_norm": 0.9893004298210144, + "learning_rate": 0.0001, + "loss": 63.7751, + "step": 8370 + }, + { + "epoch": 13.494986647856098, + "grad_norm": 0.9109547734260559, + "learning_rate": 0.0001, + "loss": 62.6697, + "step": 8380 + }, + { + "epoch": 13.511109991434473, + "grad_norm": 0.6807034015655518, + "learning_rate": 0.0001, + "loss": 62.1418, + "step": 8390 + }, + { + "epoch": 13.527233335012848, + "grad_norm": 0.915533185005188, + "learning_rate": 0.0001, + "loss": 61.7645, + "step": 8400 + }, + { + "epoch": 13.543356678591223, + "grad_norm": 0.8662852048873901, + "learning_rate": 0.0001, + "loss": 62.5546, + "step": 8410 + }, + { + "epoch": 13.559480022169598, + "grad_norm": 0.8678365349769592, + "learning_rate": 0.0001, + "loss": 63.8795, + "step": 8420 + }, + { + "epoch": 13.575603365747972, + "grad_norm": 0.6808590888977051, + "learning_rate": 0.0001, + "loss": 64.158, + "step": 8430 + }, + { + "epoch": 13.591726709326347, + "grad_norm": 0.7896480560302734, + "learning_rate": 0.0001, + "loss": 62.1052, + "step": 8440 + }, + { + "epoch": 13.607850052904721, + "grad_norm": 0.8973979949951172, + "learning_rate": 0.0001, + "loss": 64.478, + "step": 8450 + }, + { + "epoch": 13.623973396483096, + "grad_norm": 0.8228474259376526, + "learning_rate": 0.0001, + "loss": 61.2416, + "step": 8460 + }, + { + "epoch": 13.64009674006147, + "grad_norm": 0.9414492845535278, + "learning_rate": 0.0001, + "loss": 62.816, + "step": 8470 + }, + { + "epoch": 13.656220083639845, + "grad_norm": 0.9369436502456665, + "learning_rate": 0.0001, + "loss": 63.4951, + "step": 8480 + }, + { + "epoch": 13.67234342721822, + "grad_norm": 0.7152880430221558, + "learning_rate": 0.0001, + "loss": 62.4737, + "step": 8490 + }, + { + "epoch": 13.688466770796595, + "grad_norm": 0.7364594340324402, + "learning_rate": 0.0001, + "loss": 63.8688, + "step": 8500 + }, + { + "epoch": 13.704590114374968, + "grad_norm": 0.8499143123626709, + "learning_rate": 0.0001, + "loss": 64.5654, + "step": 8510 + }, + { + "epoch": 13.720713457953343, + "grad_norm": 0.7699156999588013, + "learning_rate": 0.0001, + "loss": 63.3726, + "step": 8520 + }, + { + "epoch": 13.736836801531718, + "grad_norm": 0.9146804809570312, + "learning_rate": 0.0001, + "loss": 63.6565, + "step": 8530 + }, + { + "epoch": 13.752960145110093, + "grad_norm": 0.773318350315094, + "learning_rate": 0.0001, + "loss": 65.0719, + "step": 8540 + }, + { + "epoch": 13.769083488688466, + "grad_norm": 0.6158450245857239, + "learning_rate": 0.0001, + "loss": 60.7291, + "step": 8550 + }, + { + "epoch": 13.78520683226684, + "grad_norm": 0.8053523302078247, + "learning_rate": 0.0001, + "loss": 62.5622, + "step": 8560 + }, + { + "epoch": 13.801330175845216, + "grad_norm": 0.8983457684516907, + "learning_rate": 0.0001, + "loss": 64.2921, + "step": 8570 + }, + { + "epoch": 13.81745351942359, + "grad_norm": 0.8214695453643799, + "learning_rate": 0.0001, + "loss": 62.2649, + "step": 8580 + }, + { + "epoch": 13.833576863001966, + "grad_norm": 0.8079172372817993, + "learning_rate": 0.0001, + "loss": 64.0828, + "step": 8590 + }, + { + "epoch": 13.849700206580339, + "grad_norm": 1.126568078994751, + "learning_rate": 0.0001, + "loss": 62.3927, + "step": 8600 + }, + { + "epoch": 13.865823550158714, + "grad_norm": 0.7597793936729431, + "learning_rate": 0.0001, + "loss": 62.5922, + "step": 8610 + }, + { + "epoch": 13.881946893737089, + "grad_norm": 0.671531617641449, + "learning_rate": 0.0001, + "loss": 61.9885, + "step": 8620 + }, + { + "epoch": 13.898070237315464, + "grad_norm": 0.823423445224762, + "learning_rate": 0.0001, + "loss": 62.0632, + "step": 8630 + }, + { + "epoch": 13.914193580893837, + "grad_norm": 0.7694658637046814, + "learning_rate": 0.0001, + "loss": 64.5485, + "step": 8640 + }, + { + "epoch": 13.930316924472212, + "grad_norm": 0.7394612431526184, + "learning_rate": 0.0001, + "loss": 61.9626, + "step": 8650 + }, + { + "epoch": 13.946440268050587, + "grad_norm": 0.724054217338562, + "learning_rate": 0.0001, + "loss": 62.3761, + "step": 8660 + }, + { + "epoch": 13.962563611628962, + "grad_norm": 0.7160128951072693, + "learning_rate": 0.0001, + "loss": 60.5142, + "step": 8670 + }, + { + "epoch": 13.978686955207337, + "grad_norm": 1.0045746564865112, + "learning_rate": 0.0001, + "loss": 62.7059, + "step": 8680 + }, + { + "epoch": 13.99481029878571, + "grad_norm": 0.6980776190757751, + "learning_rate": 0.0001, + "loss": 62.9427, + "step": 8690 + }, + { + "epoch": 14.009674006147025, + "grad_norm": 0.810961127281189, + "learning_rate": 0.0001, + "loss": 58.7552, + "step": 8700 + }, + { + "epoch": 14.0257973497254, + "grad_norm": 0.9210551381111145, + "learning_rate": 0.0001, + "loss": 63.7412, + "step": 8710 + }, + { + "epoch": 14.041920693303775, + "grad_norm": 0.7805785536766052, + "learning_rate": 0.0001, + "loss": 61.477, + "step": 8720 + }, + { + "epoch": 14.058044036882148, + "grad_norm": 0.8528208136558533, + "learning_rate": 0.0001, + "loss": 62.6746, + "step": 8730 + }, + { + "epoch": 14.074167380460523, + "grad_norm": 0.7467619776725769, + "learning_rate": 0.0001, + "loss": 63.8744, + "step": 8740 + }, + { + "epoch": 14.090290724038898, + "grad_norm": 0.6434402465820312, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 8750 + }, + { + "epoch": 14.106414067617273, + "grad_norm": 0.7217757701873779, + "learning_rate": 0.0001, + "loss": 64.0428, + "step": 8760 + }, + { + "epoch": 14.122537411195646, + "grad_norm": 0.8665125966072083, + "learning_rate": 0.0001, + "loss": 62.2772, + "step": 8770 + }, + { + "epoch": 14.13866075477402, + "grad_norm": 0.7868321537971497, + "learning_rate": 0.0001, + "loss": 62.0509, + "step": 8780 + }, + { + "epoch": 14.154784098352396, + "grad_norm": 0.8716760873794556, + "learning_rate": 0.0001, + "loss": 61.6021, + "step": 8790 + }, + { + "epoch": 14.17090744193077, + "grad_norm": 1.0922521352767944, + "learning_rate": 0.0001, + "loss": 64.4197, + "step": 8800 + }, + { + "epoch": 14.187030785509146, + "grad_norm": 0.6545577049255371, + "learning_rate": 0.0001, + "loss": 62.9259, + "step": 8810 + }, + { + "epoch": 14.203154129087519, + "grad_norm": 0.7634063959121704, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 8820 + }, + { + "epoch": 14.219277472665894, + "grad_norm": 0.9075300097465515, + "learning_rate": 0.0001, + "loss": 62.9128, + "step": 8830 + }, + { + "epoch": 14.235400816244269, + "grad_norm": 3.475320816040039, + "learning_rate": 0.0001, + "loss": 62.4841, + "step": 8840 + }, + { + "epoch": 14.251524159822644, + "grad_norm": 0.6839455962181091, + "learning_rate": 0.0001, + "loss": 61.5898, + "step": 8850 + }, + { + "epoch": 14.267647503401017, + "grad_norm": 0.8024231791496277, + "learning_rate": 0.0001, + "loss": 61.1875, + "step": 8860 + }, + { + "epoch": 14.283770846979392, + "grad_norm": 0.8948174118995667, + "learning_rate": 0.0001, + "loss": 62.3386, + "step": 8870 + }, + { + "epoch": 14.299894190557767, + "grad_norm": 0.7571985721588135, + "learning_rate": 0.0001, + "loss": 60.579, + "step": 8880 + }, + { + "epoch": 14.316017534136142, + "grad_norm": 1.432371973991394, + "learning_rate": 0.0001, + "loss": 62.4405, + "step": 8890 + }, + { + "epoch": 14.332140877714515, + "grad_norm": 0.6696875691413879, + "learning_rate": 0.0001, + "loss": 62.9984, + "step": 8900 + }, + { + "epoch": 14.34826422129289, + "grad_norm": 0.749396026134491, + "learning_rate": 0.0001, + "loss": 61.492, + "step": 8910 + }, + { + "epoch": 14.364387564871265, + "grad_norm": 0.664193332195282, + "learning_rate": 0.0001, + "loss": 62.9893, + "step": 8920 + }, + { + "epoch": 14.38051090844964, + "grad_norm": 0.7032697796821594, + "learning_rate": 0.0001, + "loss": 61.1682, + "step": 8930 + }, + { + "epoch": 14.396634252028015, + "grad_norm": 0.813119649887085, + "learning_rate": 0.0001, + "loss": 62.906, + "step": 8940 + }, + { + "epoch": 14.412757595606388, + "grad_norm": 0.7699596881866455, + "learning_rate": 0.0001, + "loss": 64.8087, + "step": 8950 + }, + { + "epoch": 14.428880939184763, + "grad_norm": 0.6424354910850525, + "learning_rate": 0.0001, + "loss": 62.2418, + "step": 8960 + }, + { + "epoch": 14.445004282763138, + "grad_norm": 0.8741731643676758, + "learning_rate": 0.0001, + "loss": 62.656, + "step": 8970 + }, + { + "epoch": 14.461127626341513, + "grad_norm": 0.710617184638977, + "learning_rate": 0.0001, + "loss": 60.8979, + "step": 8980 + }, + { + "epoch": 14.477250969919886, + "grad_norm": 0.6493995785713196, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 8990 + }, + { + "epoch": 14.493374313498261, + "grad_norm": 0.6655835509300232, + "learning_rate": 0.0001, + "loss": 64.151, + "step": 9000 + }, + { + "epoch": 14.509497657076636, + "grad_norm": 0.8140881657600403, + "learning_rate": 0.0001, + "loss": 62.4447, + "step": 9010 + }, + { + "epoch": 14.525621000655011, + "grad_norm": 0.7288498878479004, + "learning_rate": 0.0001, + "loss": 62.5189, + "step": 9020 + }, + { + "epoch": 14.541744344233386, + "grad_norm": 0.6972017884254456, + "learning_rate": 0.0001, + "loss": 61.5441, + "step": 9030 + }, + { + "epoch": 14.55786768781176, + "grad_norm": 0.7444366216659546, + "learning_rate": 0.0001, + "loss": 62.3233, + "step": 9040 + }, + { + "epoch": 14.573991031390134, + "grad_norm": 0.6602845191955566, + "learning_rate": 0.0001, + "loss": 61.4176, + "step": 9050 + }, + { + "epoch": 14.59011437496851, + "grad_norm": 0.7149770855903625, + "learning_rate": 0.0001, + "loss": 62.159, + "step": 9060 + }, + { + "epoch": 14.606237718546884, + "grad_norm": 0.6975772976875305, + "learning_rate": 0.0001, + "loss": 63.1857, + "step": 9070 + }, + { + "epoch": 14.622361062125258, + "grad_norm": 0.7835262417793274, + "learning_rate": 0.0001, + "loss": 61.6795, + "step": 9080 + }, + { + "epoch": 14.638484405703633, + "grad_norm": 5.790656566619873, + "learning_rate": 0.0001, + "loss": 62.7087, + "step": 9090 + }, + { + "epoch": 14.654607749282007, + "grad_norm": 1.0082288980484009, + "learning_rate": 0.0001, + "loss": 61.6808, + "step": 9100 + }, + { + "epoch": 14.670731092860382, + "grad_norm": 0.7758098840713501, + "learning_rate": 0.0001, + "loss": 61.0734, + "step": 9110 + }, + { + "epoch": 14.686854436438756, + "grad_norm": 0.7841722965240479, + "learning_rate": 0.0001, + "loss": 62.8882, + "step": 9120 + }, + { + "epoch": 14.70297778001713, + "grad_norm": 0.7695890069007874, + "learning_rate": 0.0001, + "loss": 60.7811, + "step": 9130 + }, + { + "epoch": 14.719101123595506, + "grad_norm": 0.7444761395454407, + "learning_rate": 0.0001, + "loss": 61.692, + "step": 9140 + }, + { + "epoch": 14.73522446717388, + "grad_norm": 0.7036821246147156, + "learning_rate": 0.0001, + "loss": 63.1698, + "step": 9150 + }, + { + "epoch": 14.751347810752256, + "grad_norm": 0.7891719937324524, + "learning_rate": 0.0001, + "loss": 63.8575, + "step": 9160 + }, + { + "epoch": 14.767471154330629, + "grad_norm": 0.6371109485626221, + "learning_rate": 0.0001, + "loss": 61.3358, + "step": 9170 + }, + { + "epoch": 14.783594497909004, + "grad_norm": 0.6551769375801086, + "learning_rate": 0.0001, + "loss": 62.2705, + "step": 9180 + }, + { + "epoch": 14.799717841487379, + "grad_norm": 0.6922310590744019, + "learning_rate": 0.0001, + "loss": 61.6442, + "step": 9190 + }, + { + "epoch": 14.815841185065754, + "grad_norm": 0.9615464210510254, + "learning_rate": 0.0001, + "loss": 60.932, + "step": 9200 + }, + { + "epoch": 14.831964528644127, + "grad_norm": 0.6367932558059692, + "learning_rate": 0.0001, + "loss": 61.3475, + "step": 9210 + }, + { + "epoch": 14.848087872222502, + "grad_norm": 0.5634505748748779, + "learning_rate": 0.0001, + "loss": 61.7562, + "step": 9220 + }, + { + "epoch": 14.864211215800877, + "grad_norm": 0.7021980285644531, + "learning_rate": 0.0001, + "loss": 62.0672, + "step": 9230 + }, + { + "epoch": 14.880334559379252, + "grad_norm": 0.8832064867019653, + "learning_rate": 0.0001, + "loss": 60.9882, + "step": 9240 + }, + { + "epoch": 14.896457902957625, + "grad_norm": 0.9043561220169067, + "learning_rate": 0.0001, + "loss": 63.0968, + "step": 9250 + }, + { + "epoch": 14.912581246536, + "grad_norm": 0.8958032131195068, + "learning_rate": 0.0001, + "loss": 61.5431, + "step": 9260 + }, + { + "epoch": 14.928704590114375, + "grad_norm": 0.7181615829467773, + "learning_rate": 0.0001, + "loss": 62.3626, + "step": 9270 + }, + { + "epoch": 14.94482793369275, + "grad_norm": 0.7541075348854065, + "learning_rate": 0.0001, + "loss": 61.7186, + "step": 9280 + }, + { + "epoch": 14.960951277271125, + "grad_norm": 1.5627169609069824, + "learning_rate": 0.0001, + "loss": 61.654, + "step": 9290 + }, + { + "epoch": 14.977074620849498, + "grad_norm": 0.7126879096031189, + "learning_rate": 0.0001, + "loss": 61.6806, + "step": 9300 + }, + { + "epoch": 14.993197964427873, + "grad_norm": 0.7397864460945129, + "learning_rate": 0.0001, + "loss": 61.2669, + "step": 9310 + }, + { + "epoch": 15.008061671789187, + "grad_norm": 0.7282766699790955, + "learning_rate": 0.0001, + "loss": 56.2289, + "step": 9320 + }, + { + "epoch": 15.024185015367562, + "grad_norm": 0.8346580862998962, + "learning_rate": 0.0001, + "loss": 61.4335, + "step": 9330 + }, + { + "epoch": 15.040308358945936, + "grad_norm": 0.9117271304130554, + "learning_rate": 0.0001, + "loss": 60.4636, + "step": 9340 + }, + { + "epoch": 15.05643170252431, + "grad_norm": 0.8483272194862366, + "learning_rate": 0.0001, + "loss": 61.3003, + "step": 9350 + }, + { + "epoch": 15.072555046102686, + "grad_norm": 1.1898932456970215, + "learning_rate": 0.0001, + "loss": 62.7169, + "step": 9360 + }, + { + "epoch": 15.08867838968106, + "grad_norm": 0.7258167862892151, + "learning_rate": 0.0001, + "loss": 63.5285, + "step": 9370 + }, + { + "epoch": 15.104801733259436, + "grad_norm": 0.7810865044593811, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 9380 + }, + { + "epoch": 15.120925076837809, + "grad_norm": 1.0359688997268677, + "learning_rate": 0.0001, + "loss": 62.5435, + "step": 9390 + }, + { + "epoch": 15.137048420416184, + "grad_norm": 0.7371546030044556, + "learning_rate": 0.0001, + "loss": 60.5583, + "step": 9400 + }, + { + "epoch": 15.153171763994559, + "grad_norm": 0.7578926086425781, + "learning_rate": 0.0001, + "loss": 61.2853, + "step": 9410 + }, + { + "epoch": 15.169295107572934, + "grad_norm": 0.6161713004112244, + "learning_rate": 0.0001, + "loss": 62.9725, + "step": 9420 + }, + { + "epoch": 15.185418451151307, + "grad_norm": 0.8325598835945129, + "learning_rate": 0.0001, + "loss": 62.857, + "step": 9430 + }, + { + "epoch": 15.201541794729682, + "grad_norm": 1.1438454389572144, + "learning_rate": 0.0001, + "loss": 61.8167, + "step": 9440 + }, + { + "epoch": 15.217665138308057, + "grad_norm": 1.1820837259292603, + "learning_rate": 0.0001, + "loss": 60.9899, + "step": 9450 + }, + { + "epoch": 15.233788481886432, + "grad_norm": 0.6057641506195068, + "learning_rate": 0.0001, + "loss": 62.5144, + "step": 9460 + }, + { + "epoch": 15.249911825464805, + "grad_norm": 0.6661948561668396, + "learning_rate": 0.0001, + "loss": 62.0835, + "step": 9470 + }, + { + "epoch": 15.26603516904318, + "grad_norm": 0.7623168230056763, + "learning_rate": 0.0001, + "loss": 61.9491, + "step": 9480 + }, + { + "epoch": 15.282158512621555, + "grad_norm": 0.6784029006958008, + "learning_rate": 0.0001, + "loss": 63.6006, + "step": 9490 + }, + { + "epoch": 15.29828185619993, + "grad_norm": 0.9433781504631042, + "learning_rate": 0.0001, + "loss": 61.6677, + "step": 9500 + }, + { + "epoch": 15.314405199778305, + "grad_norm": 0.6544079184532166, + "learning_rate": 0.0001, + "loss": 63.1439, + "step": 9510 + }, + { + "epoch": 15.330528543356678, + "grad_norm": 0.7979472279548645, + "learning_rate": 0.0001, + "loss": 62.719, + "step": 9520 + }, + { + "epoch": 15.346651886935053, + "grad_norm": 0.6641252040863037, + "learning_rate": 0.0001, + "loss": 63.2543, + "step": 9530 + }, + { + "epoch": 15.362775230513428, + "grad_norm": 0.7780525088310242, + "learning_rate": 0.0001, + "loss": 62.5659, + "step": 9540 + }, + { + "epoch": 15.378898574091803, + "grad_norm": 0.842349648475647, + "learning_rate": 0.0001, + "loss": 60.041, + "step": 9550 + }, + { + "epoch": 15.395021917670176, + "grad_norm": 0.8991017937660217, + "learning_rate": 0.0001, + "loss": 62.9075, + "step": 9560 + }, + { + "epoch": 15.411145261248551, + "grad_norm": 0.6734052300453186, + "learning_rate": 0.0001, + "loss": 60.8647, + "step": 9570 + }, + { + "epoch": 15.427268604826926, + "grad_norm": 0.7585113644599915, + "learning_rate": 0.0001, + "loss": 62.105, + "step": 9580 + }, + { + "epoch": 15.443391948405301, + "grad_norm": 0.7915840148925781, + "learning_rate": 0.0001, + "loss": 62.2741, + "step": 9590 + }, + { + "epoch": 15.459515291983674, + "grad_norm": 0.8676218390464783, + "learning_rate": 0.0001, + "loss": 62.4548, + "step": 9600 + }, + { + "epoch": 15.47563863556205, + "grad_norm": 0.7698650360107422, + "learning_rate": 0.0001, + "loss": 61.3412, + "step": 9610 + }, + { + "epoch": 15.491761979140424, + "grad_norm": 0.6600908637046814, + "learning_rate": 0.0001, + "loss": 61.1561, + "step": 9620 + }, + { + "epoch": 15.5078853227188, + "grad_norm": 0.5995001792907715, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 9630 + }, + { + "epoch": 15.524008666297174, + "grad_norm": 0.7258557677268982, + "learning_rate": 0.0001, + "loss": 62.4659, + "step": 9640 + }, + { + "epoch": 15.540132009875547, + "grad_norm": 0.6562618613243103, + "learning_rate": 0.0001, + "loss": 63.288, + "step": 9650 + }, + { + "epoch": 15.556255353453922, + "grad_norm": 0.9224995374679565, + "learning_rate": 0.0001, + "loss": 61.4861, + "step": 9660 + }, + { + "epoch": 15.572378697032297, + "grad_norm": 0.7053484916687012, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 9670 + }, + { + "epoch": 15.588502040610672, + "grad_norm": 1.095229148864746, + "learning_rate": 0.0001, + "loss": 62.1542, + "step": 9680 + }, + { + "epoch": 15.604625384189045, + "grad_norm": 0.7563759088516235, + "learning_rate": 0.0001, + "loss": 61.1644, + "step": 9690 + }, + { + "epoch": 15.62074872776742, + "grad_norm": 0.6508835554122925, + "learning_rate": 0.0001, + "loss": 60.1374, + "step": 9700 + }, + { + "epoch": 15.636872071345795, + "grad_norm": 0.664799153804779, + "learning_rate": 0.0001, + "loss": 60.2331, + "step": 9710 + }, + { + "epoch": 15.65299541492417, + "grad_norm": 0.6525771617889404, + "learning_rate": 0.0001, + "loss": 60.6142, + "step": 9720 + }, + { + "epoch": 15.669118758502545, + "grad_norm": 0.7395381331443787, + "learning_rate": 0.0001, + "loss": 63.5623, + "step": 9730 + }, + { + "epoch": 15.685242102080919, + "grad_norm": 0.7063724398612976, + "learning_rate": 0.0001, + "loss": 63.7133, + "step": 9740 + }, + { + "epoch": 15.701365445659293, + "grad_norm": 0.7918921113014221, + "learning_rate": 0.0001, + "loss": 62.5957, + "step": 9750 + }, + { + "epoch": 15.717488789237668, + "grad_norm": 0.8135486245155334, + "learning_rate": 0.0001, + "loss": 62.9146, + "step": 9760 + }, + { + "epoch": 15.733612132816043, + "grad_norm": 0.8422091007232666, + "learning_rate": 0.0001, + "loss": 61.1466, + "step": 9770 + }, + { + "epoch": 15.749735476394417, + "grad_norm": 0.6329447627067566, + "learning_rate": 0.0001, + "loss": 61.7247, + "step": 9780 + }, + { + "epoch": 15.765858819972792, + "grad_norm": 1.0627039670944214, + "learning_rate": 0.0001, + "loss": 60.6789, + "step": 9790 + }, + { + "epoch": 15.781982163551167, + "grad_norm": 0.7316949367523193, + "learning_rate": 0.0001, + "loss": 61.9896, + "step": 9800 + }, + { + "epoch": 15.798105507129542, + "grad_norm": 0.6968954205513, + "learning_rate": 0.0001, + "loss": 60.8571, + "step": 9810 + }, + { + "epoch": 15.814228850707915, + "grad_norm": 0.7226986289024353, + "learning_rate": 0.0001, + "loss": 62.2664, + "step": 9820 + }, + { + "epoch": 15.83035219428629, + "grad_norm": 0.6542951464653015, + "learning_rate": 0.0001, + "loss": 61.3316, + "step": 9830 + }, + { + "epoch": 15.846475537864665, + "grad_norm": 0.7662289142608643, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 9840 + }, + { + "epoch": 15.86259888144304, + "grad_norm": 0.7060043215751648, + "learning_rate": 0.0001, + "loss": 62.2647, + "step": 9850 + }, + { + "epoch": 15.878722225021415, + "grad_norm": 0.6948843002319336, + "learning_rate": 0.0001, + "loss": 62.8678, + "step": 9860 + }, + { + "epoch": 15.894845568599788, + "grad_norm": 0.7787327766418457, + "learning_rate": 0.0001, + "loss": 62.9154, + "step": 9870 + }, + { + "epoch": 15.910968912178163, + "grad_norm": 0.7077026963233948, + "learning_rate": 0.0001, + "loss": 62.9537, + "step": 9880 + }, + { + "epoch": 15.927092255756538, + "grad_norm": 0.9957109093666077, + "learning_rate": 0.0001, + "loss": 60.9461, + "step": 9890 + }, + { + "epoch": 15.943215599334913, + "grad_norm": 0.9767161011695862, + "learning_rate": 0.0001, + "loss": 62.5944, + "step": 9900 + }, + { + "epoch": 15.959338942913286, + "grad_norm": 0.6016744375228882, + "learning_rate": 0.0001, + "loss": 62.5889, + "step": 9910 + }, + { + "epoch": 15.975462286491661, + "grad_norm": 0.5464991331100464, + "learning_rate": 0.0001, + "loss": 61.3138, + "step": 9920 + }, + { + "epoch": 15.991585630070036, + "grad_norm": 0.8242359757423401, + "learning_rate": 0.0001, + "loss": 61.8091, + "step": 9930 + }, + { + "epoch": 16.00644933743135, + "grad_norm": 0.7031534314155579, + "learning_rate": 0.0001, + "loss": 57.4847, + "step": 9940 + }, + { + "epoch": 16.022572681009724, + "grad_norm": 0.8429487943649292, + "learning_rate": 0.0001, + "loss": 61.7286, + "step": 9950 + }, + { + "epoch": 16.0386960245881, + "grad_norm": 0.7200850248336792, + "learning_rate": 0.0001, + "loss": 62.6968, + "step": 9960 + }, + { + "epoch": 16.054819368166473, + "grad_norm": 0.7412559390068054, + "learning_rate": 0.0001, + "loss": 62.413, + "step": 9970 + }, + { + "epoch": 16.07094271174485, + "grad_norm": 0.8179170489311218, + "learning_rate": 0.0001, + "loss": 62.7044, + "step": 9980 + }, + { + "epoch": 16.087066055323223, + "grad_norm": 0.6939452886581421, + "learning_rate": 0.0001, + "loss": 61.9062, + "step": 9990 + }, + { + "epoch": 16.1031893989016, + "grad_norm": 0.7145045399665833, + "learning_rate": 0.0001, + "loss": 61.6201, + "step": 10000 + }, + { + "epoch": 16.119312742479973, + "grad_norm": 0.7874398231506348, + "learning_rate": 0.0001, + "loss": 60.4299, + "step": 10010 + }, + { + "epoch": 16.135436086058345, + "grad_norm": 0.6640581488609314, + "learning_rate": 0.0001, + "loss": 62.0195, + "step": 10020 + }, + { + "epoch": 16.15155942963672, + "grad_norm": 0.7903703451156616, + "learning_rate": 0.0001, + "loss": 62.501, + "step": 10030 + }, + { + "epoch": 16.167682773215095, + "grad_norm": 0.6669818162918091, + "learning_rate": 0.0001, + "loss": 59.1489, + "step": 10040 + }, + { + "epoch": 16.18380611679347, + "grad_norm": 0.8380771279335022, + "learning_rate": 0.0001, + "loss": 61.0985, + "step": 10050 + }, + { + "epoch": 16.199929460371845, + "grad_norm": 0.7925304174423218, + "learning_rate": 0.0001, + "loss": 62.9251, + "step": 10060 + }, + { + "epoch": 16.21605280395022, + "grad_norm": 1.2176182270050049, + "learning_rate": 0.0001, + "loss": 61.6772, + "step": 10070 + }, + { + "epoch": 16.232176147528595, + "grad_norm": 0.828171968460083, + "learning_rate": 0.0001, + "loss": 61.3296, + "step": 10080 + }, + { + "epoch": 16.24829949110697, + "grad_norm": 0.6884972453117371, + "learning_rate": 0.0001, + "loss": 61.5371, + "step": 10090 + }, + { + "epoch": 16.264422834685345, + "grad_norm": 0.6455051302909851, + "learning_rate": 0.0001, + "loss": 60.6548, + "step": 10100 + }, + { + "epoch": 16.280546178263716, + "grad_norm": 0.687982976436615, + "learning_rate": 0.0001, + "loss": 62.0357, + "step": 10110 + }, + { + "epoch": 16.29666952184209, + "grad_norm": 0.811744213104248, + "learning_rate": 0.0001, + "loss": 60.5664, + "step": 10120 + }, + { + "epoch": 16.312792865420466, + "grad_norm": 0.7140675187110901, + "learning_rate": 0.0001, + "loss": 62.9491, + "step": 10130 + }, + { + "epoch": 16.32891620899884, + "grad_norm": 0.9068045020103455, + "learning_rate": 0.0001, + "loss": 61.118, + "step": 10140 + }, + { + "epoch": 16.345039552577216, + "grad_norm": 0.9028809666633606, + "learning_rate": 0.0001, + "loss": 61.4308, + "step": 10150 + }, + { + "epoch": 16.36116289615559, + "grad_norm": 0.7502166628837585, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10160 + }, + { + "epoch": 16.377286239733966, + "grad_norm": 0.807391345500946, + "learning_rate": 0.0001, + "loss": 61.6825, + "step": 10170 + }, + { + "epoch": 16.39340958331234, + "grad_norm": 0.762725830078125, + "learning_rate": 0.0001, + "loss": 62.7879, + "step": 10180 + }, + { + "epoch": 16.409532926890712, + "grad_norm": 0.7973660230636597, + "learning_rate": 0.0001, + "loss": 61.0727, + "step": 10190 + }, + { + "epoch": 16.425656270469087, + "grad_norm": 1.004622459411621, + "learning_rate": 0.0001, + "loss": 59.8048, + "step": 10200 + }, + { + "epoch": 16.441779614047462, + "grad_norm": 0.8243395686149597, + "learning_rate": 0.0001, + "loss": 63.9312, + "step": 10210 + }, + { + "epoch": 16.457902957625837, + "grad_norm": 0.7006281614303589, + "learning_rate": 0.0001, + "loss": 61.3766, + "step": 10220 + }, + { + "epoch": 16.474026301204212, + "grad_norm": 0.8672197461128235, + "learning_rate": 0.0001, + "loss": 61.6659, + "step": 10230 + }, + { + "epoch": 16.490149644782587, + "grad_norm": 0.7836477160453796, + "learning_rate": 0.0001, + "loss": 61.3686, + "step": 10240 + }, + { + "epoch": 16.506272988360962, + "grad_norm": 0.5842607617378235, + "learning_rate": 0.0001, + "loss": 62.616, + "step": 10250 + }, + { + "epoch": 16.522396331939337, + "grad_norm": 0.6777102947235107, + "learning_rate": 0.0001, + "loss": 62.1135, + "step": 10260 + }, + { + "epoch": 16.538519675517712, + "grad_norm": 0.856697678565979, + "learning_rate": 0.0001, + "loss": 61.849, + "step": 10270 + }, + { + "epoch": 16.554643019096083, + "grad_norm": 0.7300705313682556, + "learning_rate": 0.0001, + "loss": 60.7678, + "step": 10280 + }, + { + "epoch": 16.57076636267446, + "grad_norm": 0.799802839756012, + "learning_rate": 0.0001, + "loss": 62.4888, + "step": 10290 + }, + { + "epoch": 16.586889706252833, + "grad_norm": 0.791251540184021, + "learning_rate": 0.0001, + "loss": 61.7589, + "step": 10300 + }, + { + "epoch": 16.60301304983121, + "grad_norm": 0.6498433947563171, + "learning_rate": 0.0001, + "loss": 61.087, + "step": 10310 + }, + { + "epoch": 16.619136393409583, + "grad_norm": 0.702925443649292, + "learning_rate": 0.0001, + "loss": 59.3493, + "step": 10320 + }, + { + "epoch": 16.63525973698796, + "grad_norm": 0.8508023619651794, + "learning_rate": 0.0001, + "loss": 61.2188, + "step": 10330 + }, + { + "epoch": 16.651383080566333, + "grad_norm": 0.8325870633125305, + "learning_rate": 0.0001, + "loss": 62.9164, + "step": 10340 + }, + { + "epoch": 16.667506424144708, + "grad_norm": 0.7140557765960693, + "learning_rate": 0.0001, + "loss": 62.5838, + "step": 10350 + }, + { + "epoch": 16.683629767723083, + "grad_norm": 0.5574952363967896, + "learning_rate": 0.0001, + "loss": 59.767, + "step": 10360 + }, + { + "epoch": 16.699753111301455, + "grad_norm": 0.8022965788841248, + "learning_rate": 0.0001, + "loss": 60.2198, + "step": 10370 + }, + { + "epoch": 16.71587645487983, + "grad_norm": 1.1291698217391968, + "learning_rate": 0.0001, + "loss": 61.4908, + "step": 10380 + }, + { + "epoch": 16.731999798458205, + "grad_norm": 0.8651410937309265, + "learning_rate": 0.0001, + "loss": 60.5228, + "step": 10390 + }, + { + "epoch": 16.74812314203658, + "grad_norm": 0.8426535129547119, + "learning_rate": 0.0001, + "loss": 62.3786, + "step": 10400 + }, + { + "epoch": 16.764246485614954, + "grad_norm": 0.6329453587532043, + "learning_rate": 0.0001, + "loss": 60.9621, + "step": 10410 + }, + { + "epoch": 16.78036982919333, + "grad_norm": 0.6745129227638245, + "learning_rate": 0.0001, + "loss": 59.3405, + "step": 10420 + }, + { + "epoch": 16.796493172771704, + "grad_norm": 0.8483593463897705, + "learning_rate": 0.0001, + "loss": 61.1527, + "step": 10430 + }, + { + "epoch": 16.81261651635008, + "grad_norm": 0.8649617433547974, + "learning_rate": 0.0001, + "loss": 61.051, + "step": 10440 + }, + { + "epoch": 16.828739859928454, + "grad_norm": 0.7309728264808655, + "learning_rate": 0.0001, + "loss": 60.3626, + "step": 10450 + }, + { + "epoch": 16.844863203506826, + "grad_norm": 0.6467884182929993, + "learning_rate": 0.0001, + "loss": 63.8254, + "step": 10460 + }, + { + "epoch": 16.8609865470852, + "grad_norm": 0.7628102898597717, + "learning_rate": 0.0001, + "loss": 59.5604, + "step": 10470 + }, + { + "epoch": 16.877109890663576, + "grad_norm": 0.78266841173172, + "learning_rate": 0.0001, + "loss": 60.9387, + "step": 10480 + }, + { + "epoch": 16.89323323424195, + "grad_norm": 0.6936920285224915, + "learning_rate": 0.0001, + "loss": 62.5492, + "step": 10490 + }, + { + "epoch": 16.909356577820326, + "grad_norm": 0.8343939185142517, + "learning_rate": 0.0001, + "loss": 60.9975, + "step": 10500 + }, + { + "epoch": 16.9254799213987, + "grad_norm": 0.798984169960022, + "learning_rate": 0.0001, + "loss": 59.5835, + "step": 10510 + }, + { + "epoch": 16.941603264977076, + "grad_norm": 1.369275450706482, + "learning_rate": 0.0001, + "loss": 61.749, + "step": 10520 + }, + { + "epoch": 16.95772660855545, + "grad_norm": 0.5993408560752869, + "learning_rate": 0.0001, + "loss": 62.1093, + "step": 10530 + }, + { + "epoch": 16.973849952133826, + "grad_norm": 0.8153544664382935, + "learning_rate": 0.0001, + "loss": 61.857, + "step": 10540 + }, + { + "epoch": 16.989973295712197, + "grad_norm": 0.7042689919471741, + "learning_rate": 0.0001, + "loss": 60.5178, + "step": 10550 + }, + { + "epoch": 17.00483700307351, + "grad_norm": 1.1931812763214111, + "learning_rate": 0.0001, + "loss": 56.4815, + "step": 10560 + }, + { + "epoch": 17.020960346651886, + "grad_norm": 0.9051854610443115, + "learning_rate": 0.0001, + "loss": 60.7729, + "step": 10570 + }, + { + "epoch": 17.03708369023026, + "grad_norm": 1.093518614768982, + "learning_rate": 0.0001, + "loss": 60.904, + "step": 10580 + }, + { + "epoch": 17.053207033808636, + "grad_norm": 0.7545284032821655, + "learning_rate": 0.0001, + "loss": 59.7165, + "step": 10590 + }, + { + "epoch": 17.06933037738701, + "grad_norm": 0.8542641401290894, + "learning_rate": 0.0001, + "loss": 60.8947, + "step": 10600 + }, + { + "epoch": 17.085453720965386, + "grad_norm": 0.7393799424171448, + "learning_rate": 0.0001, + "loss": 62.8384, + "step": 10610 + }, + { + "epoch": 17.10157706454376, + "grad_norm": 0.6756435632705688, + "learning_rate": 0.0001, + "loss": 61.164, + "step": 10620 + }, + { + "epoch": 17.117700408122133, + "grad_norm": 0.725150465965271, + "learning_rate": 0.0001, + "loss": 60.3789, + "step": 10630 + }, + { + "epoch": 17.133823751700508, + "grad_norm": 1.1529417037963867, + "learning_rate": 0.0001, + "loss": 62.2465, + "step": 10640 + }, + { + "epoch": 17.149947095278883, + "grad_norm": 0.8266693353652954, + "learning_rate": 0.0001, + "loss": 61.6755, + "step": 10650 + }, + { + "epoch": 17.166070438857258, + "grad_norm": 0.7444040775299072, + "learning_rate": 0.0001, + "loss": 62.7084, + "step": 10660 + }, + { + "epoch": 17.182193782435633, + "grad_norm": 0.8164665102958679, + "learning_rate": 0.0001, + "loss": 61.6032, + "step": 10670 + }, + { + "epoch": 17.198317126014008, + "grad_norm": 0.8215296268463135, + "learning_rate": 0.0001, + "loss": 61.033, + "step": 10680 + }, + { + "epoch": 17.214440469592382, + "grad_norm": 0.7939956188201904, + "learning_rate": 0.0001, + "loss": 60.5016, + "step": 10690 + }, + { + "epoch": 17.230563813170757, + "grad_norm": 0.9533108472824097, + "learning_rate": 0.0001, + "loss": 62.6995, + "step": 10700 + }, + { + "epoch": 17.246687156749132, + "grad_norm": 0.6602820754051208, + "learning_rate": 0.0001, + "loss": 61.2709, + "step": 10710 + }, + { + "epoch": 17.262810500327504, + "grad_norm": 0.8369201421737671, + "learning_rate": 0.0001, + "loss": 63.1138, + "step": 10720 + }, + { + "epoch": 17.27893384390588, + "grad_norm": 0.7431164979934692, + "learning_rate": 0.0001, + "loss": 62.8588, + "step": 10730 + }, + { + "epoch": 17.295057187484254, + "grad_norm": 0.7957271337509155, + "learning_rate": 0.0001, + "loss": 60.5119, + "step": 10740 + }, + { + "epoch": 17.31118053106263, + "grad_norm": 0.9407948851585388, + "learning_rate": 0.0001, + "loss": 61.1877, + "step": 10750 + }, + { + "epoch": 17.327303874641004, + "grad_norm": 0.6641581654548645, + "learning_rate": 0.0001, + "loss": 60.542, + "step": 10760 + }, + { + "epoch": 17.34342721821938, + "grad_norm": 0.7786923050880432, + "learning_rate": 0.0001, + "loss": 62.1582, + "step": 10770 + }, + { + "epoch": 17.359550561797754, + "grad_norm": 0.8150411248207092, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10780 + }, + { + "epoch": 17.37567390537613, + "grad_norm": 0.8315487504005432, + "learning_rate": 0.0001, + "loss": 60.4382, + "step": 10790 + }, + { + "epoch": 17.391797248954504, + "grad_norm": 0.8305006623268127, + "learning_rate": 0.0001, + "loss": 61.3262, + "step": 10800 + }, + { + "epoch": 17.407920592532875, + "grad_norm": 0.897731363773346, + "learning_rate": 0.0001, + "loss": 60.7809, + "step": 10810 + }, + { + "epoch": 17.42404393611125, + "grad_norm": 0.7461619973182678, + "learning_rate": 0.0001, + "loss": 61.3858, + "step": 10820 + }, + { + "epoch": 17.440167279689625, + "grad_norm": 0.6750472187995911, + "learning_rate": 0.0001, + "loss": 60.2692, + "step": 10830 + }, + { + "epoch": 17.456290623268, + "grad_norm": 0.5412933230400085, + "learning_rate": 0.0001, + "loss": 61.7872, + "step": 10840 + }, + { + "epoch": 17.472413966846375, + "grad_norm": 0.6485120058059692, + "learning_rate": 0.0001, + "loss": 59.5407, + "step": 10850 + }, + { + "epoch": 17.48853731042475, + "grad_norm": 0.818708598613739, + "learning_rate": 0.0001, + "loss": 61.8066, + "step": 10860 + }, + { + "epoch": 17.504660654003125, + "grad_norm": 0.7121092081069946, + "learning_rate": 0.0001, + "loss": 61.8086, + "step": 10870 + }, + { + "epoch": 17.5207839975815, + "grad_norm": 0.7031684517860413, + "learning_rate": 0.0001, + "loss": 60.0706, + "step": 10880 + }, + { + "epoch": 17.536907341159875, + "grad_norm": 0.6259323358535767, + "learning_rate": 0.0001, + "loss": 61.9625, + "step": 10890 + }, + { + "epoch": 17.553030684738246, + "grad_norm": 0.8526529669761658, + "learning_rate": 0.0001, + "loss": 61.3299, + "step": 10900 + }, + { + "epoch": 17.56915402831662, + "grad_norm": 0.6860859990119934, + "learning_rate": 0.0001, + "loss": 61.3193, + "step": 10910 + }, + { + "epoch": 17.585277371894996, + "grad_norm": 0.7715994119644165, + "learning_rate": 0.0001, + "loss": 61.9111, + "step": 10920 + }, + { + "epoch": 17.60140071547337, + "grad_norm": 0.7171186804771423, + "learning_rate": 0.0001, + "loss": 60.3638, + "step": 10930 + }, + { + "epoch": 17.617524059051746, + "grad_norm": 0.7796649932861328, + "learning_rate": 0.0001, + "loss": 61.9437, + "step": 10940 + }, + { + "epoch": 17.63364740263012, + "grad_norm": 0.7930101752281189, + "learning_rate": 0.0001, + "loss": 60.1085, + "step": 10950 + }, + { + "epoch": 17.649770746208496, + "grad_norm": 0.6370466351509094, + "learning_rate": 0.0001, + "loss": 61.8433, + "step": 10960 + }, + { + "epoch": 17.66589408978687, + "grad_norm": 0.7830466628074646, + "learning_rate": 0.0001, + "loss": 61.3403, + "step": 10970 + }, + { + "epoch": 17.682017433365242, + "grad_norm": 0.873456597328186, + "learning_rate": 0.0001, + "loss": 62.4567, + "step": 10980 + }, + { + "epoch": 17.698140776943617, + "grad_norm": 0.7560754418373108, + "learning_rate": 0.0001, + "loss": 60.9808, + "step": 10990 + }, + { + "epoch": 17.714264120521992, + "grad_norm": 0.6832230091094971, + "learning_rate": 0.0001, + "loss": 61.5059, + "step": 11000 + }, + { + "epoch": 17.730387464100367, + "grad_norm": 0.7701742649078369, + "learning_rate": 0.0001, + "loss": 62.4963, + "step": 11010 + }, + { + "epoch": 17.746510807678742, + "grad_norm": 0.7088269591331482, + "learning_rate": 0.0001, + "loss": 62.6467, + "step": 11020 + }, + { + "epoch": 17.762634151257117, + "grad_norm": 0.8424612879753113, + "learning_rate": 0.0001, + "loss": 60.1213, + "step": 11030 + }, + { + "epoch": 17.778757494835492, + "grad_norm": 0.9974421858787537, + "learning_rate": 0.0001, + "loss": 61.6051, + "step": 11040 + }, + { + "epoch": 17.794880838413867, + "grad_norm": 0.701970636844635, + "learning_rate": 0.0001, + "loss": 62.4155, + "step": 11050 + }, + { + "epoch": 17.811004181992242, + "grad_norm": 0.6826842427253723, + "learning_rate": 0.0001, + "loss": 60.2127, + "step": 11060 + }, + { + "epoch": 17.827127525570614, + "grad_norm": 0.7019745707511902, + "learning_rate": 0.0001, + "loss": 60.9281, + "step": 11070 + }, + { + "epoch": 17.84325086914899, + "grad_norm": 0.836280345916748, + "learning_rate": 0.0001, + "loss": 61.1217, + "step": 11080 + }, + { + "epoch": 17.859374212727364, + "grad_norm": 0.6405923366546631, + "learning_rate": 0.0001, + "loss": 61.6393, + "step": 11090 + }, + { + "epoch": 17.87549755630574, + "grad_norm": 0.6809528470039368, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 11100 + }, + { + "epoch": 17.891620899884114, + "grad_norm": 0.6983410120010376, + "learning_rate": 0.0001, + "loss": 59.933, + "step": 11110 + }, + { + "epoch": 17.90774424346249, + "grad_norm": 1.0240110158920288, + "learning_rate": 0.0001, + "loss": 61.1964, + "step": 11120 + }, + { + "epoch": 17.923867587040863, + "grad_norm": 0.860657811164856, + "learning_rate": 0.0001, + "loss": 61.1613, + "step": 11130 + }, + { + "epoch": 17.93999093061924, + "grad_norm": 0.6434107422828674, + "learning_rate": 0.0001, + "loss": 60.2435, + "step": 11140 + }, + { + "epoch": 17.956114274197613, + "grad_norm": 0.7400848269462585, + "learning_rate": 0.0001, + "loss": 62.452, + "step": 11150 + }, + { + "epoch": 17.972237617775985, + "grad_norm": 0.6526095867156982, + "learning_rate": 0.0001, + "loss": 62.3344, + "step": 11160 + }, + { + "epoch": 17.98836096135436, + "grad_norm": 0.6971097588539124, + "learning_rate": 0.0001, + "loss": 60.2415, + "step": 11170 + }, + { + "epoch": 18.003224668715674, + "grad_norm": 0.724881649017334, + "learning_rate": 0.0001, + "loss": 56.0513, + "step": 11180 + }, + { + "epoch": 18.01934801229405, + "grad_norm": 0.653710663318634, + "learning_rate": 0.0001, + "loss": 60.8364, + "step": 11190 + }, + { + "epoch": 18.035471355872424, + "grad_norm": 0.7356674075126648, + "learning_rate": 0.0001, + "loss": 61.6681, + "step": 11200 + }, + { + "epoch": 18.0515946994508, + "grad_norm": 0.6726879477500916, + "learning_rate": 0.0001, + "loss": 60.6682, + "step": 11210 + }, + { + "epoch": 18.067718043029174, + "grad_norm": 0.7367722392082214, + "learning_rate": 0.0001, + "loss": 63.0462, + "step": 11220 + }, + { + "epoch": 18.08384138660755, + "grad_norm": 0.6687121987342834, + "learning_rate": 0.0001, + "loss": 60.8707, + "step": 11230 + }, + { + "epoch": 18.09996473018592, + "grad_norm": 1.0027060508728027, + "learning_rate": 0.0001, + "loss": 62.4703, + "step": 11240 + }, + { + "epoch": 18.116088073764296, + "grad_norm": 1.0447720289230347, + "learning_rate": 0.0001, + "loss": 62.1746, + "step": 11250 + }, + { + "epoch": 18.13221141734267, + "grad_norm": 0.7687053084373474, + "learning_rate": 0.0001, + "loss": 60.1413, + "step": 11260 + }, + { + "epoch": 18.148334760921045, + "grad_norm": 0.7250781059265137, + "learning_rate": 0.0001, + "loss": 60.1143, + "step": 11270 + }, + { + "epoch": 18.16445810449942, + "grad_norm": 0.7356208562850952, + "learning_rate": 0.0001, + "loss": 62.0788, + "step": 11280 + }, + { + "epoch": 18.180581448077795, + "grad_norm": 0.7838225364685059, + "learning_rate": 0.0001, + "loss": 61.7046, + "step": 11290 + }, + { + "epoch": 18.19670479165617, + "grad_norm": 0.7600042819976807, + "learning_rate": 0.0001, + "loss": 61.4646, + "step": 11300 + }, + { + "epoch": 18.212828135234545, + "grad_norm": 0.7339571714401245, + "learning_rate": 0.0001, + "loss": 61.0983, + "step": 11310 + }, + { + "epoch": 18.22895147881292, + "grad_norm": 0.6982674598693848, + "learning_rate": 0.0001, + "loss": 60.2853, + "step": 11320 + }, + { + "epoch": 18.24507482239129, + "grad_norm": 0.7572301030158997, + "learning_rate": 0.0001, + "loss": 61.8967, + "step": 11330 + }, + { + "epoch": 18.261198165969667, + "grad_norm": 0.6206921935081482, + "learning_rate": 0.0001, + "loss": 62.1467, + "step": 11340 + }, + { + "epoch": 18.27732150954804, + "grad_norm": 0.658437967300415, + "learning_rate": 0.0001, + "loss": 61.2117, + "step": 11350 + }, + { + "epoch": 18.293444853126417, + "grad_norm": 0.7880764603614807, + "learning_rate": 0.0001, + "loss": 61.6246, + "step": 11360 + }, + { + "epoch": 18.30956819670479, + "grad_norm": 0.6601800918579102, + "learning_rate": 0.0001, + "loss": 61.4818, + "step": 11370 + }, + { + "epoch": 18.325691540283167, + "grad_norm": 0.6816290616989136, + "learning_rate": 0.0001, + "loss": 60.4834, + "step": 11380 + }, + { + "epoch": 18.34181488386154, + "grad_norm": 0.7667511105537415, + "learning_rate": 0.0001, + "loss": 60.9106, + "step": 11390 + }, + { + "epoch": 18.357938227439917, + "grad_norm": 0.6490039825439453, + "learning_rate": 0.0001, + "loss": 59.6968, + "step": 11400 + }, + { + "epoch": 18.37406157101829, + "grad_norm": 0.6865658760070801, + "learning_rate": 0.0001, + "loss": 61.7898, + "step": 11410 + }, + { + "epoch": 18.390184914596663, + "grad_norm": 0.8605737090110779, + "learning_rate": 0.0001, + "loss": 60.924, + "step": 11420 + }, + { + "epoch": 18.406308258175038, + "grad_norm": 1.0474110841751099, + "learning_rate": 0.0001, + "loss": 61.0381, + "step": 11430 + }, + { + "epoch": 18.422431601753413, + "grad_norm": 0.9241340160369873, + "learning_rate": 0.0001, + "loss": 61.2609, + "step": 11440 + }, + { + "epoch": 18.438554945331788, + "grad_norm": 1.1441621780395508, + "learning_rate": 0.0001, + "loss": 60.5495, + "step": 11450 + }, + { + "epoch": 18.454678288910163, + "grad_norm": 0.7277412414550781, + "learning_rate": 0.0001, + "loss": 60.4654, + "step": 11460 + }, + { + "epoch": 18.470801632488538, + "grad_norm": 0.6668717861175537, + "learning_rate": 0.0001, + "loss": 60.057, + "step": 11470 + }, + { + "epoch": 18.486924976066913, + "grad_norm": 0.7335819005966187, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 11480 + }, + { + "epoch": 18.503048319645288, + "grad_norm": 0.7261807918548584, + "learning_rate": 0.0001, + "loss": 61.1508, + "step": 11490 + }, + { + "epoch": 18.519171663223663, + "grad_norm": 0.7715283036231995, + "learning_rate": 0.0001, + "loss": 61.5492, + "step": 11500 + }, + { + "epoch": 18.535295006802034, + "grad_norm": 0.9603212475776672, + "learning_rate": 0.0001, + "loss": 60.4321, + "step": 11510 + }, + { + "epoch": 18.55141835038041, + "grad_norm": 0.6130046844482422, + "learning_rate": 0.0001, + "loss": 61.2942, + "step": 11520 + }, + { + "epoch": 18.567541693958784, + "grad_norm": 0.8147348165512085, + "learning_rate": 0.0001, + "loss": 61.4313, + "step": 11530 + }, + { + "epoch": 18.58366503753716, + "grad_norm": 0.7909948825836182, + "learning_rate": 0.0001, + "loss": 60.3547, + "step": 11540 + }, + { + "epoch": 18.599788381115534, + "grad_norm": 0.6470443606376648, + "learning_rate": 0.0001, + "loss": 61.4144, + "step": 11550 + }, + { + "epoch": 18.61591172469391, + "grad_norm": 0.7132186889648438, + "learning_rate": 0.0001, + "loss": 61.8102, + "step": 11560 + }, + { + "epoch": 18.632035068272284, + "grad_norm": 0.7054323554039001, + "learning_rate": 0.0001, + "loss": 62.1431, + "step": 11570 + }, + { + "epoch": 18.64815841185066, + "grad_norm": 0.8784761428833008, + "learning_rate": 0.0001, + "loss": 61.6653, + "step": 11580 + }, + { + "epoch": 18.66428175542903, + "grad_norm": 0.7143248319625854, + "learning_rate": 0.0001, + "loss": 60.1275, + "step": 11590 + }, + { + "epoch": 18.680405099007405, + "grad_norm": 7.87012243270874, + "learning_rate": 0.0001, + "loss": 61.3484, + "step": 11600 + }, + { + "epoch": 18.69652844258578, + "grad_norm": 0.7263020873069763, + "learning_rate": 0.0001, + "loss": 59.906, + "step": 11610 + }, + { + "epoch": 18.712651786164155, + "grad_norm": 0.6468373537063599, + "learning_rate": 0.0001, + "loss": 61.8781, + "step": 11620 + }, + { + "epoch": 18.72877512974253, + "grad_norm": 3.34307861328125, + "learning_rate": 0.0001, + "loss": 61.0841, + "step": 11630 + }, + { + "epoch": 18.744898473320905, + "grad_norm": 0.7348340153694153, + "learning_rate": 0.0001, + "loss": 61.3201, + "step": 11640 + }, + { + "epoch": 18.76102181689928, + "grad_norm": 0.7646427154541016, + "learning_rate": 0.0001, + "loss": 60.1858, + "step": 11650 + }, + { + "epoch": 18.777145160477655, + "grad_norm": 0.6659495234489441, + "learning_rate": 0.0001, + "loss": 60.8999, + "step": 11660 + }, + { + "epoch": 18.79326850405603, + "grad_norm": 0.9913583993911743, + "learning_rate": 0.0001, + "loss": 59.0732, + "step": 11670 + }, + { + "epoch": 18.8093918476344, + "grad_norm": 0.7480027079582214, + "learning_rate": 0.0001, + "loss": 58.6846, + "step": 11680 + }, + { + "epoch": 18.825515191212777, + "grad_norm": 0.8000630736351013, + "learning_rate": 0.0001, + "loss": 62.8883, + "step": 11690 + }, + { + "epoch": 18.84163853479115, + "grad_norm": 0.8450509309768677, + "learning_rate": 0.0001, + "loss": 63.4962, + "step": 11700 + }, + { + "epoch": 18.857761878369526, + "grad_norm": 0.8365609645843506, + "learning_rate": 0.0001, + "loss": 62.2701, + "step": 11710 + }, + { + "epoch": 18.8738852219479, + "grad_norm": 0.7197144031524658, + "learning_rate": 0.0001, + "loss": 61.3158, + "step": 11720 + }, + { + "epoch": 18.890008565526276, + "grad_norm": 0.6460379958152771, + "learning_rate": 0.0001, + "loss": 62.6645, + "step": 11730 + }, + { + "epoch": 18.90613190910465, + "grad_norm": 0.8203967213630676, + "learning_rate": 0.0001, + "loss": 62.3257, + "step": 11740 + }, + { + "epoch": 18.922255252683026, + "grad_norm": 0.682134211063385, + "learning_rate": 0.0001, + "loss": 59.9144, + "step": 11750 + }, + { + "epoch": 18.9383785962614, + "grad_norm": 0.7080821394920349, + "learning_rate": 0.0001, + "loss": 61.4325, + "step": 11760 + }, + { + "epoch": 18.954501939839773, + "grad_norm": 0.7963712811470032, + "learning_rate": 0.0001, + "loss": 58.6136, + "step": 11770 + }, + { + "epoch": 18.970625283418148, + "grad_norm": 0.7317046523094177, + "learning_rate": 0.0001, + "loss": 61.3507, + "step": 11780 + }, + { + "epoch": 18.986748626996523, + "grad_norm": 0.6586548686027527, + "learning_rate": 0.0001, + "loss": 61.4898, + "step": 11790 + }, + { + "epoch": 19.001612334357837, + "grad_norm": 0.9010424613952637, + "learning_rate": 0.0001, + "loss": 56.8915, + "step": 11800 + }, + { + "epoch": 19.017735677936212, + "grad_norm": 0.7102377414703369, + "learning_rate": 0.0001, + "loss": 62.2097, + "step": 11810 + }, + { + "epoch": 19.033859021514587, + "grad_norm": 0.7959222793579102, + "learning_rate": 0.0001, + "loss": 61.4484, + "step": 11820 + }, + { + "epoch": 19.049982365092962, + "grad_norm": 0.6483829617500305, + "learning_rate": 0.0001, + "loss": 60.3563, + "step": 11830 + }, + { + "epoch": 19.066105708671337, + "grad_norm": 1.1301988363265991, + "learning_rate": 0.0001, + "loss": 61.3698, + "step": 11840 + }, + { + "epoch": 19.082229052249712, + "grad_norm": 0.7304167151451111, + "learning_rate": 0.0001, + "loss": 59.2096, + "step": 11850 + }, + { + "epoch": 19.098352395828083, + "grad_norm": 0.7757825255393982, + "learning_rate": 0.0001, + "loss": 59.914, + "step": 11860 + }, + { + "epoch": 19.11447573940646, + "grad_norm": 0.7009251713752747, + "learning_rate": 0.0001, + "loss": 61.0768, + "step": 11870 + }, + { + "epoch": 19.130599082984833, + "grad_norm": 0.6015574336051941, + "learning_rate": 0.0001, + "loss": 61.2698, + "step": 11880 + }, + { + "epoch": 19.14672242656321, + "grad_norm": 0.8028033971786499, + "learning_rate": 0.0001, + "loss": 61.422, + "step": 11890 + }, + { + "epoch": 19.162845770141583, + "grad_norm": 0.5753505825996399, + "learning_rate": 0.0001, + "loss": 61.5066, + "step": 11900 + }, + { + "epoch": 19.17896911371996, + "grad_norm": 0.7702978849411011, + "learning_rate": 0.0001, + "loss": 60.0737, + "step": 11910 + }, + { + "epoch": 19.195092457298333, + "grad_norm": 0.6469365358352661, + "learning_rate": 0.0001, + "loss": 59.5091, + "step": 11920 + }, + { + "epoch": 19.211215800876708, + "grad_norm": 0.8400216698646545, + "learning_rate": 0.0001, + "loss": 62.3739, + "step": 11930 + }, + { + "epoch": 19.22733914445508, + "grad_norm": 1.9919443130493164, + "learning_rate": 0.0001, + "loss": 59.7271, + "step": 11940 + }, + { + "epoch": 19.243462488033455, + "grad_norm": 1.0247725248336792, + "learning_rate": 0.0001, + "loss": 59.3804, + "step": 11950 + }, + { + "epoch": 19.25958583161183, + "grad_norm": 1.0717105865478516, + "learning_rate": 0.0001, + "loss": 61.3617, + "step": 11960 + }, + { + "epoch": 19.275709175190205, + "grad_norm": 0.6324111223220825, + "learning_rate": 0.0001, + "loss": 59.6094, + "step": 11970 + }, + { + "epoch": 19.29183251876858, + "grad_norm": 0.6665195226669312, + "learning_rate": 0.0001, + "loss": 61.5879, + "step": 11980 + }, + { + "epoch": 19.307955862346954, + "grad_norm": 0.7933892011642456, + "learning_rate": 0.0001, + "loss": 60.4011, + "step": 11990 + }, + { + "epoch": 19.32407920592533, + "grad_norm": 0.8135427236557007, + "learning_rate": 0.0001, + "loss": 61.2936, + "step": 12000 + }, + { + "epoch": 19.340202549503704, + "grad_norm": 0.7154449820518494, + "learning_rate": 0.0001, + "loss": 62.3996, + "step": 12010 + }, + { + "epoch": 19.35632589308208, + "grad_norm": 0.9926077127456665, + "learning_rate": 0.0001, + "loss": 60.5219, + "step": 12020 + }, + { + "epoch": 19.37244923666045, + "grad_norm": 1.013094425201416, + "learning_rate": 0.0001, + "loss": 59.9494, + "step": 12030 + }, + { + "epoch": 19.388572580238826, + "grad_norm": 0.984582245349884, + "learning_rate": 0.0001, + "loss": 59.7877, + "step": 12040 + }, + { + "epoch": 19.4046959238172, + "grad_norm": 1.256519079208374, + "learning_rate": 0.0001, + "loss": 60.2608, + "step": 12050 + }, + { + "epoch": 19.420819267395576, + "grad_norm": 0.7013972997665405, + "learning_rate": 0.0001, + "loss": 63.4573, + "step": 12060 + }, + { + "epoch": 19.43694261097395, + "grad_norm": 0.9027557373046875, + "learning_rate": 0.0001, + "loss": 61.3252, + "step": 12070 + }, + { + "epoch": 19.453065954552326, + "grad_norm": 0.9057058691978455, + "learning_rate": 0.0001, + "loss": 60.4992, + "step": 12080 + }, + { + "epoch": 19.4691892981307, + "grad_norm": 0.6828131079673767, + "learning_rate": 0.0001, + "loss": 61.5369, + "step": 12090 + }, + { + "epoch": 19.485312641709076, + "grad_norm": 0.617858350276947, + "learning_rate": 0.0001, + "loss": 61.4837, + "step": 12100 + }, + { + "epoch": 19.50143598528745, + "grad_norm": 0.7722935676574707, + "learning_rate": 0.0001, + "loss": 61.0886, + "step": 12110 + }, + { + "epoch": 19.517559328865822, + "grad_norm": 0.7039631009101868, + "learning_rate": 0.0001, + "loss": 59.9055, + "step": 12120 + }, + { + "epoch": 19.533682672444197, + "grad_norm": 0.8353239297866821, + "learning_rate": 0.0001, + "loss": 62.0864, + "step": 12130 + }, + { + "epoch": 19.549806016022572, + "grad_norm": 0.764098048210144, + "learning_rate": 0.0001, + "loss": 58.9591, + "step": 12140 + }, + { + "epoch": 19.565929359600947, + "grad_norm": 0.7842984795570374, + "learning_rate": 0.0001, + "loss": 59.0494, + "step": 12150 + }, + { + "epoch": 19.582052703179322, + "grad_norm": 0.7140147686004639, + "learning_rate": 0.0001, + "loss": 60.8067, + "step": 12160 + }, + { + "epoch": 19.598176046757697, + "grad_norm": 0.8182031512260437, + "learning_rate": 0.0001, + "loss": 59.6013, + "step": 12170 + }, + { + "epoch": 19.614299390336072, + "grad_norm": 0.7428567409515381, + "learning_rate": 0.0001, + "loss": 61.8644, + "step": 12180 + }, + { + "epoch": 19.630422733914447, + "grad_norm": 0.6174675822257996, + "learning_rate": 0.0001, + "loss": 60.2086, + "step": 12190 + }, + { + "epoch": 19.646546077492822, + "grad_norm": 0.6416913866996765, + "learning_rate": 0.0001, + "loss": 61.3452, + "step": 12200 + }, + { + "epoch": 19.662669421071193, + "grad_norm": 1.3036755323410034, + "learning_rate": 0.0001, + "loss": 60.5332, + "step": 12210 + }, + { + "epoch": 19.678792764649568, + "grad_norm": 1.230710506439209, + "learning_rate": 0.0001, + "loss": 62.5671, + "step": 12220 + }, + { + "epoch": 19.694916108227943, + "grad_norm": 0.6914290189743042, + "learning_rate": 0.0001, + "loss": 60.7302, + "step": 12230 + }, + { + "epoch": 19.711039451806318, + "grad_norm": 0.6492108106613159, + "learning_rate": 0.0001, + "loss": 60.6465, + "step": 12240 + }, + { + "epoch": 19.727162795384693, + "grad_norm": 0.7104936838150024, + "learning_rate": 0.0001, + "loss": 61.14, + "step": 12250 + }, + { + "epoch": 19.743286138963068, + "grad_norm": 1.0740787982940674, + "learning_rate": 0.0001, + "loss": 60.1977, + "step": 12260 + }, + { + "epoch": 19.759409482541443, + "grad_norm": 0.5764663815498352, + "learning_rate": 0.0001, + "loss": 59.994, + "step": 12270 + }, + { + "epoch": 19.775532826119818, + "grad_norm": 0.6204820871353149, + "learning_rate": 0.0001, + "loss": 60.3091, + "step": 12280 + }, + { + "epoch": 19.791656169698193, + "grad_norm": 0.702944815158844, + "learning_rate": 0.0001, + "loss": 60.8637, + "step": 12290 + }, + { + "epoch": 19.807779513276564, + "grad_norm": 0.7591354250907898, + "learning_rate": 0.0001, + "loss": 59.54, + "step": 12300 + }, + { + "epoch": 19.82390285685494, + "grad_norm": 0.755224883556366, + "learning_rate": 0.0001, + "loss": 60.0285, + "step": 12310 + }, + { + "epoch": 19.840026200433314, + "grad_norm": 0.7563093900680542, + "learning_rate": 0.0001, + "loss": 60.2669, + "step": 12320 + }, + { + "epoch": 19.85614954401169, + "grad_norm": 0.8339024782180786, + "learning_rate": 0.0001, + "loss": 60.606, + "step": 12330 + }, + { + "epoch": 19.872272887590064, + "grad_norm": 0.7017751932144165, + "learning_rate": 0.0001, + "loss": 59.579, + "step": 12340 + }, + { + "epoch": 19.88839623116844, + "grad_norm": 1.3561909198760986, + "learning_rate": 0.0001, + "loss": 59.7727, + "step": 12350 + }, + { + "epoch": 19.904519574746814, + "grad_norm": 0.727020263671875, + "learning_rate": 0.0001, + "loss": 60.6587, + "step": 12360 + }, + { + "epoch": 19.92064291832519, + "grad_norm": 0.6882230043411255, + "learning_rate": 0.0001, + "loss": 60.3916, + "step": 12370 + }, + { + "epoch": 19.93676626190356, + "grad_norm": 0.6410917639732361, + "learning_rate": 0.0001, + "loss": 61.6649, + "step": 12380 + }, + { + "epoch": 19.952889605481936, + "grad_norm": 0.9747995138168335, + "learning_rate": 0.0001, + "loss": 60.4622, + "step": 12390 + }, + { + "epoch": 19.96901294906031, + "grad_norm": 1.260621190071106, + "learning_rate": 0.0001, + "loss": 60.4228, + "step": 12400 + }, + { + "epoch": 19.985136292638686, + "grad_norm": 0.6525571942329407, + "learning_rate": 0.0001, + "loss": 59.0612, + "step": 12410 + }, + { + "epoch": 20.0, + "grad_norm": 0.7864645719528198, + "learning_rate": 0.0001, + "loss": 57.2347, + "step": 12420 + }, + { + "epoch": 20.016123343578375, + "grad_norm": 0.7479318380355835, + "learning_rate": 0.0001, + "loss": 60.9263, + "step": 12430 + }, + { + "epoch": 20.03224668715675, + "grad_norm": 0.6433089375495911, + "learning_rate": 0.0001, + "loss": 59.8846, + "step": 12440 + }, + { + "epoch": 20.048370030735125, + "grad_norm": 0.7594776153564453, + "learning_rate": 0.0001, + "loss": 61.7793, + "step": 12450 + }, + { + "epoch": 20.0644933743135, + "grad_norm": 0.8710535764694214, + "learning_rate": 0.0001, + "loss": 60.5331, + "step": 12460 + }, + { + "epoch": 20.08061671789187, + "grad_norm": 0.7302114963531494, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 12470 + }, + { + "epoch": 20.096740061470246, + "grad_norm": 0.6039112210273743, + "learning_rate": 0.0001, + "loss": 60.8272, + "step": 12480 + }, + { + "epoch": 20.11286340504862, + "grad_norm": 0.7339144945144653, + "learning_rate": 0.0001, + "loss": 61.4149, + "step": 12490 + }, + { + "epoch": 20.128986748626996, + "grad_norm": 0.7278878092765808, + "learning_rate": 0.0001, + "loss": 62.631, + "step": 12500 + }, + { + "epoch": 20.14511009220537, + "grad_norm": 0.7241142988204956, + "learning_rate": 0.0001, + "loss": 59.9328, + "step": 12510 + }, + { + "epoch": 20.161233435783746, + "grad_norm": 0.6967542767524719, + "learning_rate": 0.0001, + "loss": 58.7463, + "step": 12520 + }, + { + "epoch": 20.17735677936212, + "grad_norm": 0.7326555848121643, + "learning_rate": 0.0001, + "loss": 59.7082, + "step": 12530 + }, + { + "epoch": 20.193480122940496, + "grad_norm": 0.6706270575523376, + "learning_rate": 0.0001, + "loss": 60.5978, + "step": 12540 + }, + { + "epoch": 20.20960346651887, + "grad_norm": 0.8541543483734131, + "learning_rate": 0.0001, + "loss": 58.9632, + "step": 12550 + }, + { + "epoch": 20.225726810097242, + "grad_norm": 0.6599560379981995, + "learning_rate": 0.0001, + "loss": 58.4572, + "step": 12560 + }, + { + "epoch": 20.241850153675617, + "grad_norm": 0.6668654680252075, + "learning_rate": 0.0001, + "loss": 61.8741, + "step": 12570 + }, + { + "epoch": 20.257973497253992, + "grad_norm": 0.6453808546066284, + "learning_rate": 0.0001, + "loss": 60.2538, + "step": 12580 + }, + { + "epoch": 20.274096840832367, + "grad_norm": 0.7429870963096619, + "learning_rate": 0.0001, + "loss": 60.5795, + "step": 12590 + }, + { + "epoch": 20.290220184410742, + "grad_norm": 0.7588251829147339, + "learning_rate": 0.0001, + "loss": 60.6261, + "step": 12600 + }, + { + "epoch": 20.306343527989117, + "grad_norm": 0.6430977582931519, + "learning_rate": 0.0001, + "loss": 61.0344, + "step": 12610 + }, + { + "epoch": 20.322466871567492, + "grad_norm": 0.6879712343215942, + "learning_rate": 0.0001, + "loss": 62.2668, + "step": 12620 + }, + { + "epoch": 20.338590215145867, + "grad_norm": 0.7650917172431946, + "learning_rate": 0.0001, + "loss": 60.3852, + "step": 12630 + }, + { + "epoch": 20.35471355872424, + "grad_norm": 0.5326449871063232, + "learning_rate": 0.0001, + "loss": 60.6138, + "step": 12640 + }, + { + "epoch": 20.370836902302614, + "grad_norm": 0.7018648982048035, + "learning_rate": 0.0001, + "loss": 62.1081, + "step": 12650 + }, + { + "epoch": 20.38696024588099, + "grad_norm": 0.7988570332527161, + "learning_rate": 0.0001, + "loss": 61.2661, + "step": 12660 + }, + { + "epoch": 20.403083589459364, + "grad_norm": 0.6800012588500977, + "learning_rate": 0.0001, + "loss": 59.9279, + "step": 12670 + }, + { + "epoch": 20.41920693303774, + "grad_norm": 0.6380598545074463, + "learning_rate": 0.0001, + "loss": 60.4877, + "step": 12680 + }, + { + "epoch": 20.435330276616114, + "grad_norm": 0.6755974888801575, + "learning_rate": 0.0001, + "loss": 60.9729, + "step": 12690 + }, + { + "epoch": 20.45145362019449, + "grad_norm": 0.8057810664176941, + "learning_rate": 0.0001, + "loss": 59.6747, + "step": 12700 + }, + { + "epoch": 20.467576963772864, + "grad_norm": 0.6942827105522156, + "learning_rate": 0.0001, + "loss": 60.7314, + "step": 12710 + }, + { + "epoch": 20.48370030735124, + "grad_norm": 1.107717752456665, + "learning_rate": 0.0001, + "loss": 59.1397, + "step": 12720 + }, + { + "epoch": 20.49982365092961, + "grad_norm": 2.6167991161346436, + "learning_rate": 0.0001, + "loss": 60.5604, + "step": 12730 + }, + { + "epoch": 20.515946994507985, + "grad_norm": 0.6041843891143799, + "learning_rate": 0.0001, + "loss": 60.6188, + "step": 12740 + }, + { + "epoch": 20.53207033808636, + "grad_norm": 0.5643273591995239, + "learning_rate": 0.0001, + "loss": 59.3172, + "step": 12750 + }, + { + "epoch": 20.548193681664735, + "grad_norm": 0.7615086436271667, + "learning_rate": 0.0001, + "loss": 60.9263, + "step": 12760 + }, + { + "epoch": 20.56431702524311, + "grad_norm": 0.692243754863739, + "learning_rate": 0.0001, + "loss": 58.3092, + "step": 12770 + }, + { + "epoch": 20.580440368821485, + "grad_norm": 0.6447681784629822, + "learning_rate": 0.0001, + "loss": 58.0169, + "step": 12780 + }, + { + "epoch": 20.59656371239986, + "grad_norm": 0.7043688297271729, + "learning_rate": 0.0001, + "loss": 58.8346, + "step": 12790 + }, + { + "epoch": 20.612687055978235, + "grad_norm": 0.8017414808273315, + "learning_rate": 0.0001, + "loss": 60.7451, + "step": 12800 + }, + { + "epoch": 20.62881039955661, + "grad_norm": 0.8983294367790222, + "learning_rate": 0.0001, + "loss": 59.6858, + "step": 12810 + }, + { + "epoch": 20.64493374313498, + "grad_norm": 1.0817819833755493, + "learning_rate": 0.0001, + "loss": 60.8232, + "step": 12820 + }, + { + "epoch": 20.661057086713356, + "grad_norm": 0.5739386081695557, + "learning_rate": 0.0001, + "loss": 60.7852, + "step": 12830 + }, + { + "epoch": 20.67718043029173, + "grad_norm": 1.007978916168213, + "learning_rate": 0.0001, + "loss": 61.7002, + "step": 12840 + }, + { + "epoch": 20.693303773870106, + "grad_norm": 0.7019670009613037, + "learning_rate": 0.0001, + "loss": 59.7211, + "step": 12850 + }, + { + "epoch": 20.70942711744848, + "grad_norm": 0.7891802191734314, + "learning_rate": 0.0001, + "loss": 61.4963, + "step": 12860 + }, + { + "epoch": 20.725550461026856, + "grad_norm": 0.6955318450927734, + "learning_rate": 0.0001, + "loss": 60.8528, + "step": 12870 + }, + { + "epoch": 20.74167380460523, + "grad_norm": 1.9988878965377808, + "learning_rate": 0.0001, + "loss": 58.7444, + "step": 12880 + }, + { + "epoch": 20.757797148183606, + "grad_norm": 0.7859933972358704, + "learning_rate": 0.0001, + "loss": 60.0696, + "step": 12890 + }, + { + "epoch": 20.77392049176198, + "grad_norm": 0.8253535628318787, + "learning_rate": 0.0001, + "loss": 61.3408, + "step": 12900 + }, + { + "epoch": 20.790043835340352, + "grad_norm": 0.6639575362205505, + "learning_rate": 0.0001, + "loss": 58.3044, + "step": 12910 + }, + { + "epoch": 20.806167178918727, + "grad_norm": 0.6754361391067505, + "learning_rate": 0.0001, + "loss": 59.8301, + "step": 12920 + }, + { + "epoch": 20.822290522497102, + "grad_norm": 0.7306725382804871, + "learning_rate": 0.0001, + "loss": 60.0021, + "step": 12930 + }, + { + "epoch": 20.838413866075477, + "grad_norm": 6.799670696258545, + "learning_rate": 0.0001, + "loss": 57.7934, + "step": 12940 + }, + { + "epoch": 20.854537209653852, + "grad_norm": 0.6870545744895935, + "learning_rate": 0.0001, + "loss": 59.2313, + "step": 12950 + }, + { + "epoch": 20.870660553232227, + "grad_norm": 0.6980645060539246, + "learning_rate": 0.0001, + "loss": 60.1903, + "step": 12960 + }, + { + "epoch": 20.886783896810602, + "grad_norm": 0.7006897926330566, + "learning_rate": 0.0001, + "loss": 60.9525, + "step": 12970 + }, + { + "epoch": 20.902907240388977, + "grad_norm": 0.7185907363891602, + "learning_rate": 0.0001, + "loss": 61.8833, + "step": 12980 + }, + { + "epoch": 20.91903058396735, + "grad_norm": 0.6029082536697388, + "learning_rate": 0.0001, + "loss": 60.3764, + "step": 12990 + }, + { + "epoch": 20.935153927545723, + "grad_norm": 0.8502999544143677, + "learning_rate": 0.0001, + "loss": 59.0051, + "step": 13000 + }, + { + "epoch": 20.9512772711241, + "grad_norm": 0.908899188041687, + "learning_rate": 0.0001, + "loss": 59.0677, + "step": 13010 + }, + { + "epoch": 20.967400614702473, + "grad_norm": 0.6983920335769653, + "learning_rate": 0.0001, + "loss": 60.8523, + "step": 13020 + }, + { + "epoch": 20.98352395828085, + "grad_norm": 0.9079043865203857, + "learning_rate": 0.0001, + "loss": 58.8014, + "step": 13030 + }, + { + "epoch": 20.999647301859223, + "grad_norm": 0.7139968872070312, + "learning_rate": 0.0001, + "loss": 59.3019, + "step": 13040 + }, + { + "epoch": 21.014511009220538, + "grad_norm": 0.7472447156906128, + "learning_rate": 0.0001, + "loss": 56.1178, + "step": 13050 + }, + { + "epoch": 21.030634352798913, + "grad_norm": 0.5645695328712463, + "learning_rate": 0.0001, + "loss": 60.5944, + "step": 13060 + }, + { + "epoch": 21.046757696377288, + "grad_norm": 0.6419529318809509, + "learning_rate": 0.0001, + "loss": 59.5792, + "step": 13070 + }, + { + "epoch": 21.06288103995566, + "grad_norm": 0.6508122682571411, + "learning_rate": 0.0001, + "loss": 58.995, + "step": 13080 + }, + { + "epoch": 21.079004383534034, + "grad_norm": 0.6605037450790405, + "learning_rate": 0.0001, + "loss": 60.9783, + "step": 13090 + }, + { + "epoch": 21.09512772711241, + "grad_norm": 0.6636376976966858, + "learning_rate": 0.0001, + "loss": 60.3516, + "step": 13100 + }, + { + "epoch": 21.111251070690784, + "grad_norm": 0.7671985030174255, + "learning_rate": 0.0001, + "loss": 61.8918, + "step": 13110 + }, + { + "epoch": 21.12737441426916, + "grad_norm": 0.859834611415863, + "learning_rate": 0.0001, + "loss": 61.0766, + "step": 13120 + }, + { + "epoch": 21.143497757847534, + "grad_norm": 0.7551945447921753, + "learning_rate": 0.0001, + "loss": 60.6674, + "step": 13130 + }, + { + "epoch": 21.15962110142591, + "grad_norm": 0.6225672960281372, + "learning_rate": 0.0001, + "loss": 60.4107, + "step": 13140 + }, + { + "epoch": 21.175744445004284, + "grad_norm": 0.6100388765335083, + "learning_rate": 0.0001, + "loss": 58.8803, + "step": 13150 + }, + { + "epoch": 21.19186778858266, + "grad_norm": 0.669247567653656, + "learning_rate": 0.0001, + "loss": 60.4534, + "step": 13160 + }, + { + "epoch": 21.20799113216103, + "grad_norm": 0.8899680972099304, + "learning_rate": 0.0001, + "loss": 59.3081, + "step": 13170 + }, + { + "epoch": 21.224114475739405, + "grad_norm": 0.6804965138435364, + "learning_rate": 0.0001, + "loss": 58.1406, + "step": 13180 + }, + { + "epoch": 21.24023781931778, + "grad_norm": 0.7442401051521301, + "learning_rate": 0.0001, + "loss": 61.9136, + "step": 13190 + }, + { + "epoch": 21.256361162896155, + "grad_norm": 0.6071797013282776, + "learning_rate": 0.0001, + "loss": 59.7979, + "step": 13200 + }, + { + "epoch": 21.27248450647453, + "grad_norm": 0.6128804683685303, + "learning_rate": 0.0001, + "loss": 58.802, + "step": 13210 + }, + { + "epoch": 21.288607850052905, + "grad_norm": 0.6430484652519226, + "learning_rate": 0.0001, + "loss": 59.9492, + "step": 13220 + }, + { + "epoch": 21.30473119363128, + "grad_norm": 0.6414241194725037, + "learning_rate": 0.0001, + "loss": 60.699, + "step": 13230 + }, + { + "epoch": 21.320854537209655, + "grad_norm": 0.7222250699996948, + "learning_rate": 0.0001, + "loss": 58.5909, + "step": 13240 + }, + { + "epoch": 21.33697788078803, + "grad_norm": 1.0913405418395996, + "learning_rate": 0.0001, + "loss": 59.9657, + "step": 13250 + }, + { + "epoch": 21.3531012243664, + "grad_norm": 0.8245854377746582, + "learning_rate": 0.0001, + "loss": 60.2276, + "step": 13260 + }, + { + "epoch": 21.369224567944777, + "grad_norm": 0.7204747200012207, + "learning_rate": 0.0001, + "loss": 60.1854, + "step": 13270 + }, + { + "epoch": 21.38534791152315, + "grad_norm": 0.5877585411071777, + "learning_rate": 0.0001, + "loss": 60.8509, + "step": 13280 + }, + { + "epoch": 21.401471255101526, + "grad_norm": 0.906605064868927, + "learning_rate": 0.0001, + "loss": 57.9792, + "step": 13290 + }, + { + "epoch": 21.4175945986799, + "grad_norm": 0.910301923751831, + "learning_rate": 0.0001, + "loss": 58.986, + "step": 13300 + }, + { + "epoch": 21.433717942258276, + "grad_norm": 0.7098278403282166, + "learning_rate": 0.0001, + "loss": 60.6727, + "step": 13310 + }, + { + "epoch": 21.44984128583665, + "grad_norm": 0.842569887638092, + "learning_rate": 0.0001, + "loss": 59.3789, + "step": 13320 + }, + { + "epoch": 21.465964629415026, + "grad_norm": 0.7320014834403992, + "learning_rate": 0.0001, + "loss": 60.7055, + "step": 13330 + }, + { + "epoch": 21.482087972993398, + "grad_norm": 0.7192698121070862, + "learning_rate": 0.0001, + "loss": 59.7494, + "step": 13340 + }, + { + "epoch": 21.498211316571773, + "grad_norm": 0.6317388415336609, + "learning_rate": 0.0001, + "loss": 59.2083, + "step": 13350 + }, + { + "epoch": 21.514334660150148, + "grad_norm": 0.7012689709663391, + "learning_rate": 0.0001, + "loss": 58.6733, + "step": 13360 + }, + { + "epoch": 21.530458003728523, + "grad_norm": 0.8148412704467773, + "learning_rate": 0.0001, + "loss": 60.6708, + "step": 13370 + }, + { + "epoch": 21.546581347306898, + "grad_norm": 0.6740885376930237, + "learning_rate": 0.0001, + "loss": 60.2317, + "step": 13380 + }, + { + "epoch": 21.562704690885273, + "grad_norm": 0.6835291981697083, + "learning_rate": 0.0001, + "loss": 60.1807, + "step": 13390 + }, + { + "epoch": 21.578828034463648, + "grad_norm": 0.9604494571685791, + "learning_rate": 0.0001, + "loss": 60.811, + "step": 13400 + }, + { + "epoch": 21.594951378042023, + "grad_norm": 0.765917956829071, + "learning_rate": 0.0001, + "loss": 60.4539, + "step": 13410 + }, + { + "epoch": 21.611074721620398, + "grad_norm": 0.7830843329429626, + "learning_rate": 0.0001, + "loss": 59.2483, + "step": 13420 + }, + { + "epoch": 21.62719806519877, + "grad_norm": 0.6538785099983215, + "learning_rate": 0.0001, + "loss": 60.4062, + "step": 13430 + }, + { + "epoch": 21.643321408777144, + "grad_norm": 0.7341951727867126, + "learning_rate": 0.0001, + "loss": 59.3654, + "step": 13440 + }, + { + "epoch": 21.65944475235552, + "grad_norm": 0.684467077255249, + "learning_rate": 0.0001, + "loss": 58.9634, + "step": 13450 + }, + { + "epoch": 21.675568095933894, + "grad_norm": 0.6942781805992126, + "learning_rate": 0.0001, + "loss": 60.3677, + "step": 13460 + }, + { + "epoch": 21.69169143951227, + "grad_norm": 0.7780037522315979, + "learning_rate": 0.0001, + "loss": 61.6705, + "step": 13470 + }, + { + "epoch": 21.707814783090644, + "grad_norm": 0.8363171219825745, + "learning_rate": 0.0001, + "loss": 59.7802, + "step": 13480 + }, + { + "epoch": 21.72393812666902, + "grad_norm": 0.7392274737358093, + "learning_rate": 0.0001, + "loss": 59.8264, + "step": 13490 + }, + { + "epoch": 21.740061470247394, + "grad_norm": 0.6051466464996338, + "learning_rate": 0.0001, + "loss": 59.6685, + "step": 13500 + }, + { + "epoch": 21.75618481382577, + "grad_norm": 0.7772730588912964, + "learning_rate": 0.0001, + "loss": 58.261, + "step": 13510 + }, + { + "epoch": 21.77230815740414, + "grad_norm": 0.7481459379196167, + "learning_rate": 0.0001, + "loss": 59.3212, + "step": 13520 + }, + { + "epoch": 21.788431500982515, + "grad_norm": 0.8020729422569275, + "learning_rate": 0.0001, + "loss": 59.0113, + "step": 13530 + }, + { + "epoch": 21.80455484456089, + "grad_norm": 0.8504058122634888, + "learning_rate": 0.0001, + "loss": 59.7243, + "step": 13540 + }, + { + "epoch": 21.820678188139265, + "grad_norm": 0.7256322503089905, + "learning_rate": 0.0001, + "loss": 60.7478, + "step": 13550 + }, + { + "epoch": 21.83680153171764, + "grad_norm": 0.6632773876190186, + "learning_rate": 0.0001, + "loss": 59.2331, + "step": 13560 + }, + { + "epoch": 21.852924875296015, + "grad_norm": 0.6225188970565796, + "learning_rate": 0.0001, + "loss": 59.8263, + "step": 13570 + }, + { + "epoch": 21.86904821887439, + "grad_norm": 0.6302409768104553, + "learning_rate": 0.0001, + "loss": 60.8574, + "step": 13580 + }, + { + "epoch": 21.885171562452765, + "grad_norm": 0.7654368877410889, + "learning_rate": 0.0001, + "loss": 59.8633, + "step": 13590 + }, + { + "epoch": 21.90129490603114, + "grad_norm": 1.227220892906189, + "learning_rate": 0.0001, + "loss": 60.7098, + "step": 13600 + }, + { + "epoch": 21.91741824960951, + "grad_norm": 0.6679421067237854, + "learning_rate": 0.0001, + "loss": 59.0663, + "step": 13610 + }, + { + "epoch": 21.933541593187886, + "grad_norm": 0.8533850312232971, + "learning_rate": 0.0001, + "loss": 60.8597, + "step": 13620 + }, + { + "epoch": 21.94966493676626, + "grad_norm": 0.6417121887207031, + "learning_rate": 0.0001, + "loss": 60.1464, + "step": 13630 + }, + { + "epoch": 21.965788280344636, + "grad_norm": 0.679053544998169, + "learning_rate": 0.0001, + "loss": 60.5066, + "step": 13640 + }, + { + "epoch": 21.98191162392301, + "grad_norm": 0.6914607286453247, + "learning_rate": 0.0001, + "loss": 59.6815, + "step": 13650 + }, + { + "epoch": 21.998034967501386, + "grad_norm": 0.5661413669586182, + "learning_rate": 0.0001, + "loss": 61.6076, + "step": 13660 + }, + { + "epoch": 22.0128986748627, + "grad_norm": 1.4387016296386719, + "learning_rate": 0.0001, + "loss": 55.0652, + "step": 13670 + }, + { + "epoch": 22.029022018441076, + "grad_norm": 0.7774295210838318, + "learning_rate": 0.0001, + "loss": 61.4395, + "step": 13680 + }, + { + "epoch": 22.045145362019447, + "grad_norm": 0.7438349723815918, + "learning_rate": 0.0001, + "loss": 60.998, + "step": 13690 + }, + { + "epoch": 22.061268705597822, + "grad_norm": 0.7506164312362671, + "learning_rate": 0.0001, + "loss": 60.6929, + "step": 13700 + }, + { + "epoch": 22.077392049176197, + "grad_norm": 0.6262985467910767, + "learning_rate": 0.0001, + "loss": 59.4921, + "step": 13710 + }, + { + "epoch": 22.093515392754572, + "grad_norm": 1.826015591621399, + "learning_rate": 0.0001, + "loss": 61.3791, + "step": 13720 + }, + { + "epoch": 22.109638736332947, + "grad_norm": 0.6410622596740723, + "learning_rate": 0.0001, + "loss": 62.5485, + "step": 13730 + }, + { + "epoch": 22.125762079911322, + "grad_norm": 0.7042442560195923, + "learning_rate": 0.0001, + "loss": 60.0654, + "step": 13740 + }, + { + "epoch": 22.141885423489697, + "grad_norm": 0.741142988204956, + "learning_rate": 0.0001, + "loss": 59.9831, + "step": 13750 + }, + { + "epoch": 22.158008767068072, + "grad_norm": 0.836445152759552, + "learning_rate": 0.0001, + "loss": 59.6805, + "step": 13760 + }, + { + "epoch": 22.174132110646447, + "grad_norm": 0.8353415727615356, + "learning_rate": 0.0001, + "loss": 60.1912, + "step": 13770 + }, + { + "epoch": 22.19025545422482, + "grad_norm": 0.6416126489639282, + "learning_rate": 0.0001, + "loss": 59.5518, + "step": 13780 + }, + { + "epoch": 22.206378797803193, + "grad_norm": 0.8702544569969177, + "learning_rate": 0.0001, + "loss": 57.7428, + "step": 13790 + }, + { + "epoch": 22.222502141381568, + "grad_norm": 0.6681593656539917, + "learning_rate": 0.0001, + "loss": 59.9499, + "step": 13800 + }, + { + "epoch": 22.238625484959943, + "grad_norm": 0.6185410022735596, + "learning_rate": 0.0001, + "loss": 59.8062, + "step": 13810 + }, + { + "epoch": 22.254748828538318, + "grad_norm": 0.7695328593254089, + "learning_rate": 0.0001, + "loss": 61.3512, + "step": 13820 + }, + { + "epoch": 22.270872172116693, + "grad_norm": 0.6550581455230713, + "learning_rate": 0.0001, + "loss": 59.3379, + "step": 13830 + }, + { + "epoch": 22.286995515695068, + "grad_norm": 0.7764340043067932, + "learning_rate": 0.0001, + "loss": 59.984, + "step": 13840 + }, + { + "epoch": 22.303118859273443, + "grad_norm": 0.6377913355827332, + "learning_rate": 0.0001, + "loss": 59.1862, + "step": 13850 + }, + { + "epoch": 22.319242202851818, + "grad_norm": 0.7837334275245667, + "learning_rate": 0.0001, + "loss": 60.8087, + "step": 13860 + }, + { + "epoch": 22.33536554643019, + "grad_norm": 0.7966547012329102, + "learning_rate": 0.0001, + "loss": 59.9109, + "step": 13870 + }, + { + "epoch": 22.351488890008564, + "grad_norm": 0.6303540468215942, + "learning_rate": 0.0001, + "loss": 61.0486, + "step": 13880 + }, + { + "epoch": 22.36761223358694, + "grad_norm": 0.6628289818763733, + "learning_rate": 0.0001, + "loss": 61.7723, + "step": 13890 + }, + { + "epoch": 22.383735577165314, + "grad_norm": 1.2156575918197632, + "learning_rate": 0.0001, + "loss": 58.0992, + "step": 13900 + }, + { + "epoch": 22.39985892074369, + "grad_norm": 1.0548591613769531, + "learning_rate": 0.0001, + "loss": 58.9139, + "step": 13910 + }, + { + "epoch": 22.415982264322064, + "grad_norm": 0.6389120817184448, + "learning_rate": 0.0001, + "loss": 59.1494, + "step": 13920 + }, + { + "epoch": 22.43210560790044, + "grad_norm": 0.6384537220001221, + "learning_rate": 0.0001, + "loss": 59.7135, + "step": 13930 + }, + { + "epoch": 22.448228951478814, + "grad_norm": 0.7434266209602356, + "learning_rate": 0.0001, + "loss": 60.7941, + "step": 13940 + }, + { + "epoch": 22.46435229505719, + "grad_norm": 0.7207310199737549, + "learning_rate": 0.0001, + "loss": 57.9466, + "step": 13950 + }, + { + "epoch": 22.48047563863556, + "grad_norm": 0.9680810570716858, + "learning_rate": 0.0001, + "loss": 57.5014, + "step": 13960 + }, + { + "epoch": 22.496598982213936, + "grad_norm": 0.5354718565940857, + "learning_rate": 0.0001, + "loss": 59.8771, + "step": 13970 + }, + { + "epoch": 22.51272232579231, + "grad_norm": 1.1624388694763184, + "learning_rate": 0.0001, + "loss": 60.2305, + "step": 13980 + }, + { + "epoch": 22.528845669370686, + "grad_norm": 0.6650974154472351, + "learning_rate": 0.0001, + "loss": 58.1649, + "step": 13990 + }, + { + "epoch": 22.54496901294906, + "grad_norm": 0.6151170134544373, + "learning_rate": 0.0001, + "loss": 60.4392, + "step": 14000 + }, + { + "epoch": 22.561092356527435, + "grad_norm": 0.6685694456100464, + "learning_rate": 0.0001, + "loss": 60.7055, + "step": 14010 + }, + { + "epoch": 22.57721570010581, + "grad_norm": 0.8092939257621765, + "learning_rate": 0.0001, + "loss": 57.8347, + "step": 14020 + }, + { + "epoch": 22.593339043684185, + "grad_norm": 0.7520709037780762, + "learning_rate": 0.0001, + "loss": 60.359, + "step": 14030 + }, + { + "epoch": 22.609462387262557, + "grad_norm": 0.8913780450820923, + "learning_rate": 0.0001, + "loss": 59.0833, + "step": 14040 + }, + { + "epoch": 22.625585730840932, + "grad_norm": 0.5508324503898621, + "learning_rate": 0.0001, + "loss": 60.8168, + "step": 14050 + }, + { + "epoch": 22.641709074419307, + "grad_norm": 0.7124258279800415, + "learning_rate": 0.0001, + "loss": 60.9433, + "step": 14060 + }, + { + "epoch": 22.65783241799768, + "grad_norm": 0.7286089062690735, + "learning_rate": 0.0001, + "loss": 59.8127, + "step": 14070 + }, + { + "epoch": 22.673955761576057, + "grad_norm": 0.8104858994483948, + "learning_rate": 0.0001, + "loss": 59.764, + "step": 14080 + }, + { + "epoch": 22.69007910515443, + "grad_norm": 0.7618247270584106, + "learning_rate": 0.0001, + "loss": 59.5563, + "step": 14090 + }, + { + "epoch": 22.706202448732807, + "grad_norm": 0.8512730598449707, + "learning_rate": 0.0001, + "loss": 59.3122, + "step": 14100 + }, + { + "epoch": 22.72232579231118, + "grad_norm": 0.6115924715995789, + "learning_rate": 0.0001, + "loss": 61.2874, + "step": 14110 + }, + { + "epoch": 22.738449135889557, + "grad_norm": 0.6883788704872131, + "learning_rate": 0.0001, + "loss": 59.427, + "step": 14120 + }, + { + "epoch": 22.754572479467928, + "grad_norm": 0.6240411400794983, + "learning_rate": 0.0001, + "loss": 58.9808, + "step": 14130 + }, + { + "epoch": 22.770695823046303, + "grad_norm": 0.6807436347007751, + "learning_rate": 0.0001, + "loss": 58.9609, + "step": 14140 + }, + { + "epoch": 22.786819166624678, + "grad_norm": 0.6661691665649414, + "learning_rate": 0.0001, + "loss": 59.8573, + "step": 14150 + }, + { + "epoch": 22.802942510203053, + "grad_norm": 0.7038732171058655, + "learning_rate": 0.0001, + "loss": 59.0114, + "step": 14160 + }, + { + "epoch": 22.819065853781428, + "grad_norm": 0.7736789584159851, + "learning_rate": 0.0001, + "loss": 59.6709, + "step": 14170 + }, + { + "epoch": 22.835189197359803, + "grad_norm": 0.6095895767211914, + "learning_rate": 0.0001, + "loss": 60.4468, + "step": 14180 + }, + { + "epoch": 22.851312540938178, + "grad_norm": 0.6531018018722534, + "learning_rate": 0.0001, + "loss": 61.4419, + "step": 14190 + }, + { + "epoch": 22.867435884516553, + "grad_norm": 0.7522090077400208, + "learning_rate": 0.0001, + "loss": 57.6574, + "step": 14200 + }, + { + "epoch": 22.883559228094928, + "grad_norm": 0.8028358817100525, + "learning_rate": 0.0001, + "loss": 59.0353, + "step": 14210 + }, + { + "epoch": 22.8996825716733, + "grad_norm": 0.7619246244430542, + "learning_rate": 0.0001, + "loss": 59.3586, + "step": 14220 + }, + { + "epoch": 22.915805915251674, + "grad_norm": 0.8312122821807861, + "learning_rate": 0.0001, + "loss": 60.8772, + "step": 14230 + }, + { + "epoch": 22.93192925883005, + "grad_norm": 0.6536790132522583, + "learning_rate": 0.0001, + "loss": 60.5941, + "step": 14240 + }, + { + "epoch": 22.948052602408424, + "grad_norm": 0.6973451972007751, + "learning_rate": 0.0001, + "loss": 59.5332, + "step": 14250 + }, + { + "epoch": 22.9641759459868, + "grad_norm": 0.6776084899902344, + "learning_rate": 0.0001, + "loss": 60.2337, + "step": 14260 + }, + { + "epoch": 22.980299289565174, + "grad_norm": 0.5728079080581665, + "learning_rate": 0.0001, + "loss": 61.0065, + "step": 14270 + }, + { + "epoch": 22.99642263314355, + "grad_norm": 0.798130452632904, + "learning_rate": 0.0001, + "loss": 59.7552, + "step": 14280 + }, + { + "epoch": 23.011286340504864, + "grad_norm": 0.6825193166732788, + "learning_rate": 0.0001, + "loss": 54.8593, + "step": 14290 + }, + { + "epoch": 23.02740968408324, + "grad_norm": 0.794450044631958, + "learning_rate": 0.0001, + "loss": 58.1472, + "step": 14300 + }, + { + "epoch": 23.04353302766161, + "grad_norm": 0.8178662657737732, + "learning_rate": 0.0001, + "loss": 58.8868, + "step": 14310 + }, + { + "epoch": 23.059656371239985, + "grad_norm": 0.6562325358390808, + "learning_rate": 0.0001, + "loss": 59.2406, + "step": 14320 + }, + { + "epoch": 23.07577971481836, + "grad_norm": 0.7282053828239441, + "learning_rate": 0.0001, + "loss": 58.1195, + "step": 14330 + }, + { + "epoch": 23.091903058396735, + "grad_norm": 0.6804936528205872, + "learning_rate": 0.0001, + "loss": 60.3755, + "step": 14340 + }, + { + "epoch": 23.10802640197511, + "grad_norm": 0.5851950645446777, + "learning_rate": 0.0001, + "loss": 59.2353, + "step": 14350 + }, + { + "epoch": 23.124149745553485, + "grad_norm": 0.5784706473350525, + "learning_rate": 0.0001, + "loss": 59.351, + "step": 14360 + }, + { + "epoch": 23.14027308913186, + "grad_norm": 0.7353758215904236, + "learning_rate": 0.0001, + "loss": 60.7708, + "step": 14370 + }, + { + "epoch": 23.156396432710235, + "grad_norm": 0.7761120796203613, + "learning_rate": 0.0001, + "loss": 59.6935, + "step": 14380 + }, + { + "epoch": 23.172519776288606, + "grad_norm": 0.6392366290092468, + "learning_rate": 0.0001, + "loss": 61.0263, + "step": 14390 + }, + { + "epoch": 23.18864311986698, + "grad_norm": 0.5722920894622803, + "learning_rate": 0.0001, + "loss": 59.4247, + "step": 14400 + }, + { + "epoch": 23.204766463445356, + "grad_norm": 0.6938275694847107, + "learning_rate": 0.0001, + "loss": 61.0933, + "step": 14410 + }, + { + "epoch": 23.22088980702373, + "grad_norm": 0.9164982438087463, + "learning_rate": 0.0001, + "loss": 58.0045, + "step": 14420 + }, + { + "epoch": 23.237013150602106, + "grad_norm": 0.6641019582748413, + "learning_rate": 0.0001, + "loss": 62.2162, + "step": 14430 + }, + { + "epoch": 23.25313649418048, + "grad_norm": 0.6775721311569214, + "learning_rate": 0.0001, + "loss": 60.2123, + "step": 14440 + }, + { + "epoch": 23.269259837758856, + "grad_norm": 0.6306013464927673, + "learning_rate": 0.0001, + "loss": 60.0638, + "step": 14450 + }, + { + "epoch": 23.28538318133723, + "grad_norm": 0.7770274877548218, + "learning_rate": 0.0001, + "loss": 60.87, + "step": 14460 + }, + { + "epoch": 23.301506524915606, + "grad_norm": 0.6893128752708435, + "learning_rate": 0.0001, + "loss": 59.2247, + "step": 14470 + }, + { + "epoch": 23.317629868493977, + "grad_norm": 0.7028453946113586, + "learning_rate": 0.0001, + "loss": 60.2405, + "step": 14480 + }, + { + "epoch": 23.333753212072352, + "grad_norm": 0.6184173822402954, + "learning_rate": 0.0001, + "loss": 58.5765, + "step": 14490 + }, + { + "epoch": 23.349876555650727, + "grad_norm": 0.793502151966095, + "learning_rate": 0.0001, + "loss": 57.5978, + "step": 14500 + }, + { + "epoch": 23.365999899229102, + "grad_norm": 0.660885751247406, + "learning_rate": 0.0001, + "loss": 60.5535, + "step": 14510 + }, + { + "epoch": 23.382123242807477, + "grad_norm": 0.9039516448974609, + "learning_rate": 0.0001, + "loss": 58.9432, + "step": 14520 + }, + { + "epoch": 23.398246586385852, + "grad_norm": 0.7348603010177612, + "learning_rate": 0.0001, + "loss": 58.3708, + "step": 14530 + }, + { + "epoch": 23.414369929964227, + "grad_norm": 0.5571108460426331, + "learning_rate": 0.0001, + "loss": 60.2059, + "step": 14540 + }, + { + "epoch": 23.430493273542602, + "grad_norm": 0.77266526222229, + "learning_rate": 0.0001, + "loss": 59.9602, + "step": 14550 + }, + { + "epoch": 23.446616617120977, + "grad_norm": 0.7564616203308105, + "learning_rate": 0.0001, + "loss": 59.0114, + "step": 14560 + }, + { + "epoch": 23.46273996069935, + "grad_norm": 1.0693135261535645, + "learning_rate": 0.0001, + "loss": 60.6075, + "step": 14570 + }, + { + "epoch": 23.478863304277724, + "grad_norm": 0.7800086140632629, + "learning_rate": 0.0001, + "loss": 60.6608, + "step": 14580 + }, + { + "epoch": 23.4949866478561, + "grad_norm": 0.8552407026290894, + "learning_rate": 0.0001, + "loss": 60.2378, + "step": 14590 + }, + { + "epoch": 23.511109991434473, + "grad_norm": 0.7270957231521606, + "learning_rate": 0.0001, + "loss": 60.8936, + "step": 14600 + }, + { + "epoch": 23.52723333501285, + "grad_norm": 0.5652869343757629, + "learning_rate": 0.0001, + "loss": 59.9882, + "step": 14610 + }, + { + "epoch": 23.543356678591223, + "grad_norm": 0.5465036034584045, + "learning_rate": 0.0001, + "loss": 57.4725, + "step": 14620 + }, + { + "epoch": 23.5594800221696, + "grad_norm": 0.6404461860656738, + "learning_rate": 0.0001, + "loss": 60.3962, + "step": 14630 + }, + { + "epoch": 23.575603365747973, + "grad_norm": 0.7860730886459351, + "learning_rate": 0.0001, + "loss": 60.3245, + "step": 14640 + }, + { + "epoch": 23.591726709326345, + "grad_norm": 0.6626171469688416, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 14650 + }, + { + "epoch": 23.60785005290472, + "grad_norm": 1.5377638339996338, + "learning_rate": 0.0001, + "loss": 58.9615, + "step": 14660 + }, + { + "epoch": 23.623973396483095, + "grad_norm": 0.6656123995780945, + "learning_rate": 0.0001, + "loss": 60.7947, + "step": 14670 + }, + { + "epoch": 23.64009674006147, + "grad_norm": 0.5155975818634033, + "learning_rate": 0.0001, + "loss": 57.7742, + "step": 14680 + }, + { + "epoch": 23.656220083639845, + "grad_norm": 0.6278973817825317, + "learning_rate": 0.0001, + "loss": 58.6188, + "step": 14690 + }, + { + "epoch": 23.67234342721822, + "grad_norm": 0.7261143326759338, + "learning_rate": 0.0001, + "loss": 56.8495, + "step": 14700 + }, + { + "epoch": 23.688466770796595, + "grad_norm": 0.8105781078338623, + "learning_rate": 0.0001, + "loss": 57.3139, + "step": 14710 + }, + { + "epoch": 23.70459011437497, + "grad_norm": 0.7012073993682861, + "learning_rate": 0.0001, + "loss": 61.1186, + "step": 14720 + }, + { + "epoch": 23.720713457953345, + "grad_norm": 0.6352536082267761, + "learning_rate": 0.0001, + "loss": 59.0136, + "step": 14730 + }, + { + "epoch": 23.736836801531716, + "grad_norm": 0.7279717326164246, + "learning_rate": 0.0001, + "loss": 58.7408, + "step": 14740 + }, + { + "epoch": 23.75296014511009, + "grad_norm": 0.6714015603065491, + "learning_rate": 0.0001, + "loss": 58.6431, + "step": 14750 + }, + { + "epoch": 23.769083488688466, + "grad_norm": 1.3876261711120605, + "learning_rate": 0.0001, + "loss": 59.7586, + "step": 14760 + }, + { + "epoch": 23.78520683226684, + "grad_norm": 0.7192941904067993, + "learning_rate": 0.0001, + "loss": 58.8487, + "step": 14770 + }, + { + "epoch": 23.801330175845216, + "grad_norm": 0.7333327531814575, + "learning_rate": 0.0001, + "loss": 60.5608, + "step": 14780 + }, + { + "epoch": 23.81745351942359, + "grad_norm": 0.8186061382293701, + "learning_rate": 0.0001, + "loss": 59.4232, + "step": 14790 + }, + { + "epoch": 23.833576863001966, + "grad_norm": 1.037039041519165, + "learning_rate": 0.0001, + "loss": 58.2384, + "step": 14800 + }, + { + "epoch": 23.84970020658034, + "grad_norm": 0.6739209890365601, + "learning_rate": 0.0001, + "loss": 57.7256, + "step": 14810 + }, + { + "epoch": 23.865823550158716, + "grad_norm": 0.6515845656394958, + "learning_rate": 0.0001, + "loss": 61.0039, + "step": 14820 + }, + { + "epoch": 23.881946893737087, + "grad_norm": 0.559317946434021, + "learning_rate": 0.0001, + "loss": 60.4458, + "step": 14830 + }, + { + "epoch": 23.898070237315462, + "grad_norm": 0.663934588432312, + "learning_rate": 0.0001, + "loss": 59.1646, + "step": 14840 + }, + { + "epoch": 23.914193580893837, + "grad_norm": 0.6409069299697876, + "learning_rate": 0.0001, + "loss": 58.9508, + "step": 14850 + }, + { + "epoch": 23.930316924472212, + "grad_norm": 0.7624269127845764, + "learning_rate": 0.0001, + "loss": 60.7256, + "step": 14860 + }, + { + "epoch": 23.946440268050587, + "grad_norm": 0.6949140429496765, + "learning_rate": 0.0001, + "loss": 58.8987, + "step": 14870 + }, + { + "epoch": 23.962563611628962, + "grad_norm": 0.714063823223114, + "learning_rate": 0.0001, + "loss": 59.255, + "step": 14880 + }, + { + "epoch": 23.978686955207337, + "grad_norm": 0.6570878028869629, + "learning_rate": 0.0001, + "loss": 59.2637, + "step": 14890 + }, + { + "epoch": 23.994810298785712, + "grad_norm": 0.7898263335227966, + "learning_rate": 0.0001, + "loss": 61.0218, + "step": 14900 + }, + { + "epoch": 24.009674006147026, + "grad_norm": 0.6673877239227295, + "learning_rate": 0.0001, + "loss": 53.8562, + "step": 14910 + }, + { + "epoch": 24.025797349725398, + "grad_norm": 0.5548285245895386, + "learning_rate": 0.0001, + "loss": 59.4445, + "step": 14920 + }, + { + "epoch": 24.041920693303773, + "grad_norm": 0.6273561716079712, + "learning_rate": 0.0001, + "loss": 59.3292, + "step": 14930 + }, + { + "epoch": 24.058044036882148, + "grad_norm": 0.7050267457962036, + "learning_rate": 0.0001, + "loss": 60.4896, + "step": 14940 + }, + { + "epoch": 24.074167380460523, + "grad_norm": 0.7030066251754761, + "learning_rate": 0.0001, + "loss": 59.1622, + "step": 14950 + }, + { + "epoch": 24.090290724038898, + "grad_norm": 0.8014484643936157, + "learning_rate": 0.0001, + "loss": 59.746, + "step": 14960 + }, + { + "epoch": 24.106414067617273, + "grad_norm": 0.869323194026947, + "learning_rate": 0.0001, + "loss": 60.2156, + "step": 14970 + }, + { + "epoch": 24.122537411195648, + "grad_norm": 0.7157219648361206, + "learning_rate": 0.0001, + "loss": 58.7229, + "step": 14980 + }, + { + "epoch": 24.138660754774023, + "grad_norm": 0.6318342685699463, + "learning_rate": 0.0001, + "loss": 61.1832, + "step": 14990 + }, + { + "epoch": 24.154784098352394, + "grad_norm": 0.8689974546432495, + "learning_rate": 0.0001, + "loss": 58.8083, + "step": 15000 + }, + { + "epoch": 24.17090744193077, + "grad_norm": 1.1719586849212646, + "learning_rate": 0.0001, + "loss": 57.745, + "step": 15010 + }, + { + "epoch": 24.187030785509144, + "grad_norm": 0.676295280456543, + "learning_rate": 0.0001, + "loss": 60.6236, + "step": 15020 + }, + { + "epoch": 24.20315412908752, + "grad_norm": 0.6745544672012329, + "learning_rate": 0.0001, + "loss": 59.494, + "step": 15030 + }, + { + "epoch": 24.219277472665894, + "grad_norm": 0.6221221685409546, + "learning_rate": 0.0001, + "loss": 58.894, + "step": 15040 + }, + { + "epoch": 24.23540081624427, + "grad_norm": 0.7013781070709229, + "learning_rate": 0.0001, + "loss": 59.935, + "step": 15050 + }, + { + "epoch": 24.251524159822644, + "grad_norm": 0.6094849705696106, + "learning_rate": 0.0001, + "loss": 59.8276, + "step": 15060 + }, + { + "epoch": 24.26764750340102, + "grad_norm": 0.6969972848892212, + "learning_rate": 0.0001, + "loss": 59.7568, + "step": 15070 + }, + { + "epoch": 24.283770846979394, + "grad_norm": 0.7223725318908691, + "learning_rate": 0.0001, + "loss": 57.9065, + "step": 15080 + }, + { + "epoch": 24.299894190557765, + "grad_norm": 0.7008336186408997, + "learning_rate": 0.0001, + "loss": 59.5206, + "step": 15090 + }, + { + "epoch": 24.31601753413614, + "grad_norm": 0.6490783095359802, + "learning_rate": 0.0001, + "loss": 59.4457, + "step": 15100 + }, + { + "epoch": 24.332140877714515, + "grad_norm": 0.6096290349960327, + "learning_rate": 0.0001, + "loss": 60.8434, + "step": 15110 + }, + { + "epoch": 24.34826422129289, + "grad_norm": 0.771045446395874, + "learning_rate": 0.0001, + "loss": 60.3167, + "step": 15120 + }, + { + "epoch": 24.364387564871265, + "grad_norm": 0.7178729772567749, + "learning_rate": 0.0001, + "loss": 57.6667, + "step": 15130 + }, + { + "epoch": 24.38051090844964, + "grad_norm": 0.6484465003013611, + "learning_rate": 0.0001, + "loss": 60.1199, + "step": 15140 + }, + { + "epoch": 24.396634252028015, + "grad_norm": 1.0030865669250488, + "learning_rate": 0.0001, + "loss": 57.9288, + "step": 15150 + }, + { + "epoch": 24.41275759560639, + "grad_norm": 0.8297843933105469, + "learning_rate": 0.0001, + "loss": 61.0941, + "step": 15160 + }, + { + "epoch": 24.428880939184765, + "grad_norm": 0.9115694165229797, + "learning_rate": 0.0001, + "loss": 58.9574, + "step": 15170 + }, + { + "epoch": 24.445004282763136, + "grad_norm": 0.7758508920669556, + "learning_rate": 0.0001, + "loss": 58.8852, + "step": 15180 + }, + { + "epoch": 24.46112762634151, + "grad_norm": 0.5539108514785767, + "learning_rate": 0.0001, + "loss": 60.9549, + "step": 15190 + }, + { + "epoch": 24.477250969919886, + "grad_norm": 0.7292829751968384, + "learning_rate": 0.0001, + "loss": 58.0818, + "step": 15200 + }, + { + "epoch": 24.49337431349826, + "grad_norm": 0.6253948211669922, + "learning_rate": 0.0001, + "loss": 59.656, + "step": 15210 + }, + { + "epoch": 24.509497657076636, + "grad_norm": 0.8054394125938416, + "learning_rate": 0.0001, + "loss": 60.934, + "step": 15220 + }, + { + "epoch": 24.52562100065501, + "grad_norm": 0.8183079361915588, + "learning_rate": 0.0001, + "loss": 59.5303, + "step": 15230 + }, + { + "epoch": 24.541744344233386, + "grad_norm": 0.6063419580459595, + "learning_rate": 0.0001, + "loss": 59.2852, + "step": 15240 + }, + { + "epoch": 24.55786768781176, + "grad_norm": 0.6372854113578796, + "learning_rate": 0.0001, + "loss": 58.9647, + "step": 15250 + }, + { + "epoch": 24.573991031390136, + "grad_norm": 0.6270446181297302, + "learning_rate": 0.0001, + "loss": 58.5641, + "step": 15260 + }, + { + "epoch": 24.590114374968508, + "grad_norm": 0.6229680776596069, + "learning_rate": 0.0001, + "loss": 58.4142, + "step": 15270 + }, + { + "epoch": 24.606237718546883, + "grad_norm": 0.6567183136940002, + "learning_rate": 0.0001, + "loss": 60.1162, + "step": 15280 + }, + { + "epoch": 24.622361062125258, + "grad_norm": 0.6975446343421936, + "learning_rate": 0.0001, + "loss": 58.7862, + "step": 15290 + }, + { + "epoch": 24.638484405703633, + "grad_norm": 0.6961022615432739, + "learning_rate": 0.0001, + "loss": 58.3247, + "step": 15300 + }, + { + "epoch": 24.654607749282007, + "grad_norm": 5.25744104385376, + "learning_rate": 0.0001, + "loss": 59.3863, + "step": 15310 + }, + { + "epoch": 24.670731092860382, + "grad_norm": 0.8439975380897522, + "learning_rate": 0.0001, + "loss": 59.7157, + "step": 15320 + }, + { + "epoch": 24.686854436438757, + "grad_norm": 0.596845269203186, + "learning_rate": 0.0001, + "loss": 60.0859, + "step": 15330 + }, + { + "epoch": 24.702977780017132, + "grad_norm": 0.6758428812026978, + "learning_rate": 0.0001, + "loss": 59.9305, + "step": 15340 + }, + { + "epoch": 24.719101123595507, + "grad_norm": 0.7418860197067261, + "learning_rate": 0.0001, + "loss": 59.0929, + "step": 15350 + }, + { + "epoch": 24.73522446717388, + "grad_norm": 0.6856047511100769, + "learning_rate": 0.0001, + "loss": 58.7201, + "step": 15360 + }, + { + "epoch": 24.751347810752254, + "grad_norm": 0.5327897071838379, + "learning_rate": 0.0001, + "loss": 59.5899, + "step": 15370 + }, + { + "epoch": 24.76747115433063, + "grad_norm": 0.7127685546875, + "learning_rate": 0.0001, + "loss": 59.7729, + "step": 15380 + }, + { + "epoch": 24.783594497909004, + "grad_norm": 1.8885949850082397, + "learning_rate": 0.0001, + "loss": 58.3813, + "step": 15390 + }, + { + "epoch": 24.79971784148738, + "grad_norm": 0.7997881174087524, + "learning_rate": 0.0001, + "loss": 61.9402, + "step": 15400 + }, + { + "epoch": 24.815841185065754, + "grad_norm": 0.9729613661766052, + "learning_rate": 0.0001, + "loss": 59.7961, + "step": 15410 + }, + { + "epoch": 24.83196452864413, + "grad_norm": 0.8450629115104675, + "learning_rate": 0.0001, + "loss": 59.5969, + "step": 15420 + }, + { + "epoch": 24.848087872222504, + "grad_norm": 0.720668613910675, + "learning_rate": 0.0001, + "loss": 58.187, + "step": 15430 + }, + { + "epoch": 24.864211215800875, + "grad_norm": 0.638506293296814, + "learning_rate": 0.0001, + "loss": 59.4599, + "step": 15440 + }, + { + "epoch": 24.88033455937925, + "grad_norm": 0.6158926486968994, + "learning_rate": 0.0001, + "loss": 61.3858, + "step": 15450 + }, + { + "epoch": 24.896457902957625, + "grad_norm": 0.8130396604537964, + "learning_rate": 0.0001, + "loss": 59.3795, + "step": 15460 + }, + { + "epoch": 24.912581246536, + "grad_norm": 0.7966095805168152, + "learning_rate": 0.0001, + "loss": 60.3319, + "step": 15470 + }, + { + "epoch": 24.928704590114375, + "grad_norm": 0.6240444779396057, + "learning_rate": 0.0001, + "loss": 59.0407, + "step": 15480 + }, + { + "epoch": 24.94482793369275, + "grad_norm": 0.7439318895339966, + "learning_rate": 0.0001, + "loss": 57.7692, + "step": 15490 + }, + { + "epoch": 24.960951277271125, + "grad_norm": 0.6942011713981628, + "learning_rate": 0.0001, + "loss": 56.8418, + "step": 15500 + }, + { + "epoch": 24.9770746208495, + "grad_norm": 0.6470242738723755, + "learning_rate": 0.0001, + "loss": 59.8361, + "step": 15510 + }, + { + "epoch": 24.993197964427875, + "grad_norm": 0.6935222148895264, + "learning_rate": 0.0001, + "loss": 59.5357, + "step": 15520 + }, + { + "epoch": 25.008061671789186, + "grad_norm": 0.8943103551864624, + "learning_rate": 0.0001, + "loss": 54.8185, + "step": 15530 + }, + { + "epoch": 25.02418501536756, + "grad_norm": 0.7637899518013, + "learning_rate": 0.0001, + "loss": 58.4746, + "step": 15540 + }, + { + "epoch": 25.040308358945936, + "grad_norm": 0.8261918425559998, + "learning_rate": 0.0001, + "loss": 57.3993, + "step": 15550 + }, + { + "epoch": 25.05643170252431, + "grad_norm": 0.6395642161369324, + "learning_rate": 0.0001, + "loss": 58.123, + "step": 15560 + }, + { + "epoch": 25.072555046102686, + "grad_norm": 0.7899443507194519, + "learning_rate": 0.0001, + "loss": 57.1487, + "step": 15570 + }, + { + "epoch": 25.08867838968106, + "grad_norm": 0.6202112436294556, + "learning_rate": 0.0001, + "loss": 59.4995, + "step": 15580 + }, + { + "epoch": 25.104801733259436, + "grad_norm": 0.7683939933776855, + "learning_rate": 0.0001, + "loss": 59.1155, + "step": 15590 + }, + { + "epoch": 25.12092507683781, + "grad_norm": 0.6531215906143188, + "learning_rate": 0.0001, + "loss": 61.9998, + "step": 15600 + }, + { + "epoch": 25.137048420416185, + "grad_norm": 0.6354383826255798, + "learning_rate": 0.0001, + "loss": 59.3861, + "step": 15610 + }, + { + "epoch": 25.153171763994557, + "grad_norm": 0.6507435441017151, + "learning_rate": 0.0001, + "loss": 59.939, + "step": 15620 + }, + { + "epoch": 25.169295107572932, + "grad_norm": 0.8014065623283386, + "learning_rate": 0.0001, + "loss": 59.4927, + "step": 15630 + }, + { + "epoch": 25.185418451151307, + "grad_norm": 0.53805011510849, + "learning_rate": 0.0001, + "loss": 58.526, + "step": 15640 + }, + { + "epoch": 25.201541794729682, + "grad_norm": 4.835650444030762, + "learning_rate": 0.0001, + "loss": 58.6711, + "step": 15650 + }, + { + "epoch": 25.217665138308057, + "grad_norm": 0.723210871219635, + "learning_rate": 0.0001, + "loss": 60.3953, + "step": 15660 + }, + { + "epoch": 25.23378848188643, + "grad_norm": 0.7176834940910339, + "learning_rate": 0.0001, + "loss": 58.8576, + "step": 15670 + }, + { + "epoch": 25.249911825464807, + "grad_norm": 0.8280025124549866, + "learning_rate": 0.0001, + "loss": 58.7146, + "step": 15680 + }, + { + "epoch": 25.26603516904318, + "grad_norm": 0.6296973824501038, + "learning_rate": 0.0001, + "loss": 60.7341, + "step": 15690 + }, + { + "epoch": 25.282158512621557, + "grad_norm": 1.3184518814086914, + "learning_rate": 0.0001, + "loss": 60.6906, + "step": 15700 + }, + { + "epoch": 25.298281856199928, + "grad_norm": 1.0408564805984497, + "learning_rate": 0.0001, + "loss": 57.4703, + "step": 15710 + }, + { + "epoch": 25.314405199778303, + "grad_norm": 0.7140268683433533, + "learning_rate": 0.0001, + "loss": 59.8452, + "step": 15720 + }, + { + "epoch": 25.330528543356678, + "grad_norm": 0.7822073698043823, + "learning_rate": 0.0001, + "loss": 60.4394, + "step": 15730 + }, + { + "epoch": 25.346651886935053, + "grad_norm": 0.6579483151435852, + "learning_rate": 0.0001, + "loss": 59.5358, + "step": 15740 + }, + { + "epoch": 25.362775230513428, + "grad_norm": 0.6936199069023132, + "learning_rate": 0.0001, + "loss": 60.2117, + "step": 15750 + }, + { + "epoch": 25.378898574091803, + "grad_norm": 0.7166867852210999, + "learning_rate": 0.0001, + "loss": 58.6835, + "step": 15760 + }, + { + "epoch": 25.395021917670178, + "grad_norm": 0.7240379452705383, + "learning_rate": 0.0001, + "loss": 59.6293, + "step": 15770 + }, + { + "epoch": 25.411145261248553, + "grad_norm": 0.7465791702270508, + "learning_rate": 0.0001, + "loss": 58.2512, + "step": 15780 + }, + { + "epoch": 25.427268604826924, + "grad_norm": 0.6713439226150513, + "learning_rate": 0.0001, + "loss": 58.4149, + "step": 15790 + }, + { + "epoch": 25.4433919484053, + "grad_norm": 0.587433397769928, + "learning_rate": 0.0001, + "loss": 60.7315, + "step": 15800 + }, + { + "epoch": 25.459515291983674, + "grad_norm": 0.8216325044631958, + "learning_rate": 0.0001, + "loss": 58.5339, + "step": 15810 + }, + { + "epoch": 25.47563863556205, + "grad_norm": 0.8947033286094666, + "learning_rate": 0.0001, + "loss": 58.7762, + "step": 15820 + }, + { + "epoch": 25.491761979140424, + "grad_norm": 0.6659840941429138, + "learning_rate": 0.0001, + "loss": 58.9931, + "step": 15830 + }, + { + "epoch": 25.5078853227188, + "grad_norm": 0.7239252328872681, + "learning_rate": 0.0001, + "loss": 59.7113, + "step": 15840 + }, + { + "epoch": 25.524008666297174, + "grad_norm": 0.7233046889305115, + "learning_rate": 0.0001, + "loss": 58.973, + "step": 15850 + }, + { + "epoch": 25.54013200987555, + "grad_norm": 0.6620002388954163, + "learning_rate": 0.0001, + "loss": 57.1255, + "step": 15860 + }, + { + "epoch": 25.556255353453924, + "grad_norm": 0.6544366478919983, + "learning_rate": 0.0001, + "loss": 58.5897, + "step": 15870 + }, + { + "epoch": 25.572378697032295, + "grad_norm": 0.8955110311508179, + "learning_rate": 0.0001, + "loss": 58.3241, + "step": 15880 + }, + { + "epoch": 25.58850204061067, + "grad_norm": 0.6869794726371765, + "learning_rate": 0.0001, + "loss": 58.1951, + "step": 15890 + }, + { + "epoch": 25.604625384189045, + "grad_norm": 0.6819241642951965, + "learning_rate": 0.0001, + "loss": 58.6505, + "step": 15900 + }, + { + "epoch": 25.62074872776742, + "grad_norm": 0.6019999980926514, + "learning_rate": 0.0001, + "loss": 58.1397, + "step": 15910 + }, + { + "epoch": 25.636872071345795, + "grad_norm": 0.6664146184921265, + "learning_rate": 0.0001, + "loss": 58.6735, + "step": 15920 + }, + { + "epoch": 25.65299541492417, + "grad_norm": 0.7511261701583862, + "learning_rate": 0.0001, + "loss": 58.9189, + "step": 15930 + }, + { + "epoch": 25.669118758502545, + "grad_norm": 0.6886443495750427, + "learning_rate": 0.0001, + "loss": 60.1288, + "step": 15940 + }, + { + "epoch": 25.68524210208092, + "grad_norm": 0.7647057771682739, + "learning_rate": 0.0001, + "loss": 58.5042, + "step": 15950 + }, + { + "epoch": 25.701365445659295, + "grad_norm": 0.6636016368865967, + "learning_rate": 0.0001, + "loss": 56.7644, + "step": 15960 + }, + { + "epoch": 25.717488789237667, + "grad_norm": 0.6909965872764587, + "learning_rate": 0.0001, + "loss": 58.9248, + "step": 15970 + }, + { + "epoch": 25.73361213281604, + "grad_norm": 0.7853394150733948, + "learning_rate": 0.0001, + "loss": 58.7267, + "step": 15980 + }, + { + "epoch": 25.749735476394417, + "grad_norm": 0.5992239713668823, + "learning_rate": 0.0001, + "loss": 58.4463, + "step": 15990 + }, + { + "epoch": 25.76585881997279, + "grad_norm": 0.5730507969856262, + "learning_rate": 0.0001, + "loss": 58.2946, + "step": 16000 + }, + { + "epoch": 25.781982163551167, + "grad_norm": 0.8904218077659607, + "learning_rate": 0.0001, + "loss": 58.8344, + "step": 16010 + }, + { + "epoch": 25.79810550712954, + "grad_norm": 0.8680905103683472, + "learning_rate": 0.0001, + "loss": 58.8969, + "step": 16020 + }, + { + "epoch": 25.814228850707917, + "grad_norm": 0.7501125335693359, + "learning_rate": 0.0001, + "loss": 59.0709, + "step": 16030 + }, + { + "epoch": 25.83035219428629, + "grad_norm": 0.7211290597915649, + "learning_rate": 0.0001, + "loss": 58.1067, + "step": 16040 + }, + { + "epoch": 25.846475537864663, + "grad_norm": 0.6130759119987488, + "learning_rate": 0.0001, + "loss": 58.7301, + "step": 16050 + }, + { + "epoch": 25.862598881443038, + "grad_norm": 0.5713672041893005, + "learning_rate": 0.0001, + "loss": 59.7271, + "step": 16060 + }, + { + "epoch": 25.878722225021413, + "grad_norm": 0.5994897484779358, + "learning_rate": 0.0001, + "loss": 59.3829, + "step": 16070 + }, + { + "epoch": 25.894845568599788, + "grad_norm": 0.686657190322876, + "learning_rate": 0.0001, + "loss": 59.2881, + "step": 16080 + }, + { + "epoch": 25.910968912178163, + "grad_norm": 0.6650110483169556, + "learning_rate": 0.0001, + "loss": 56.2951, + "step": 16090 + }, + { + "epoch": 25.927092255756538, + "grad_norm": 0.7304360866546631, + "learning_rate": 0.0001, + "loss": 60.5764, + "step": 16100 + }, + { + "epoch": 25.943215599334913, + "grad_norm": 0.6502479910850525, + "learning_rate": 0.0001, + "loss": 59.3936, + "step": 16110 + }, + { + "epoch": 25.959338942913288, + "grad_norm": 0.7156543731689453, + "learning_rate": 0.0001, + "loss": 58.3354, + "step": 16120 + }, + { + "epoch": 25.975462286491663, + "grad_norm": 0.951116144657135, + "learning_rate": 0.0001, + "loss": 60.3694, + "step": 16130 + }, + { + "epoch": 25.991585630070034, + "grad_norm": 0.6068692207336426, + "learning_rate": 0.0001, + "loss": 58.4947, + "step": 16140 + }, + { + "epoch": 26.00644933743135, + "grad_norm": 0.7921227216720581, + "learning_rate": 0.0001, + "loss": 54.2985, + "step": 16150 + }, + { + "epoch": 26.022572681009724, + "grad_norm": 0.7406607866287231, + "learning_rate": 0.0001, + "loss": 59.1799, + "step": 16160 + }, + { + "epoch": 26.0386960245881, + "grad_norm": 0.6859458088874817, + "learning_rate": 0.0001, + "loss": 58.5136, + "step": 16170 + }, + { + "epoch": 26.054819368166473, + "grad_norm": 0.5251442790031433, + "learning_rate": 0.0001, + "loss": 58.0473, + "step": 16180 + }, + { + "epoch": 26.07094271174485, + "grad_norm": 0.8837170600891113, + "learning_rate": 0.0001, + "loss": 58.7922, + "step": 16190 + }, + { + "epoch": 26.087066055323223, + "grad_norm": 0.6178808808326721, + "learning_rate": 0.0001, + "loss": 58.5929, + "step": 16200 + }, + { + "epoch": 26.1031893989016, + "grad_norm": 0.7404752373695374, + "learning_rate": 0.0001, + "loss": 60.8605, + "step": 16210 + }, + { + "epoch": 26.119312742479973, + "grad_norm": 0.812877357006073, + "learning_rate": 0.0001, + "loss": 60.1625, + "step": 16220 + }, + { + "epoch": 26.135436086058345, + "grad_norm": 0.7631093859672546, + "learning_rate": 0.0001, + "loss": 59.0114, + "step": 16230 + }, + { + "epoch": 26.15155942963672, + "grad_norm": 0.5635222792625427, + "learning_rate": 0.0001, + "loss": 58.3207, + "step": 16240 + }, + { + "epoch": 26.167682773215095, + "grad_norm": 0.6111196875572205, + "learning_rate": 0.0001, + "loss": 60.1611, + "step": 16250 + }, + { + "epoch": 26.18380611679347, + "grad_norm": 0.6021577715873718, + "learning_rate": 0.0001, + "loss": 59.2199, + "step": 16260 + }, + { + "epoch": 26.199929460371845, + "grad_norm": 0.6264477968215942, + "learning_rate": 0.0001, + "loss": 58.9156, + "step": 16270 + }, + { + "epoch": 26.21605280395022, + "grad_norm": 0.5620694160461426, + "learning_rate": 0.0001, + "loss": 58.699, + "step": 16280 + }, + { + "epoch": 26.232176147528595, + "grad_norm": 0.6636842489242554, + "learning_rate": 0.0001, + "loss": 58.1521, + "step": 16290 + }, + { + "epoch": 26.24829949110697, + "grad_norm": 1.903132677078247, + "learning_rate": 0.0001, + "loss": 56.8953, + "step": 16300 + }, + { + "epoch": 26.264422834685345, + "grad_norm": 0.6112220883369446, + "learning_rate": 0.0001, + "loss": 58.6921, + "step": 16310 + }, + { + "epoch": 26.280546178263716, + "grad_norm": 0.6523616909980774, + "learning_rate": 0.0001, + "loss": 59.8097, + "step": 16320 + }, + { + "epoch": 26.29666952184209, + "grad_norm": 0.6033239364624023, + "learning_rate": 0.0001, + "loss": 58.2301, + "step": 16330 + }, + { + "epoch": 26.312792865420466, + "grad_norm": 0.6334905624389648, + "learning_rate": 0.0001, + "loss": 57.4851, + "step": 16340 + }, + { + "epoch": 26.32891620899884, + "grad_norm": 0.8884124159812927, + "learning_rate": 0.0001, + "loss": 58.4404, + "step": 16350 + }, + { + "epoch": 26.345039552577216, + "grad_norm": 0.6134583950042725, + "learning_rate": 0.0001, + "loss": 59.7355, + "step": 16360 + }, + { + "epoch": 26.36116289615559, + "grad_norm": 0.5804709792137146, + "learning_rate": 0.0001, + "loss": 59.3307, + "step": 16370 + }, + { + "epoch": 26.377286239733966, + "grad_norm": 1.3471691608428955, + "learning_rate": 0.0001, + "loss": 60.4356, + "step": 16380 + }, + { + "epoch": 26.39340958331234, + "grad_norm": 0.7593581080436707, + "learning_rate": 0.0001, + "loss": 60.6596, + "step": 16390 + }, + { + "epoch": 26.409532926890712, + "grad_norm": 0.7059800624847412, + "learning_rate": 0.0001, + "loss": 58.8393, + "step": 16400 + }, + { + "epoch": 26.425656270469087, + "grad_norm": 0.836167573928833, + "learning_rate": 0.0001, + "loss": 59.7399, + "step": 16410 + }, + { + "epoch": 26.441779614047462, + "grad_norm": 0.7547316551208496, + "learning_rate": 0.0001, + "loss": 57.8424, + "step": 16420 + }, + { + "epoch": 26.457902957625837, + "grad_norm": 0.6655163168907166, + "learning_rate": 0.0001, + "loss": 58.7625, + "step": 16430 + }, + { + "epoch": 26.474026301204212, + "grad_norm": 0.6393458247184753, + "learning_rate": 0.0001, + "loss": 58.88, + "step": 16440 + }, + { + "epoch": 26.490149644782587, + "grad_norm": 0.7284976840019226, + "learning_rate": 0.0001, + "loss": 59.734, + "step": 16450 + }, + { + "epoch": 26.506272988360962, + "grad_norm": 0.7217265963554382, + "learning_rate": 0.0001, + "loss": 58.9343, + "step": 16460 + }, + { + "epoch": 26.522396331939337, + "grad_norm": 0.6045732498168945, + "learning_rate": 0.0001, + "loss": 58.758, + "step": 16470 + }, + { + "epoch": 26.538519675517712, + "grad_norm": 0.6189675331115723, + "learning_rate": 0.0001, + "loss": 56.4975, + "step": 16480 + }, + { + "epoch": 26.554643019096083, + "grad_norm": 0.5564262270927429, + "learning_rate": 0.0001, + "loss": 58.6003, + "step": 16490 + }, + { + "epoch": 26.57076636267446, + "grad_norm": 0.727336049079895, + "learning_rate": 0.0001, + "loss": 59.097, + "step": 16500 + }, + { + "epoch": 26.586889706252833, + "grad_norm": 0.5768782496452332, + "learning_rate": 0.0001, + "loss": 59.4801, + "step": 16510 + }, + { + "epoch": 26.60301304983121, + "grad_norm": 0.5879726409912109, + "learning_rate": 0.0001, + "loss": 58.8839, + "step": 16520 + }, + { + "epoch": 26.619136393409583, + "grad_norm": 0.6342545747756958, + "learning_rate": 0.0001, + "loss": 58.4209, + "step": 16530 + }, + { + "epoch": 26.63525973698796, + "grad_norm": 0.7538003325462341, + "learning_rate": 0.0001, + "loss": 58.973, + "step": 16540 + }, + { + "epoch": 26.651383080566333, + "grad_norm": 0.7319458723068237, + "learning_rate": 0.0001, + "loss": 59.5449, + "step": 16550 + }, + { + "epoch": 26.667506424144708, + "grad_norm": 0.6014299392700195, + "learning_rate": 0.0001, + "loss": 59.2934, + "step": 16560 + }, + { + "epoch": 26.683629767723083, + "grad_norm": 0.6779450178146362, + "learning_rate": 0.0001, + "loss": 59.5138, + "step": 16570 + }, + { + "epoch": 26.699753111301455, + "grad_norm": 0.6749789118766785, + "learning_rate": 0.0001, + "loss": 58.2765, + "step": 16580 + }, + { + "epoch": 26.71587645487983, + "grad_norm": 0.6327691674232483, + "learning_rate": 0.0001, + "loss": 59.766, + "step": 16590 + }, + { + "epoch": 26.731999798458205, + "grad_norm": 0.5834855437278748, + "learning_rate": 0.0001, + "loss": 57.2493, + "step": 16600 + }, + { + "epoch": 26.74812314203658, + "grad_norm": 0.6618366837501526, + "learning_rate": 0.0001, + "loss": 57.2663, + "step": 16610 + }, + { + "epoch": 26.764246485614954, + "grad_norm": 0.7070173621177673, + "learning_rate": 0.0001, + "loss": 61.1754, + "step": 16620 + }, + { + "epoch": 26.78036982919333, + "grad_norm": 0.651278555393219, + "learning_rate": 0.0001, + "loss": 59.033, + "step": 16630 + }, + { + "epoch": 26.796493172771704, + "grad_norm": 0.634915828704834, + "learning_rate": 0.0001, + "loss": 58.2336, + "step": 16640 + }, + { + "epoch": 26.81261651635008, + "grad_norm": 0.8514180779457092, + "learning_rate": 0.0001, + "loss": 57.1678, + "step": 16650 + }, + { + "epoch": 26.828739859928454, + "grad_norm": 0.6718772053718567, + "learning_rate": 0.0001, + "loss": 58.5139, + "step": 16660 + }, + { + "epoch": 26.844863203506826, + "grad_norm": 0.6963548064231873, + "learning_rate": 0.0001, + "loss": 58.8449, + "step": 16670 + }, + { + "epoch": 26.8609865470852, + "grad_norm": 0.6028134822845459, + "learning_rate": 0.0001, + "loss": 58.4613, + "step": 16680 + }, + { + "epoch": 26.877109890663576, + "grad_norm": 0.7414282560348511, + "learning_rate": 0.0001, + "loss": 58.4078, + "step": 16690 + }, + { + "epoch": 26.89323323424195, + "grad_norm": 0.6019772291183472, + "learning_rate": 0.0001, + "loss": 58.086, + "step": 16700 + }, + { + "epoch": 26.909356577820326, + "grad_norm": 0.6267445683479309, + "learning_rate": 0.0001, + "loss": 58.5921, + "step": 16710 + }, + { + "epoch": 26.9254799213987, + "grad_norm": 0.7474961280822754, + "learning_rate": 0.0001, + "loss": 59.1194, + "step": 16720 + }, + { + "epoch": 26.941603264977076, + "grad_norm": 0.7952353358268738, + "learning_rate": 0.0001, + "loss": 58.4545, + "step": 16730 + }, + { + "epoch": 26.95772660855545, + "grad_norm": 0.6562651991844177, + "learning_rate": 0.0001, + "loss": 58.699, + "step": 16740 + }, + { + "epoch": 26.973849952133826, + "grad_norm": 0.6084649562835693, + "learning_rate": 0.0001, + "loss": 58.5552, + "step": 16750 + }, + { + "epoch": 26.989973295712197, + "grad_norm": 0.5753876566886902, + "learning_rate": 0.0001, + "loss": 58.5317, + "step": 16760 + }, + { + "epoch": 27.00483700307351, + "grad_norm": 0.7396492958068848, + "learning_rate": 0.0001, + "loss": 54.4917, + "step": 16770 + }, + { + "epoch": 27.020960346651886, + "grad_norm": 0.5759479999542236, + "learning_rate": 0.0001, + "loss": 60.5276, + "step": 16780 + }, + { + "epoch": 27.03708369023026, + "grad_norm": 0.6621933579444885, + "learning_rate": 0.0001, + "loss": 57.8109, + "step": 16790 + }, + { + "epoch": 27.053207033808636, + "grad_norm": 0.6219635605812073, + "learning_rate": 0.0001, + "loss": 59.1457, + "step": 16800 + }, + { + "epoch": 27.06933037738701, + "grad_norm": 0.6395589113235474, + "learning_rate": 0.0001, + "loss": 58.2069, + "step": 16810 + }, + { + "epoch": 27.085453720965386, + "grad_norm": 0.9114459753036499, + "learning_rate": 0.0001, + "loss": 58.7224, + "step": 16820 + }, + { + "epoch": 27.10157706454376, + "grad_norm": 0.6759451031684875, + "learning_rate": 0.0001, + "loss": 59.3057, + "step": 16830 + }, + { + "epoch": 27.117700408122133, + "grad_norm": 0.6997210383415222, + "learning_rate": 0.0001, + "loss": 59.7155, + "step": 16840 + }, + { + "epoch": 27.133823751700508, + "grad_norm": 1.4591188430786133, + "learning_rate": 0.0001, + "loss": 59.7251, + "step": 16850 + }, + { + "epoch": 27.149947095278883, + "grad_norm": 0.661625325679779, + "learning_rate": 0.0001, + "loss": 59.065, + "step": 16860 + }, + { + "epoch": 27.166070438857258, + "grad_norm": 0.7471212148666382, + "learning_rate": 0.0001, + "loss": 60.2483, + "step": 16870 + }, + { + "epoch": 27.182193782435633, + "grad_norm": 0.7325915098190308, + "learning_rate": 0.0001, + "loss": 59.5878, + "step": 16880 + }, + { + "epoch": 27.198317126014008, + "grad_norm": 0.6631053686141968, + "learning_rate": 0.0001, + "loss": 60.045, + "step": 16890 + }, + { + "epoch": 27.214440469592382, + "grad_norm": 0.7794095277786255, + "learning_rate": 0.0001, + "loss": 58.6427, + "step": 16900 + }, + { + "epoch": 27.230563813170757, + "grad_norm": 0.683525025844574, + "learning_rate": 0.0001, + "loss": 58.2072, + "step": 16910 + }, + { + "epoch": 27.246687156749132, + "grad_norm": 0.7676731944084167, + "learning_rate": 0.0001, + "loss": 59.5006, + "step": 16920 + }, + { + "epoch": 27.262810500327504, + "grad_norm": 0.6451899409294128, + "learning_rate": 0.0001, + "loss": 59.5833, + "step": 16930 + }, + { + "epoch": 27.27893384390588, + "grad_norm": 0.8003790378570557, + "learning_rate": 0.0001, + "loss": 59.6564, + "step": 16940 + }, + { + "epoch": 27.295057187484254, + "grad_norm": 0.8254024386405945, + "learning_rate": 0.0001, + "loss": 59.8074, + "step": 16950 + }, + { + "epoch": 27.31118053106263, + "grad_norm": 0.8280795812606812, + "learning_rate": 0.0001, + "loss": 57.1944, + "step": 16960 + }, + { + "epoch": 27.327303874641004, + "grad_norm": 0.6558408141136169, + "learning_rate": 0.0001, + "loss": 55.8846, + "step": 16970 + }, + { + "epoch": 27.34342721821938, + "grad_norm": 0.8485943078994751, + "learning_rate": 0.0001, + "loss": 57.6715, + "step": 16980 + }, + { + "epoch": 27.359550561797754, + "grad_norm": 0.7805924415588379, + "learning_rate": 0.0001, + "loss": 61.0427, + "step": 16990 + }, + { + "epoch": 27.37567390537613, + "grad_norm": 0.7988429069519043, + "learning_rate": 0.0001, + "loss": 57.6632, + "step": 17000 + }, + { + "epoch": 27.391797248954504, + "grad_norm": 0.816113293170929, + "learning_rate": 0.0001, + "loss": 59.2601, + "step": 17010 + }, + { + "epoch": 27.407920592532875, + "grad_norm": 0.7985144257545471, + "learning_rate": 0.0001, + "loss": 60.3656, + "step": 17020 + }, + { + "epoch": 27.42404393611125, + "grad_norm": 0.7022702097892761, + "learning_rate": 0.0001, + "loss": 58.657, + "step": 17030 + }, + { + "epoch": 27.440167279689625, + "grad_norm": 0.8588566780090332, + "learning_rate": 0.0001, + "loss": 58.9401, + "step": 17040 + }, + { + "epoch": 27.456290623268, + "grad_norm": 0.6444416046142578, + "learning_rate": 0.0001, + "loss": 57.79, + "step": 17050 + }, + { + "epoch": 27.472413966846375, + "grad_norm": 0.7800275683403015, + "learning_rate": 0.0001, + "loss": 60.0955, + "step": 17060 + }, + { + "epoch": 27.48853731042475, + "grad_norm": 0.5514053702354431, + "learning_rate": 0.0001, + "loss": 59.2458, + "step": 17070 + }, + { + "epoch": 27.504660654003125, + "grad_norm": 0.6667745113372803, + "learning_rate": 0.0001, + "loss": 59.872, + "step": 17080 + }, + { + "epoch": 27.5207839975815, + "grad_norm": 0.6694201231002808, + "learning_rate": 0.0001, + "loss": 59.4699, + "step": 17090 + }, + { + "epoch": 27.536907341159875, + "grad_norm": 0.721091091632843, + "learning_rate": 0.0001, + "loss": 59.8662, + "step": 17100 + }, + { + "epoch": 27.553030684738246, + "grad_norm": 0.8182216882705688, + "learning_rate": 0.0001, + "loss": 57.3634, + "step": 17110 + }, + { + "epoch": 27.56915402831662, + "grad_norm": 1.0078271627426147, + "learning_rate": 0.0001, + "loss": 59.3628, + "step": 17120 + }, + { + "epoch": 27.585277371894996, + "grad_norm": 0.6158314347267151, + "learning_rate": 0.0001, + "loss": 59.123, + "step": 17130 + }, + { + "epoch": 27.60140071547337, + "grad_norm": 0.5983971357345581, + "learning_rate": 0.0001, + "loss": 58.6259, + "step": 17140 + }, + { + "epoch": 27.617524059051746, + "grad_norm": 0.4854629337787628, + "learning_rate": 0.0001, + "loss": 58.781, + "step": 17150 + }, + { + "epoch": 27.63364740263012, + "grad_norm": 0.582637369632721, + "learning_rate": 0.0001, + "loss": 59.6194, + "step": 17160 + }, + { + "epoch": 27.649770746208496, + "grad_norm": 0.7129100561141968, + "learning_rate": 0.0001, + "loss": 58.495, + "step": 17170 + }, + { + "epoch": 27.66589408978687, + "grad_norm": 0.8179757595062256, + "learning_rate": 0.0001, + "loss": 58.7483, + "step": 17180 + }, + { + "epoch": 27.682017433365242, + "grad_norm": 0.7066773772239685, + "learning_rate": 0.0001, + "loss": 58.2444, + "step": 17190 + }, + { + "epoch": 27.698140776943617, + "grad_norm": 0.725228488445282, + "learning_rate": 0.0001, + "loss": 58.0686, + "step": 17200 + }, + { + "epoch": 27.714264120521992, + "grad_norm": 0.630705714225769, + "learning_rate": 0.0001, + "loss": 56.873, + "step": 17210 + }, + { + "epoch": 27.730387464100367, + "grad_norm": 0.7248587608337402, + "learning_rate": 0.0001, + "loss": 59.6893, + "step": 17220 + }, + { + "epoch": 27.746510807678742, + "grad_norm": 0.715254545211792, + "learning_rate": 0.0001, + "loss": 57.7157, + "step": 17230 + }, + { + "epoch": 27.762634151257117, + "grad_norm": 0.6525955200195312, + "learning_rate": 0.0001, + "loss": 58.6077, + "step": 17240 + }, + { + "epoch": 27.778757494835492, + "grad_norm": 0.7938500642776489, + "learning_rate": 0.0001, + "loss": 60.325, + "step": 17250 + }, + { + "epoch": 27.794880838413867, + "grad_norm": 0.6821472644805908, + "learning_rate": 0.0001, + "loss": 56.5372, + "step": 17260 + }, + { + "epoch": 27.811004181992242, + "grad_norm": 0.5398596525192261, + "learning_rate": 0.0001, + "loss": 57.433, + "step": 17270 + }, + { + "epoch": 27.827127525570614, + "grad_norm": 0.6805903911590576, + "learning_rate": 0.0001, + "loss": 58.6501, + "step": 17280 + }, + { + "epoch": 27.84325086914899, + "grad_norm": 0.6547013521194458, + "learning_rate": 0.0001, + "loss": 57.9083, + "step": 17290 + }, + { + "epoch": 27.859374212727364, + "grad_norm": 0.6482307314872742, + "learning_rate": 0.0001, + "loss": 59.2978, + "step": 17300 + }, + { + "epoch": 27.87549755630574, + "grad_norm": 0.867108941078186, + "learning_rate": 0.0001, + "loss": 59.452, + "step": 17310 + }, + { + "epoch": 27.891620899884114, + "grad_norm": 0.5809998512268066, + "learning_rate": 0.0001, + "loss": 59.0863, + "step": 17320 + }, + { + "epoch": 27.90774424346249, + "grad_norm": 0.7422227263450623, + "learning_rate": 0.0001, + "loss": 58.7768, + "step": 17330 + }, + { + "epoch": 27.923867587040863, + "grad_norm": 0.6092364192008972, + "learning_rate": 0.0001, + "loss": 58.1772, + "step": 17340 + }, + { + "epoch": 27.93999093061924, + "grad_norm": 0.688312292098999, + "learning_rate": 0.0001, + "loss": 58.5475, + "step": 17350 + }, + { + "epoch": 27.956114274197613, + "grad_norm": 0.8303641080856323, + "learning_rate": 0.0001, + "loss": 59.8533, + "step": 17360 + }, + { + "epoch": 27.972237617775985, + "grad_norm": 0.903464138507843, + "learning_rate": 0.0001, + "loss": 58.5677, + "step": 17370 + }, + { + "epoch": 27.98836096135436, + "grad_norm": 0.7864969968795776, + "learning_rate": 0.0001, + "loss": 59.1479, + "step": 17380 + }, + { + "epoch": 28.003224668715674, + "grad_norm": 0.7318345904350281, + "learning_rate": 0.0001, + "loss": 56.2894, + "step": 17390 + }, + { + "epoch": 28.01934801229405, + "grad_norm": 0.6740039587020874, + "learning_rate": 0.0001, + "loss": 60.3117, + "step": 17400 + }, + { + "epoch": 28.035471355872424, + "grad_norm": 0.6491691470146179, + "learning_rate": 0.0001, + "loss": 59.2796, + "step": 17410 + }, + { + "epoch": 28.0515946994508, + "grad_norm": 0.8077054619789124, + "learning_rate": 0.0001, + "loss": 60.4482, + "step": 17420 + }, + { + "epoch": 28.067718043029174, + "grad_norm": 0.8239777088165283, + "learning_rate": 0.0001, + "loss": 58.5352, + "step": 17430 + }, + { + "epoch": 28.08384138660755, + "grad_norm": 0.6907490491867065, + "learning_rate": 0.0001, + "loss": 60.3665, + "step": 17440 + }, + { + "epoch": 28.09996473018592, + "grad_norm": 0.7226580381393433, + "learning_rate": 0.0001, + "loss": 58.3535, + "step": 17450 + }, + { + "epoch": 28.116088073764296, + "grad_norm": 0.5712003707885742, + "learning_rate": 0.0001, + "loss": 57.8514, + "step": 17460 + }, + { + "epoch": 28.13221141734267, + "grad_norm": 0.532891571521759, + "learning_rate": 0.0001, + "loss": 57.5121, + "step": 17470 + }, + { + "epoch": 28.148334760921045, + "grad_norm": 0.6007943153381348, + "learning_rate": 0.0001, + "loss": 59.1072, + "step": 17480 + }, + { + "epoch": 28.16445810449942, + "grad_norm": 0.9194851517677307, + "learning_rate": 0.0001, + "loss": 59.8352, + "step": 17490 + }, + { + "epoch": 28.180581448077795, + "grad_norm": 0.5962793827056885, + "learning_rate": 0.0001, + "loss": 58.8552, + "step": 17500 + }, + { + "epoch": 28.19670479165617, + "grad_norm": 0.6472097635269165, + "learning_rate": 0.0001, + "loss": 60.072, + "step": 17510 + }, + { + "epoch": 28.212828135234545, + "grad_norm": 0.7954792976379395, + "learning_rate": 0.0001, + "loss": 57.754, + "step": 17520 + }, + { + "epoch": 28.22895147881292, + "grad_norm": 0.5923368334770203, + "learning_rate": 0.0001, + "loss": 56.9673, + "step": 17530 + }, + { + "epoch": 28.24507482239129, + "grad_norm": 1.0150458812713623, + "learning_rate": 0.0001, + "loss": 57.1309, + "step": 17540 + }, + { + "epoch": 28.261198165969667, + "grad_norm": 0.597544252872467, + "learning_rate": 0.0001, + "loss": 58.6526, + "step": 17550 + }, + { + "epoch": 28.27732150954804, + "grad_norm": 0.8652740716934204, + "learning_rate": 0.0001, + "loss": 58.7396, + "step": 17560 + }, + { + "epoch": 28.293444853126417, + "grad_norm": 0.8436110615730286, + "learning_rate": 0.0001, + "loss": 59.0186, + "step": 17570 + }, + { + "epoch": 28.30956819670479, + "grad_norm": 0.6380386352539062, + "learning_rate": 0.0001, + "loss": 58.3723, + "step": 17580 + }, + { + "epoch": 28.325691540283167, + "grad_norm": 0.6668462157249451, + "learning_rate": 0.0001, + "loss": 58.0738, + "step": 17590 + }, + { + "epoch": 28.34181488386154, + "grad_norm": 0.7173913717269897, + "learning_rate": 0.0001, + "loss": 58.1463, + "step": 17600 + }, + { + "epoch": 28.357938227439917, + "grad_norm": 0.6073333621025085, + "learning_rate": 0.0001, + "loss": 58.0279, + "step": 17610 + }, + { + "epoch": 28.37406157101829, + "grad_norm": 0.6092989444732666, + "learning_rate": 0.0001, + "loss": 56.1147, + "step": 17620 + }, + { + "epoch": 28.390184914596663, + "grad_norm": 0.7214272618293762, + "learning_rate": 0.0001, + "loss": 58.1375, + "step": 17630 + }, + { + "epoch": 28.406308258175038, + "grad_norm": 0.5026358366012573, + "learning_rate": 0.0001, + "loss": 59.2814, + "step": 17640 + }, + { + "epoch": 28.422431601753413, + "grad_norm": 0.6362181901931763, + "learning_rate": 0.0001, + "loss": 60.6815, + "step": 17650 + }, + { + "epoch": 28.438554945331788, + "grad_norm": 0.7994716167449951, + "learning_rate": 0.0001, + "loss": 58.4757, + "step": 17660 + }, + { + "epoch": 28.454678288910163, + "grad_norm": 0.6012625694274902, + "learning_rate": 0.0001, + "loss": 58.1216, + "step": 17670 + }, + { + "epoch": 28.470801632488538, + "grad_norm": 0.5765920877456665, + "learning_rate": 0.0001, + "loss": 60.1921, + "step": 17680 + }, + { + "epoch": 28.486924976066913, + "grad_norm": 0.6323729753494263, + "learning_rate": 0.0001, + "loss": 59.5405, + "step": 17690 + }, + { + "epoch": 28.503048319645288, + "grad_norm": 0.6641665697097778, + "learning_rate": 0.0001, + "loss": 59.3311, + "step": 17700 + }, + { + "epoch": 28.519171663223663, + "grad_norm": 0.7021269202232361, + "learning_rate": 0.0001, + "loss": 58.3317, + "step": 17710 + }, + { + "epoch": 28.535295006802034, + "grad_norm": 0.6319525241851807, + "learning_rate": 0.0001, + "loss": 57.4676, + "step": 17720 + }, + { + "epoch": 28.55141835038041, + "grad_norm": 0.6366249322891235, + "learning_rate": 0.0001, + "loss": 60.761, + "step": 17730 + }, + { + "epoch": 28.567541693958784, + "grad_norm": 1.487281322479248, + "learning_rate": 0.0001, + "loss": 58.2667, + "step": 17740 + }, + { + "epoch": 28.58366503753716, + "grad_norm": 0.6286079287528992, + "learning_rate": 0.0001, + "loss": 56.5199, + "step": 17750 + }, + { + "epoch": 28.599788381115534, + "grad_norm": 0.6458479762077332, + "learning_rate": 0.0001, + "loss": 59.0782, + "step": 17760 + }, + { + "epoch": 28.61591172469391, + "grad_norm": 0.6947740316390991, + "learning_rate": 0.0001, + "loss": 61.1563, + "step": 17770 + }, + { + "epoch": 28.632035068272284, + "grad_norm": 1.3151084184646606, + "learning_rate": 0.0001, + "loss": 58.7798, + "step": 17780 + }, + { + "epoch": 28.64815841185066, + "grad_norm": 0.5333471894264221, + "learning_rate": 0.0001, + "loss": 58.1223, + "step": 17790 + }, + { + "epoch": 28.66428175542903, + "grad_norm": 0.6268185973167419, + "learning_rate": 0.0001, + "loss": 58.9371, + "step": 17800 + }, + { + "epoch": 28.680405099007405, + "grad_norm": 0.5982078313827515, + "learning_rate": 0.0001, + "loss": 57.9883, + "step": 17810 + }, + { + "epoch": 28.69652844258578, + "grad_norm": 0.5744349360466003, + "learning_rate": 0.0001, + "loss": 59.4091, + "step": 17820 + }, + { + "epoch": 28.712651786164155, + "grad_norm": 0.7235150337219238, + "learning_rate": 0.0001, + "loss": 58.1419, + "step": 17830 + }, + { + "epoch": 28.72877512974253, + "grad_norm": 0.5630679726600647, + "learning_rate": 0.0001, + "loss": 59.547, + "step": 17840 + }, + { + "epoch": 28.744898473320905, + "grad_norm": 0.854364812374115, + "learning_rate": 0.0001, + "loss": 58.413, + "step": 17850 + }, + { + "epoch": 28.76102181689928, + "grad_norm": 0.6486361622810364, + "learning_rate": 0.0001, + "loss": 58.0385, + "step": 17860 + }, + { + "epoch": 28.777145160477655, + "grad_norm": 0.5967833995819092, + "learning_rate": 0.0001, + "loss": 57.4478, + "step": 17870 + }, + { + "epoch": 28.79326850405603, + "grad_norm": 0.6199812889099121, + "learning_rate": 0.0001, + "loss": 56.8104, + "step": 17880 + }, + { + "epoch": 28.8093918476344, + "grad_norm": 0.6966105699539185, + "learning_rate": 0.0001, + "loss": 59.4416, + "step": 17890 + }, + { + "epoch": 28.825515191212777, + "grad_norm": 0.7352570295333862, + "learning_rate": 0.0001, + "loss": 58.9475, + "step": 17900 + }, + { + "epoch": 28.84163853479115, + "grad_norm": 0.7693241238594055, + "learning_rate": 0.0001, + "loss": 57.4324, + "step": 17910 + }, + { + "epoch": 28.857761878369526, + "grad_norm": 1.0005452632904053, + "learning_rate": 0.0001, + "loss": 59.0267, + "step": 17920 + }, + { + "epoch": 28.8738852219479, + "grad_norm": 0.6526029109954834, + "learning_rate": 0.0001, + "loss": 57.7836, + "step": 17930 + }, + { + "epoch": 28.890008565526276, + "grad_norm": 0.7833093404769897, + "learning_rate": 0.0001, + "loss": 59.3319, + "step": 17940 + }, + { + "epoch": 28.90613190910465, + "grad_norm": 0.6504502892494202, + "learning_rate": 0.0001, + "loss": 57.5452, + "step": 17950 + }, + { + "epoch": 28.922255252683026, + "grad_norm": 0.5760630369186401, + "learning_rate": 0.0001, + "loss": 56.2115, + "step": 17960 + }, + { + "epoch": 28.9383785962614, + "grad_norm": 0.7640287280082703, + "learning_rate": 0.0001, + "loss": 59.051, + "step": 17970 + }, + { + "epoch": 28.954501939839773, + "grad_norm": 0.6596423983573914, + "learning_rate": 0.0001, + "loss": 59.4821, + "step": 17980 + }, + { + "epoch": 28.970625283418148, + "grad_norm": 2.1421217918395996, + "learning_rate": 0.0001, + "loss": 58.2994, + "step": 17990 + }, + { + "epoch": 28.986748626996523, + "grad_norm": 0.6420243382453918, + "learning_rate": 0.0001, + "loss": 55.9762, + "step": 18000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 33, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.2014235447546138e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ade09d5233289ba4bc5076af39b8652078582dee --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016a4e9b5d71a1947317d3801b00fe5d0604edbef1f981d7b6c92f219131717c +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-18000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/latest new file mode 100644 index 0000000000000000000000000000000000000000..2a79fdc19587e6bc9de060e90633f3a151b04516 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/latest @@ -0,0 +1 @@ +global_step2000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0f01bd0f43fd90dac08a0d83d5a6fbed8d99928d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ca3e66eb854dcbacc7672f45336c3e78e276d70a503b2f27d6e6fbbfe2aa483 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..3ee9682fa2299dbaa7f34f31df5379d8fe8028dd --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e72cad5a34d0d13cbf22d07143ddc771ea88322958e473d370e8ab26dff9c76c +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..7048a12cad436be2ea962ff738dff592d8e6e4dc --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a66ca6bb7f7a9732ca84845b3653b8aa8898048e1ed744163d01d74390e451b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e269dbe34d193bc38cab8226dfb7a1ce4642a74d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44195fe47a712ac4188cd36c3240b76f21d08da5439e934793d315e89bf304ff +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..54666ce27669dfe972d3ed9c745c4f92a73b12c7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70dcec40123739ff4d914b83911dcd6947f902d85432bdd48e180101a19a5aba +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..d9628e83a4c4937af993a0613d587d373e4fec5a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de923defade06d5d2d6eb3b881130ee5925212155d3ec13834dfe40816beb053 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..cdfe1aff1845c7c498dfed9a61491b2b6d7c9653 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:604b58c0fb4fae5e68d2dab753b09c50ed2091771de66b89fe0e4e745b34df5b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..344321a12821ccea35c6c68f278d26692486c39b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec1811f4c7211db9b990692cee7800b1409bcd0d89f4e90047231cc2ff1ad727 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..eaf7d10b476947489be323bc02d89e4672f7cb51 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcb51293911a28f40faeed9773e017920d8ce6e02217b791fd276d7d07b581f0 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..059097d31013bcb853c1f29bd20164bc1deb61d8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6a4e6ea22558ae7a3d868dc31596451ac2fbd9303bffcf03fa9584c5490e3ca +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..cfa2bd4d7f44b0725080cdad9964a5c7114f2b95 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/trainer_state.json @@ -0,0 +1,1434 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.2208898070237315, + "eval_steps": 2000, + "global_step": 2000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.016123343578374565, + "grad_norm": 48.11856460571289, + "learning_rate": 0.0001, + "loss": 299.1088, + "step": 10 + }, + { + "epoch": 0.03224668715674913, + "grad_norm": 9.07711410522461, + "learning_rate": 0.0001, + "loss": 234.0888, + "step": 20 + }, + { + "epoch": 0.0483700307351237, + "grad_norm": 3.5935330390930176, + "learning_rate": 0.0001, + "loss": 231.4619, + "step": 30 + }, + { + "epoch": 0.06449337431349826, + "grad_norm": 3.5784752368927, + "learning_rate": 0.0001, + "loss": 229.7082, + "step": 40 + }, + { + "epoch": 0.08061671789187283, + "grad_norm": 3.5757956504821777, + "learning_rate": 0.0001, + "loss": 230.1062, + "step": 50 + }, + { + "epoch": 0.0967400614702474, + "grad_norm": 3.8763978481292725, + "learning_rate": 0.0001, + "loss": 229.1914, + "step": 60 + }, + { + "epoch": 0.11286340504862195, + "grad_norm": 3.006988525390625, + "learning_rate": 0.0001, + "loss": 229.6482, + "step": 70 + }, + { + "epoch": 0.12898674862699652, + "grad_norm": 2.3527095317840576, + "learning_rate": 0.0001, + "loss": 228.1141, + "step": 80 + }, + { + "epoch": 0.14511009220537108, + "grad_norm": 5.353086471557617, + "learning_rate": 0.0001, + "loss": 228.1448, + "step": 90 + }, + { + "epoch": 0.16123343578374566, + "grad_norm": 2.251854419708252, + "learning_rate": 0.0001, + "loss": 228.3897, + "step": 100 + }, + { + "epoch": 0.17735677936212021, + "grad_norm": 1.5789915323257446, + "learning_rate": 0.0001, + "loss": 227.7662, + "step": 110 + }, + { + "epoch": 0.1934801229404948, + "grad_norm": 3.7646398544311523, + "learning_rate": 0.0001, + "loss": 223.6411, + "step": 120 + }, + { + "epoch": 0.20960346651886935, + "grad_norm": 3.444000720977783, + "learning_rate": 0.0001, + "loss": 221.2073, + "step": 130 + }, + { + "epoch": 0.2257268100972439, + "grad_norm": 2.1871185302734375, + "learning_rate": 0.0001, + "loss": 215.9944, + "step": 140 + }, + { + "epoch": 0.2418501536756185, + "grad_norm": 2.266244411468506, + "learning_rate": 0.0001, + "loss": 212.3066, + "step": 150 + }, + { + "epoch": 0.25797349725399304, + "grad_norm": 2.043832778930664, + "learning_rate": 0.0001, + "loss": 208.1343, + "step": 160 + }, + { + "epoch": 0.2740968408323676, + "grad_norm": 2.2641494274139404, + "learning_rate": 0.0001, + "loss": 203.1485, + "step": 170 + }, + { + "epoch": 0.29022018441074215, + "grad_norm": 2.9727790355682373, + "learning_rate": 0.0001, + "loss": 198.181, + "step": 180 + }, + { + "epoch": 0.30634352798911674, + "grad_norm": 2.751523494720459, + "learning_rate": 0.0001, + "loss": 194.8422, + "step": 190 + }, + { + "epoch": 0.3224668715674913, + "grad_norm": 1.9561562538146973, + "learning_rate": 0.0001, + "loss": 189.9401, + "step": 200 + }, + { + "epoch": 0.33859021514586585, + "grad_norm": 2.3264107704162598, + "learning_rate": 0.0001, + "loss": 186.1309, + "step": 210 + }, + { + "epoch": 0.35471355872424043, + "grad_norm": 5.212426662445068, + "learning_rate": 0.0001, + "loss": 182.2255, + "step": 220 + }, + { + "epoch": 0.370836902302615, + "grad_norm": 2.488032102584839, + "learning_rate": 0.0001, + "loss": 181.0103, + "step": 230 + }, + { + "epoch": 0.3869602458809896, + "grad_norm": 2.6173202991485596, + "learning_rate": 0.0001, + "loss": 177.6162, + "step": 240 + }, + { + "epoch": 0.4030835894593641, + "grad_norm": 2.6245291233062744, + "learning_rate": 0.0001, + "loss": 173.0431, + "step": 250 + }, + { + "epoch": 0.4192069330377387, + "grad_norm": 5.356006145477295, + "learning_rate": 0.0001, + "loss": 172.2371, + "step": 260 + }, + { + "epoch": 0.4353302766161133, + "grad_norm": 2.0453829765319824, + "learning_rate": 0.0001, + "loss": 169.7781, + "step": 270 + }, + { + "epoch": 0.4514536201944878, + "grad_norm": 1.7002843618392944, + "learning_rate": 0.0001, + "loss": 166.9564, + "step": 280 + }, + { + "epoch": 0.4675769637728624, + "grad_norm": 2.181410312652588, + "learning_rate": 0.0001, + "loss": 166.0094, + "step": 290 + }, + { + "epoch": 0.483700307351237, + "grad_norm": 2.2948265075683594, + "learning_rate": 0.0001, + "loss": 162.4388, + "step": 300 + }, + { + "epoch": 0.4998236509296115, + "grad_norm": 3.0272789001464844, + "learning_rate": 0.0001, + "loss": 160.3446, + "step": 310 + }, + { + "epoch": 0.5159469945079861, + "grad_norm": 1.948184847831726, + "learning_rate": 0.0001, + "loss": 158.488, + "step": 320 + }, + { + "epoch": 0.5320703380863606, + "grad_norm": 1.7981538772583008, + "learning_rate": 0.0001, + "loss": 156.2107, + "step": 330 + }, + { + "epoch": 0.5481936816647353, + "grad_norm": 2.049323797225952, + "learning_rate": 0.0001, + "loss": 155.5434, + "step": 340 + }, + { + "epoch": 0.5643170252431098, + "grad_norm": 1.9415442943572998, + "learning_rate": 0.0001, + "loss": 153.8682, + "step": 350 + }, + { + "epoch": 0.5804403688214843, + "grad_norm": 2.2168045043945312, + "learning_rate": 0.0001, + "loss": 150.4087, + "step": 360 + }, + { + "epoch": 0.596563712399859, + "grad_norm": 1.510750651359558, + "learning_rate": 0.0001, + "loss": 151.7655, + "step": 370 + }, + { + "epoch": 0.6126870559782335, + "grad_norm": 2.7469751834869385, + "learning_rate": 0.0001, + "loss": 147.4665, + "step": 380 + }, + { + "epoch": 0.628810399556608, + "grad_norm": 2.0436975955963135, + "learning_rate": 0.0001, + "loss": 147.4405, + "step": 390 + }, + { + "epoch": 0.6449337431349826, + "grad_norm": 1.8219181299209595, + "learning_rate": 0.0001, + "loss": 146.6332, + "step": 400 + }, + { + "epoch": 0.6610570867133572, + "grad_norm": 1.948431134223938, + "learning_rate": 0.0001, + "loss": 143.3298, + "step": 410 + }, + { + "epoch": 0.6771804302917317, + "grad_norm": 1.8887908458709717, + "learning_rate": 0.0001, + "loss": 140.5058, + "step": 420 + }, + { + "epoch": 0.6933037738701063, + "grad_norm": 1.8739320039749146, + "learning_rate": 0.0001, + "loss": 137.6334, + "step": 430 + }, + { + "epoch": 0.7094271174484809, + "grad_norm": 2.057689666748047, + "learning_rate": 0.0001, + "loss": 135.0343, + "step": 440 + }, + { + "epoch": 0.7255504610268555, + "grad_norm": 1.6926478147506714, + "learning_rate": 0.0001, + "loss": 132.8766, + "step": 450 + }, + { + "epoch": 0.74167380460523, + "grad_norm": 1.5039501190185547, + "learning_rate": 0.0001, + "loss": 131.6774, + "step": 460 + }, + { + "epoch": 0.7577971481836046, + "grad_norm": 1.6903547048568726, + "learning_rate": 0.0001, + "loss": 129.3116, + "step": 470 + }, + { + "epoch": 0.7739204917619792, + "grad_norm": 3.7062244415283203, + "learning_rate": 0.0001, + "loss": 127.9314, + "step": 480 + }, + { + "epoch": 0.7900438353403537, + "grad_norm": 2.0586934089660645, + "learning_rate": 0.0001, + "loss": 126.6398, + "step": 490 + }, + { + "epoch": 0.8061671789187282, + "grad_norm": 1.7598215341567993, + "learning_rate": 0.0001, + "loss": 125.1501, + "step": 500 + }, + { + "epoch": 0.8222905224971029, + "grad_norm": 1.5992189645767212, + "learning_rate": 0.0001, + "loss": 124.7039, + "step": 510 + }, + { + "epoch": 0.8384138660754774, + "grad_norm": 2.8735458850860596, + "learning_rate": 0.0001, + "loss": 122.1619, + "step": 520 + }, + { + "epoch": 0.8545372096538519, + "grad_norm": 1.867098093032837, + "learning_rate": 0.0001, + "loss": 122.4506, + "step": 530 + }, + { + "epoch": 0.8706605532322266, + "grad_norm": 2.2088418006896973, + "learning_rate": 0.0001, + "loss": 122.1047, + "step": 540 + }, + { + "epoch": 0.8867838968106011, + "grad_norm": 1.642657995223999, + "learning_rate": 0.0001, + "loss": 120.1141, + "step": 550 + }, + { + "epoch": 0.9029072403889756, + "grad_norm": 1.5567114353179932, + "learning_rate": 0.0001, + "loss": 118.532, + "step": 560 + }, + { + "epoch": 0.9190305839673503, + "grad_norm": 1.667843222618103, + "learning_rate": 0.0001, + "loss": 118.0734, + "step": 570 + }, + { + "epoch": 0.9351539275457248, + "grad_norm": 1.5304442644119263, + "learning_rate": 0.0001, + "loss": 114.4925, + "step": 580 + }, + { + "epoch": 0.9512772711240993, + "grad_norm": 1.6920255422592163, + "learning_rate": 0.0001, + "loss": 114.0729, + "step": 590 + }, + { + "epoch": 0.967400614702474, + "grad_norm": 1.6313318014144897, + "learning_rate": 0.0001, + "loss": 114.2879, + "step": 600 + }, + { + "epoch": 0.9835239582808485, + "grad_norm": 1.9033657312393188, + "learning_rate": 0.0001, + "loss": 110.5299, + "step": 610 + }, + { + "epoch": 0.999647301859223, + "grad_norm": 1.9010754823684692, + "learning_rate": 0.0001, + "loss": 109.1145, + "step": 620 + }, + { + "epoch": 1.0145110092205372, + "grad_norm": 3.5390989780426025, + "learning_rate": 0.0001, + "loss": 101.0258, + "step": 630 + }, + { + "epoch": 1.0306343527989117, + "grad_norm": 2.2925171852111816, + "learning_rate": 0.0001, + "loss": 107.3693, + "step": 640 + }, + { + "epoch": 1.0467576963772862, + "grad_norm": 2.329108238220215, + "learning_rate": 0.0001, + "loss": 104.3778, + "step": 650 + }, + { + "epoch": 1.0628810399556607, + "grad_norm": 1.7325661182403564, + "learning_rate": 0.0001, + "loss": 102.8409, + "step": 660 + }, + { + "epoch": 1.0790043835340353, + "grad_norm": 1.989586353302002, + "learning_rate": 0.0001, + "loss": 104.2577, + "step": 670 + }, + { + "epoch": 1.09512772711241, + "grad_norm": 1.900484323501587, + "learning_rate": 0.0001, + "loss": 102.3051, + "step": 680 + }, + { + "epoch": 1.1112510706907845, + "grad_norm": 2.015859842300415, + "learning_rate": 0.0001, + "loss": 100.893, + "step": 690 + }, + { + "epoch": 1.127374414269159, + "grad_norm": 1.7695508003234863, + "learning_rate": 0.0001, + "loss": 99.215, + "step": 700 + }, + { + "epoch": 1.1434977578475336, + "grad_norm": 2.025923490524292, + "learning_rate": 0.0001, + "loss": 98.8768, + "step": 710 + }, + { + "epoch": 1.1596211014259081, + "grad_norm": 2.946708917617798, + "learning_rate": 0.0001, + "loss": 96.5588, + "step": 720 + }, + { + "epoch": 1.1757444450042827, + "grad_norm": 2.4511258602142334, + "learning_rate": 0.0001, + "loss": 95.0545, + "step": 730 + }, + { + "epoch": 1.1918677885826574, + "grad_norm": 2.3094682693481445, + "learning_rate": 0.0001, + "loss": 97.004, + "step": 740 + }, + { + "epoch": 1.207991132161032, + "grad_norm": 2.1313347816467285, + "learning_rate": 0.0001, + "loss": 94.8819, + "step": 750 + }, + { + "epoch": 1.2241144757394065, + "grad_norm": 1.614182710647583, + "learning_rate": 0.0001, + "loss": 95.5339, + "step": 760 + }, + { + "epoch": 1.240237819317781, + "grad_norm": 2.029953956604004, + "learning_rate": 0.0001, + "loss": 93.7303, + "step": 770 + }, + { + "epoch": 1.2563611628961555, + "grad_norm": 1.7871663570404053, + "learning_rate": 0.0001, + "loss": 93.8354, + "step": 780 + }, + { + "epoch": 1.2724845064745303, + "grad_norm": 1.72455632686615, + "learning_rate": 0.0001, + "loss": 94.7348, + "step": 790 + }, + { + "epoch": 1.2886078500529048, + "grad_norm": 1.9776639938354492, + "learning_rate": 0.0001, + "loss": 93.5957, + "step": 800 + }, + { + "epoch": 1.3047311936312793, + "grad_norm": 1.7187533378601074, + "learning_rate": 0.0001, + "loss": 92.9583, + "step": 810 + }, + { + "epoch": 1.3208545372096538, + "grad_norm": 1.5949162244796753, + "learning_rate": 0.0001, + "loss": 93.3088, + "step": 820 + }, + { + "epoch": 1.3369778807880284, + "grad_norm": 1.4616206884384155, + "learning_rate": 0.0001, + "loss": 93.2125, + "step": 830 + }, + { + "epoch": 1.353101224366403, + "grad_norm": 2.832163095474243, + "learning_rate": 0.0001, + "loss": 90.3301, + "step": 840 + }, + { + "epoch": 1.3692245679447774, + "grad_norm": 2.174501419067383, + "learning_rate": 0.0001, + "loss": 91.6988, + "step": 850 + }, + { + "epoch": 1.3853479115231522, + "grad_norm": 1.2156339883804321, + "learning_rate": 0.0001, + "loss": 92.7419, + "step": 860 + }, + { + "epoch": 1.4014712551015267, + "grad_norm": 1.608261227607727, + "learning_rate": 0.0001, + "loss": 89.8255, + "step": 870 + }, + { + "epoch": 1.4175945986799012, + "grad_norm": 1.6671923398971558, + "learning_rate": 0.0001, + "loss": 90.9683, + "step": 880 + }, + { + "epoch": 1.4337179422582758, + "grad_norm": 1.2346433401107788, + "learning_rate": 0.0001, + "loss": 90.6092, + "step": 890 + }, + { + "epoch": 1.4498412858366505, + "grad_norm": 1.561085820198059, + "learning_rate": 0.0001, + "loss": 88.0842, + "step": 900 + }, + { + "epoch": 1.465964629415025, + "grad_norm": 1.5475201606750488, + "learning_rate": 0.0001, + "loss": 89.8786, + "step": 910 + }, + { + "epoch": 1.4820879729933996, + "grad_norm": 1.6044800281524658, + "learning_rate": 0.0001, + "loss": 88.0364, + "step": 920 + }, + { + "epoch": 1.498211316571774, + "grad_norm": 1.4081908464431763, + "learning_rate": 0.0001, + "loss": 88.8799, + "step": 930 + }, + { + "epoch": 1.5143346601501486, + "grad_norm": 2.341053009033203, + "learning_rate": 0.0001, + "loss": 88.2023, + "step": 940 + }, + { + "epoch": 1.5304580037285231, + "grad_norm": 1.3976799249649048, + "learning_rate": 0.0001, + "loss": 86.6493, + "step": 950 + }, + { + "epoch": 1.5465813473068977, + "grad_norm": 1.2485483884811401, + "learning_rate": 0.0001, + "loss": 88.9224, + "step": 960 + }, + { + "epoch": 1.5627046908852722, + "grad_norm": 1.6824235916137695, + "learning_rate": 0.0001, + "loss": 89.6484, + "step": 970 + }, + { + "epoch": 1.578828034463647, + "grad_norm": 1.64790940284729, + "learning_rate": 0.0001, + "loss": 87.2479, + "step": 980 + }, + { + "epoch": 1.5949513780420215, + "grad_norm": 1.2093443870544434, + "learning_rate": 0.0001, + "loss": 86.8241, + "step": 990 + }, + { + "epoch": 1.611074721620396, + "grad_norm": 1.5306812524795532, + "learning_rate": 0.0001, + "loss": 88.0006, + "step": 1000 + }, + { + "epoch": 1.6271980651987707, + "grad_norm": 1.2465763092041016, + "learning_rate": 0.0001, + "loss": 86.8143, + "step": 1010 + }, + { + "epoch": 1.6433214087771453, + "grad_norm": 1.5224069356918335, + "learning_rate": 0.0001, + "loss": 87.0102, + "step": 1020 + }, + { + "epoch": 1.6594447523555198, + "grad_norm": 1.312179684638977, + "learning_rate": 0.0001, + "loss": 88.7395, + "step": 1030 + }, + { + "epoch": 1.6755680959338943, + "grad_norm": 1.2343158721923828, + "learning_rate": 0.0001, + "loss": 87.2868, + "step": 1040 + }, + { + "epoch": 1.6916914395122689, + "grad_norm": 1.2778265476226807, + "learning_rate": 0.0001, + "loss": 87.043, + "step": 1050 + }, + { + "epoch": 1.7078147830906434, + "grad_norm": 1.2799627780914307, + "learning_rate": 0.0001, + "loss": 87.7843, + "step": 1060 + }, + { + "epoch": 1.723938126669018, + "grad_norm": 1.3688952922821045, + "learning_rate": 0.0001, + "loss": 85.8985, + "step": 1070 + }, + { + "epoch": 1.7400614702473924, + "grad_norm": 1.4317796230316162, + "learning_rate": 0.0001, + "loss": 85.5408, + "step": 1080 + }, + { + "epoch": 1.756184813825767, + "grad_norm": 1.164801001548767, + "learning_rate": 0.0001, + "loss": 85.8484, + "step": 1090 + }, + { + "epoch": 1.7723081574041417, + "grad_norm": 1.3929173946380615, + "learning_rate": 0.0001, + "loss": 86.9031, + "step": 1100 + }, + { + "epoch": 1.7884315009825162, + "grad_norm": 2.116119861602783, + "learning_rate": 0.0001, + "loss": 86.5343, + "step": 1110 + }, + { + "epoch": 1.8045548445608908, + "grad_norm": 1.4117276668548584, + "learning_rate": 0.0001, + "loss": 85.2678, + "step": 1120 + }, + { + "epoch": 1.8206781881392655, + "grad_norm": 1.3213504552841187, + "learning_rate": 0.0001, + "loss": 85.8935, + "step": 1130 + }, + { + "epoch": 1.83680153171764, + "grad_norm": 1.3589770793914795, + "learning_rate": 0.0001, + "loss": 85.032, + "step": 1140 + }, + { + "epoch": 1.8529248752960146, + "grad_norm": 1.3336007595062256, + "learning_rate": 0.0001, + "loss": 84.0993, + "step": 1150 + }, + { + "epoch": 1.869048218874389, + "grad_norm": 1.1081451177597046, + "learning_rate": 0.0001, + "loss": 84.2225, + "step": 1160 + }, + { + "epoch": 1.8851715624527636, + "grad_norm": 1.482038974761963, + "learning_rate": 0.0001, + "loss": 82.1439, + "step": 1170 + }, + { + "epoch": 1.9012949060311382, + "grad_norm": 1.1571478843688965, + "learning_rate": 0.0001, + "loss": 82.5045, + "step": 1180 + }, + { + "epoch": 1.9174182496095127, + "grad_norm": 1.2665468454360962, + "learning_rate": 0.0001, + "loss": 82.8761, + "step": 1190 + }, + { + "epoch": 1.9335415931878872, + "grad_norm": 1.0771509408950806, + "learning_rate": 0.0001, + "loss": 81.5162, + "step": 1200 + }, + { + "epoch": 1.949664936766262, + "grad_norm": 1.2992299795150757, + "learning_rate": 0.0001, + "loss": 82.4038, + "step": 1210 + }, + { + "epoch": 1.9657882803446365, + "grad_norm": 1.2946668863296509, + "learning_rate": 0.0001, + "loss": 83.4837, + "step": 1220 + }, + { + "epoch": 1.981911623923011, + "grad_norm": 1.3505686521530151, + "learning_rate": 0.0001, + "loss": 81.8993, + "step": 1230 + }, + { + "epoch": 1.9980349675013858, + "grad_norm": 1.1247715950012207, + "learning_rate": 0.0001, + "loss": 82.9006, + "step": 1240 + }, + { + "epoch": 2.0128986748627, + "grad_norm": 1.5652127265930176, + "learning_rate": 0.0001, + "loss": 75.8347, + "step": 1250 + }, + { + "epoch": 2.0290220184410743, + "grad_norm": 7.311964988708496, + "learning_rate": 0.0001, + "loss": 80.5435, + "step": 1260 + }, + { + "epoch": 2.045145362019449, + "grad_norm": 1.2229893207550049, + "learning_rate": 0.0001, + "loss": 83.3277, + "step": 1270 + }, + { + "epoch": 2.0612687055978234, + "grad_norm": 1.1387693881988525, + "learning_rate": 0.0001, + "loss": 81.3252, + "step": 1280 + }, + { + "epoch": 2.077392049176198, + "grad_norm": 1.307201623916626, + "learning_rate": 0.0001, + "loss": 82.3311, + "step": 1290 + }, + { + "epoch": 2.0935153927545724, + "grad_norm": 1.1502896547317505, + "learning_rate": 0.0001, + "loss": 81.1682, + "step": 1300 + }, + { + "epoch": 2.109638736332947, + "grad_norm": 0.9291583299636841, + "learning_rate": 0.0001, + "loss": 81.7017, + "step": 1310 + }, + { + "epoch": 2.1257620799113215, + "grad_norm": 1.0334666967391968, + "learning_rate": 0.0001, + "loss": 81.718, + "step": 1320 + }, + { + "epoch": 2.141885423489696, + "grad_norm": 1.1099056005477905, + "learning_rate": 0.0001, + "loss": 81.2232, + "step": 1330 + }, + { + "epoch": 2.1580087670680705, + "grad_norm": 1.289478063583374, + "learning_rate": 0.0001, + "loss": 81.8736, + "step": 1340 + }, + { + "epoch": 2.174132110646445, + "grad_norm": 1.1634794473648071, + "learning_rate": 0.0001, + "loss": 79.9434, + "step": 1350 + }, + { + "epoch": 2.19025545422482, + "grad_norm": 0.9563440680503845, + "learning_rate": 0.0001, + "loss": 80.628, + "step": 1360 + }, + { + "epoch": 2.2063787978031946, + "grad_norm": 1.1268844604492188, + "learning_rate": 0.0001, + "loss": 82.1389, + "step": 1370 + }, + { + "epoch": 2.222502141381569, + "grad_norm": 1.1050443649291992, + "learning_rate": 0.0001, + "loss": 78.9023, + "step": 1380 + }, + { + "epoch": 2.2386254849599436, + "grad_norm": 1.1936142444610596, + "learning_rate": 0.0001, + "loss": 81.2118, + "step": 1390 + }, + { + "epoch": 2.254748828538318, + "grad_norm": 1.7825465202331543, + "learning_rate": 0.0001, + "loss": 80.8804, + "step": 1400 + }, + { + "epoch": 2.2708721721166927, + "grad_norm": 2.437251329421997, + "learning_rate": 0.0001, + "loss": 79.3807, + "step": 1410 + }, + { + "epoch": 2.286995515695067, + "grad_norm": 1.0604854822158813, + "learning_rate": 0.0001, + "loss": 79.5881, + "step": 1420 + }, + { + "epoch": 2.3031188592734417, + "grad_norm": 1.2285529375076294, + "learning_rate": 0.0001, + "loss": 79.5517, + "step": 1430 + }, + { + "epoch": 2.3192422028518163, + "grad_norm": 1.1916660070419312, + "learning_rate": 0.0001, + "loss": 80.5887, + "step": 1440 + }, + { + "epoch": 2.335365546430191, + "grad_norm": 1.224304437637329, + "learning_rate": 0.0001, + "loss": 79.8109, + "step": 1450 + }, + { + "epoch": 2.3514888900085653, + "grad_norm": 0.9228448867797852, + "learning_rate": 0.0001, + "loss": 79.2787, + "step": 1460 + }, + { + "epoch": 2.3676122335869403, + "grad_norm": 1.2085226774215698, + "learning_rate": 0.0001, + "loss": 78.4983, + "step": 1470 + }, + { + "epoch": 2.383735577165315, + "grad_norm": 2.424083948135376, + "learning_rate": 0.0001, + "loss": 79.8657, + "step": 1480 + }, + { + "epoch": 2.3998589207436893, + "grad_norm": 1.0277849435806274, + "learning_rate": 0.0001, + "loss": 80.6915, + "step": 1490 + }, + { + "epoch": 2.415982264322064, + "grad_norm": 1.0226075649261475, + "learning_rate": 0.0001, + "loss": 78.9751, + "step": 1500 + }, + { + "epoch": 2.4321056079004384, + "grad_norm": 0.8860388398170471, + "learning_rate": 0.0001, + "loss": 78.3915, + "step": 1510 + }, + { + "epoch": 2.448228951478813, + "grad_norm": 1.265637755393982, + "learning_rate": 0.0001, + "loss": 78.8737, + "step": 1520 + }, + { + "epoch": 2.4643522950571874, + "grad_norm": 1.2518186569213867, + "learning_rate": 0.0001, + "loss": 76.5169, + "step": 1530 + }, + { + "epoch": 2.480475638635562, + "grad_norm": 1.074277400970459, + "learning_rate": 0.0001, + "loss": 77.9156, + "step": 1540 + }, + { + "epoch": 2.4965989822139365, + "grad_norm": 1.0666711330413818, + "learning_rate": 0.0001, + "loss": 78.4103, + "step": 1550 + }, + { + "epoch": 2.512722325792311, + "grad_norm": 0.9492248892784119, + "learning_rate": 0.0001, + "loss": 76.7393, + "step": 1560 + }, + { + "epoch": 2.5288456693706856, + "grad_norm": 1.127238154411316, + "learning_rate": 0.0001, + "loss": 79.729, + "step": 1570 + }, + { + "epoch": 2.5449690129490605, + "grad_norm": 1.1520992517471313, + "learning_rate": 0.0001, + "loss": 78.5618, + "step": 1580 + }, + { + "epoch": 2.5610923565274346, + "grad_norm": 0.8632035255432129, + "learning_rate": 0.0001, + "loss": 79.032, + "step": 1590 + }, + { + "epoch": 2.5772157001058096, + "grad_norm": 1.0506329536437988, + "learning_rate": 0.0001, + "loss": 77.6204, + "step": 1600 + }, + { + "epoch": 2.593339043684184, + "grad_norm": 1.019895315170288, + "learning_rate": 0.0001, + "loss": 79.2634, + "step": 1610 + }, + { + "epoch": 2.6094623872625586, + "grad_norm": 1.1685571670532227, + "learning_rate": 0.0001, + "loss": 78.8371, + "step": 1620 + }, + { + "epoch": 2.625585730840933, + "grad_norm": 1.147546648979187, + "learning_rate": 0.0001, + "loss": 78.2113, + "step": 1630 + }, + { + "epoch": 2.6417090744193077, + "grad_norm": 1.7336841821670532, + "learning_rate": 0.0001, + "loss": 78.7866, + "step": 1640 + }, + { + "epoch": 2.657832417997682, + "grad_norm": 1.2150614261627197, + "learning_rate": 0.0001, + "loss": 78.5621, + "step": 1650 + }, + { + "epoch": 2.6739557615760567, + "grad_norm": 1.2257503271102905, + "learning_rate": 0.0001, + "loss": 77.5835, + "step": 1660 + }, + { + "epoch": 2.6900791051544313, + "grad_norm": 1.0375311374664307, + "learning_rate": 0.0001, + "loss": 77.7509, + "step": 1670 + }, + { + "epoch": 2.706202448732806, + "grad_norm": 1.2551004886627197, + "learning_rate": 0.0001, + "loss": 77.8615, + "step": 1680 + }, + { + "epoch": 2.7223257923111808, + "grad_norm": 1.0464223623275757, + "learning_rate": 0.0001, + "loss": 77.2837, + "step": 1690 + }, + { + "epoch": 2.738449135889555, + "grad_norm": 1.0254806280136108, + "learning_rate": 0.0001, + "loss": 76.9616, + "step": 1700 + }, + { + "epoch": 2.75457247946793, + "grad_norm": 1.1665682792663574, + "learning_rate": 0.0001, + "loss": 77.9108, + "step": 1710 + }, + { + "epoch": 2.7706958230463044, + "grad_norm": 1.1349453926086426, + "learning_rate": 0.0001, + "loss": 78.7636, + "step": 1720 + }, + { + "epoch": 2.786819166624679, + "grad_norm": 1.4961057901382446, + "learning_rate": 0.0001, + "loss": 76.3862, + "step": 1730 + }, + { + "epoch": 2.8029425102030534, + "grad_norm": 1.10721755027771, + "learning_rate": 0.0001, + "loss": 77.4105, + "step": 1740 + }, + { + "epoch": 2.819065853781428, + "grad_norm": 0.9878576397895813, + "learning_rate": 0.0001, + "loss": 78.4855, + "step": 1750 + }, + { + "epoch": 2.8351891973598025, + "grad_norm": 1.1722687482833862, + "learning_rate": 0.0001, + "loss": 75.1589, + "step": 1760 + }, + { + "epoch": 2.851312540938177, + "grad_norm": 1.6624362468719482, + "learning_rate": 0.0001, + "loss": 77.1647, + "step": 1770 + }, + { + "epoch": 2.8674358845165515, + "grad_norm": 0.9542295932769775, + "learning_rate": 0.0001, + "loss": 77.1656, + "step": 1780 + }, + { + "epoch": 2.883559228094926, + "grad_norm": 2.328542947769165, + "learning_rate": 0.0001, + "loss": 76.4686, + "step": 1790 + }, + { + "epoch": 2.899682571673301, + "grad_norm": 1.2532029151916504, + "learning_rate": 0.0001, + "loss": 76.1727, + "step": 1800 + }, + { + "epoch": 2.915805915251675, + "grad_norm": 0.9604716897010803, + "learning_rate": 0.0001, + "loss": 74.3132, + "step": 1810 + }, + { + "epoch": 2.93192925883005, + "grad_norm": 0.9182595610618591, + "learning_rate": 0.0001, + "loss": 77.8689, + "step": 1820 + }, + { + "epoch": 2.9480526024084246, + "grad_norm": 0.9623042941093445, + "learning_rate": 0.0001, + "loss": 77.7531, + "step": 1830 + }, + { + "epoch": 2.964175945986799, + "grad_norm": 1.0500119924545288, + "learning_rate": 0.0001, + "loss": 77.6007, + "step": 1840 + }, + { + "epoch": 2.9802992895651736, + "grad_norm": 1.0874103307724, + "learning_rate": 0.0001, + "loss": 77.042, + "step": 1850 + }, + { + "epoch": 2.996422633143548, + "grad_norm": 0.9955475330352783, + "learning_rate": 0.0001, + "loss": 75.468, + "step": 1860 + }, + { + "epoch": 3.011286340504862, + "grad_norm": 1.4281760454177856, + "learning_rate": 0.0001, + "loss": 69.5972, + "step": 1870 + }, + { + "epoch": 3.0274096840832367, + "grad_norm": 1.0292123556137085, + "learning_rate": 0.0001, + "loss": 76.0186, + "step": 1880 + }, + { + "epoch": 3.0435330276616113, + "grad_norm": 0.9535816311836243, + "learning_rate": 0.0001, + "loss": 76.0979, + "step": 1890 + }, + { + "epoch": 3.059656371239986, + "grad_norm": 0.9099674820899963, + "learning_rate": 0.0001, + "loss": 75.9641, + "step": 1900 + }, + { + "epoch": 3.0757797148183603, + "grad_norm": 0.9568380117416382, + "learning_rate": 0.0001, + "loss": 75.0656, + "step": 1910 + }, + { + "epoch": 3.091903058396735, + "grad_norm": 0.9701060652732849, + "learning_rate": 0.0001, + "loss": 75.1422, + "step": 1920 + }, + { + "epoch": 3.1080264019751094, + "grad_norm": 1.3879098892211914, + "learning_rate": 0.0001, + "loss": 76.0731, + "step": 1930 + }, + { + "epoch": 3.1241497455534843, + "grad_norm": 0.9782084226608276, + "learning_rate": 0.0001, + "loss": 75.9666, + "step": 1940 + }, + { + "epoch": 3.140273089131859, + "grad_norm": 1.0856964588165283, + "learning_rate": 0.0001, + "loss": 75.0217, + "step": 1950 + }, + { + "epoch": 3.1563964327102334, + "grad_norm": 0.8506444692611694, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 1960 + }, + { + "epoch": 3.172519776288608, + "grad_norm": 1.044980525970459, + "learning_rate": 0.0001, + "loss": 75.3471, + "step": 1970 + }, + { + "epoch": 3.1886431198669825, + "grad_norm": 0.8432291746139526, + "learning_rate": 0.0001, + "loss": 75.3563, + "step": 1980 + }, + { + "epoch": 3.204766463445357, + "grad_norm": 0.9651834964752197, + "learning_rate": 0.0001, + "loss": 76.3991, + "step": 1990 + }, + { + "epoch": 3.2208898070237315, + "grad_norm": 0.957943320274353, + "learning_rate": 0.0001, + "loss": 77.2613, + "step": 2000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 33, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.4461324543116444e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ade09d5233289ba4bc5076af39b8652078582dee --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016a4e9b5d71a1947317d3801b00fe5d0604edbef1f981d7b6c92f219131717c +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-2000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcb63c019337e45eb09ec41f7e004badf452cad0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdb43fa4b5e14701116c544ad8a8a8647bd142e1111d6f4a94711ae1abff07b7 +size 1853729968 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..162f84d4e714e7fdfb3a5212ab9657c7206df58a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd112b09e96e3556af68c935bf0b7e8e7647ff111ee98231eababf02b6d7cec6 +size 1853730480 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cdc20285647e3bf2f8bdc689ef638425a347b51e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb19bfc7d3cb6eb78963c48a445bcb3d79c4931573d82bedcbca337e00b5bde8 +size 1853731312 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1c3cf628275549e5438a56abb8ac7f9e60c9259 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cd9bfe400b257513b8612a94dafdc86dee529319e3c9f6a154427c4ce8d12de +size 1853731120 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a753e5f4d1d4f5dcaeeae8dc69ff0700c6be1f2 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15cfd8a892df06537afd5e88a880647b212291e04387ba204e1eb4e516294be9 +size 1853731312 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ddd3e4ecb76a53918296132c1d0c465b8959607 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b633062e41709b5e49eff0c5477584d9bd5907bdf2ee7f72cb1341dac3912e5a +size 1853731120 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b327375630d0eaad37e5720601d09da5b00063e7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05c904ddf59191318b8aed28bc2267c35407f58453059d2097426cb11299800d +size 1853731440 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c1bead60619ebff08af9299e3b69e561b125b60 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:531217beded5226d1bb1ad9ec58d3ea737f4fdca06ca0136bad3bfaf518bc235 +size 1853731056 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/mp_rank_00_model_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f0deb8e7c49b2dfaf377cc1db55bd854ca5ce92 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/global_step20000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b66f821dd313196fbb054863e60d59c609cbd85c0e1c0682bb5cc899352a45e +size 2471676664 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/latest new file mode 100644 index 0000000000000000000000000000000000000000..50908603509898f37e005b455ca2e7cad40a4bb0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/latest @@ -0,0 +1 @@ +global_step20000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a7df050dd11be615bc163c9ad60ac590dac7006a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaa4aa35c540b7c31fc1aeca194b762db4a5623aa15829c5017f1171f8c17776 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..6a9599b19c51b9cce0f4e33a7c1ce70f79db8308 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c52188336e0dae379314367c9c7b9db0785031cc249c518d113e51bdd897b977 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..570e8e483e950aac19e7dd02a0e8a8c51bd4e029 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9099b8aeb4229c0ea0735ba1c008ff11a102832b33bdcb514ae295f3f4c6bdcf +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e668bf324c1f0d3b51200ebe22d8b699ded59ca9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51fa7ce39c67281fa9ce486feed0e6d6ac6a19587cf23846e2e260f4b08e378f +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..c758a83a472a07ff0056aabf64e5d60507034f1b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20f85890033a049dc0e79901223c755953496ff27fd5301d367c4a81f3864178 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..0559eca3624f62bddb00ecc60971981463bb0234 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3abdebc390b5b424155b5b3fc9076790358cc0a56aaaf151bd6d6d25c650bb8d +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..bb19890fb758671c7e0e31aa0a46822173cd5d41 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32c55917ef6c2ff12b5bf647f68f0c3531020d0cca8008d23bc3d6dbb8edf39a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..c8ee5bd7193ee66abcb2bb6afc8ed095dd3958f4 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba9413049ce2f929ce8773c0a06bc16fec3b4d09e07ea3dc838fd434be5b9ed3 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..8bdb7e0c867c9a5be904af38ec404ea54b2af0b3 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efbc19a04ef7aea30facf68fb396f0d8c681d2e8d059b6c502fd3f590361e524 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6ad5478feb2017608170db0b9c10db611f98e8f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f03f24b02bdd285f31b8705583130ff132519634aa9400055ed20b5470b95107 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..17650bd43d12c7a62782997e1ee55cf1c4ad7119 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/trainer_state.json @@ -0,0 +1,14034 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 32.2063787978032, + "eval_steps": 2000, + "global_step": 20000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.016123343578374565, + "grad_norm": 48.11856460571289, + "learning_rate": 0.0001, + "loss": 299.1088, + "step": 10 + }, + { + "epoch": 0.03224668715674913, + "grad_norm": 9.07711410522461, + "learning_rate": 0.0001, + "loss": 234.0888, + "step": 20 + }, + { + "epoch": 0.0483700307351237, + "grad_norm": 3.5935330390930176, + "learning_rate": 0.0001, + "loss": 231.4619, + "step": 30 + }, + { + "epoch": 0.06449337431349826, + "grad_norm": 3.5784752368927, + "learning_rate": 0.0001, + "loss": 229.7082, + "step": 40 + }, + { + "epoch": 0.08061671789187283, + "grad_norm": 3.5757956504821777, + "learning_rate": 0.0001, + "loss": 230.1062, + "step": 50 + }, + { + "epoch": 0.0967400614702474, + "grad_norm": 3.8763978481292725, + "learning_rate": 0.0001, + "loss": 229.1914, + "step": 60 + }, + { + "epoch": 0.11286340504862195, + "grad_norm": 3.006988525390625, + "learning_rate": 0.0001, + "loss": 229.6482, + "step": 70 + }, + { + "epoch": 0.12898674862699652, + "grad_norm": 2.3527095317840576, + "learning_rate": 0.0001, + "loss": 228.1141, + "step": 80 + }, + { + "epoch": 0.14511009220537108, + "grad_norm": 5.353086471557617, + "learning_rate": 0.0001, + "loss": 228.1448, + "step": 90 + }, + { + "epoch": 0.16123343578374566, + "grad_norm": 2.251854419708252, + "learning_rate": 0.0001, + "loss": 228.3897, + "step": 100 + }, + { + "epoch": 0.17735677936212021, + "grad_norm": 1.5789915323257446, + "learning_rate": 0.0001, + "loss": 227.7662, + "step": 110 + }, + { + "epoch": 0.1934801229404948, + "grad_norm": 3.7646398544311523, + "learning_rate": 0.0001, + "loss": 223.6411, + "step": 120 + }, + { + "epoch": 0.20960346651886935, + "grad_norm": 3.444000720977783, + "learning_rate": 0.0001, + "loss": 221.2073, + "step": 130 + }, + { + "epoch": 0.2257268100972439, + "grad_norm": 2.1871185302734375, + "learning_rate": 0.0001, + "loss": 215.9944, + "step": 140 + }, + { + "epoch": 0.2418501536756185, + "grad_norm": 2.266244411468506, + "learning_rate": 0.0001, + "loss": 212.3066, + "step": 150 + }, + { + "epoch": 0.25797349725399304, + "grad_norm": 2.043832778930664, + "learning_rate": 0.0001, + "loss": 208.1343, + "step": 160 + }, + { + "epoch": 0.2740968408323676, + "grad_norm": 2.2641494274139404, + "learning_rate": 0.0001, + "loss": 203.1485, + "step": 170 + }, + { + "epoch": 0.29022018441074215, + "grad_norm": 2.9727790355682373, + "learning_rate": 0.0001, + "loss": 198.181, + "step": 180 + }, + { + "epoch": 0.30634352798911674, + "grad_norm": 2.751523494720459, + "learning_rate": 0.0001, + "loss": 194.8422, + "step": 190 + }, + { + "epoch": 0.3224668715674913, + "grad_norm": 1.9561562538146973, + "learning_rate": 0.0001, + "loss": 189.9401, + "step": 200 + }, + { + "epoch": 0.33859021514586585, + "grad_norm": 2.3264107704162598, + "learning_rate": 0.0001, + "loss": 186.1309, + "step": 210 + }, + { + "epoch": 0.35471355872424043, + "grad_norm": 5.212426662445068, + "learning_rate": 0.0001, + "loss": 182.2255, + "step": 220 + }, + { + "epoch": 0.370836902302615, + "grad_norm": 2.488032102584839, + "learning_rate": 0.0001, + "loss": 181.0103, + "step": 230 + }, + { + "epoch": 0.3869602458809896, + "grad_norm": 2.6173202991485596, + "learning_rate": 0.0001, + "loss": 177.6162, + "step": 240 + }, + { + "epoch": 0.4030835894593641, + "grad_norm": 2.6245291233062744, + "learning_rate": 0.0001, + "loss": 173.0431, + "step": 250 + }, + { + "epoch": 0.4192069330377387, + "grad_norm": 5.356006145477295, + "learning_rate": 0.0001, + "loss": 172.2371, + "step": 260 + }, + { + "epoch": 0.4353302766161133, + "grad_norm": 2.0453829765319824, + "learning_rate": 0.0001, + "loss": 169.7781, + "step": 270 + }, + { + "epoch": 0.4514536201944878, + "grad_norm": 1.7002843618392944, + "learning_rate": 0.0001, + "loss": 166.9564, + "step": 280 + }, + { + "epoch": 0.4675769637728624, + "grad_norm": 2.181410312652588, + "learning_rate": 0.0001, + "loss": 166.0094, + "step": 290 + }, + { + "epoch": 0.483700307351237, + "grad_norm": 2.2948265075683594, + "learning_rate": 0.0001, + "loss": 162.4388, + "step": 300 + }, + { + "epoch": 0.4998236509296115, + "grad_norm": 3.0272789001464844, + "learning_rate": 0.0001, + "loss": 160.3446, + "step": 310 + }, + { + "epoch": 0.5159469945079861, + "grad_norm": 1.948184847831726, + "learning_rate": 0.0001, + "loss": 158.488, + "step": 320 + }, + { + "epoch": 0.5320703380863606, + "grad_norm": 1.7981538772583008, + "learning_rate": 0.0001, + "loss": 156.2107, + "step": 330 + }, + { + "epoch": 0.5481936816647353, + "grad_norm": 2.049323797225952, + "learning_rate": 0.0001, + "loss": 155.5434, + "step": 340 + }, + { + "epoch": 0.5643170252431098, + "grad_norm": 1.9415442943572998, + "learning_rate": 0.0001, + "loss": 153.8682, + "step": 350 + }, + { + "epoch": 0.5804403688214843, + "grad_norm": 2.2168045043945312, + "learning_rate": 0.0001, + "loss": 150.4087, + "step": 360 + }, + { + "epoch": 0.596563712399859, + "grad_norm": 1.510750651359558, + "learning_rate": 0.0001, + "loss": 151.7655, + "step": 370 + }, + { + "epoch": 0.6126870559782335, + "grad_norm": 2.7469751834869385, + "learning_rate": 0.0001, + "loss": 147.4665, + "step": 380 + }, + { + "epoch": 0.628810399556608, + "grad_norm": 2.0436975955963135, + "learning_rate": 0.0001, + "loss": 147.4405, + "step": 390 + }, + { + "epoch": 0.6449337431349826, + "grad_norm": 1.8219181299209595, + "learning_rate": 0.0001, + "loss": 146.6332, + "step": 400 + }, + { + "epoch": 0.6610570867133572, + "grad_norm": 1.948431134223938, + "learning_rate": 0.0001, + "loss": 143.3298, + "step": 410 + }, + { + "epoch": 0.6771804302917317, + "grad_norm": 1.8887908458709717, + "learning_rate": 0.0001, + "loss": 140.5058, + "step": 420 + }, + { + "epoch": 0.6933037738701063, + "grad_norm": 1.8739320039749146, + "learning_rate": 0.0001, + "loss": 137.6334, + "step": 430 + }, + { + "epoch": 0.7094271174484809, + "grad_norm": 2.057689666748047, + "learning_rate": 0.0001, + "loss": 135.0343, + "step": 440 + }, + { + "epoch": 0.7255504610268555, + "grad_norm": 1.6926478147506714, + "learning_rate": 0.0001, + "loss": 132.8766, + "step": 450 + }, + { + "epoch": 0.74167380460523, + "grad_norm": 1.5039501190185547, + "learning_rate": 0.0001, + "loss": 131.6774, + "step": 460 + }, + { + "epoch": 0.7577971481836046, + "grad_norm": 1.6903547048568726, + "learning_rate": 0.0001, + "loss": 129.3116, + "step": 470 + }, + { + "epoch": 0.7739204917619792, + "grad_norm": 3.7062244415283203, + "learning_rate": 0.0001, + "loss": 127.9314, + "step": 480 + }, + { + "epoch": 0.7900438353403537, + "grad_norm": 2.0586934089660645, + "learning_rate": 0.0001, + "loss": 126.6398, + "step": 490 + }, + { + "epoch": 0.8061671789187282, + "grad_norm": 1.7598215341567993, + "learning_rate": 0.0001, + "loss": 125.1501, + "step": 500 + }, + { + "epoch": 0.8222905224971029, + "grad_norm": 1.5992189645767212, + "learning_rate": 0.0001, + "loss": 124.7039, + "step": 510 + }, + { + "epoch": 0.8384138660754774, + "grad_norm": 2.8735458850860596, + "learning_rate": 0.0001, + "loss": 122.1619, + "step": 520 + }, + { + "epoch": 0.8545372096538519, + "grad_norm": 1.867098093032837, + "learning_rate": 0.0001, + "loss": 122.4506, + "step": 530 + }, + { + "epoch": 0.8706605532322266, + "grad_norm": 2.2088418006896973, + "learning_rate": 0.0001, + "loss": 122.1047, + "step": 540 + }, + { + "epoch": 0.8867838968106011, + "grad_norm": 1.642657995223999, + "learning_rate": 0.0001, + "loss": 120.1141, + "step": 550 + }, + { + "epoch": 0.9029072403889756, + "grad_norm": 1.5567114353179932, + "learning_rate": 0.0001, + "loss": 118.532, + "step": 560 + }, + { + "epoch": 0.9190305839673503, + "grad_norm": 1.667843222618103, + "learning_rate": 0.0001, + "loss": 118.0734, + "step": 570 + }, + { + "epoch": 0.9351539275457248, + "grad_norm": 1.5304442644119263, + "learning_rate": 0.0001, + "loss": 114.4925, + "step": 580 + }, + { + "epoch": 0.9512772711240993, + "grad_norm": 1.6920255422592163, + "learning_rate": 0.0001, + "loss": 114.0729, + "step": 590 + }, + { + "epoch": 0.967400614702474, + "grad_norm": 1.6313318014144897, + "learning_rate": 0.0001, + "loss": 114.2879, + "step": 600 + }, + { + "epoch": 0.9835239582808485, + "grad_norm": 1.9033657312393188, + "learning_rate": 0.0001, + "loss": 110.5299, + "step": 610 + }, + { + "epoch": 0.999647301859223, + "grad_norm": 1.9010754823684692, + "learning_rate": 0.0001, + "loss": 109.1145, + "step": 620 + }, + { + "epoch": 1.0145110092205372, + "grad_norm": 3.5390989780426025, + "learning_rate": 0.0001, + "loss": 101.0258, + "step": 630 + }, + { + "epoch": 1.0306343527989117, + "grad_norm": 2.2925171852111816, + "learning_rate": 0.0001, + "loss": 107.3693, + "step": 640 + }, + { + "epoch": 1.0467576963772862, + "grad_norm": 2.329108238220215, + "learning_rate": 0.0001, + "loss": 104.3778, + "step": 650 + }, + { + "epoch": 1.0628810399556607, + "grad_norm": 1.7325661182403564, + "learning_rate": 0.0001, + "loss": 102.8409, + "step": 660 + }, + { + "epoch": 1.0790043835340353, + "grad_norm": 1.989586353302002, + "learning_rate": 0.0001, + "loss": 104.2577, + "step": 670 + }, + { + "epoch": 1.09512772711241, + "grad_norm": 1.900484323501587, + "learning_rate": 0.0001, + "loss": 102.3051, + "step": 680 + }, + { + "epoch": 1.1112510706907845, + "grad_norm": 2.015859842300415, + "learning_rate": 0.0001, + "loss": 100.893, + "step": 690 + }, + { + "epoch": 1.127374414269159, + "grad_norm": 1.7695508003234863, + "learning_rate": 0.0001, + "loss": 99.215, + "step": 700 + }, + { + "epoch": 1.1434977578475336, + "grad_norm": 2.025923490524292, + "learning_rate": 0.0001, + "loss": 98.8768, + "step": 710 + }, + { + "epoch": 1.1596211014259081, + "grad_norm": 2.946708917617798, + "learning_rate": 0.0001, + "loss": 96.5588, + "step": 720 + }, + { + "epoch": 1.1757444450042827, + "grad_norm": 2.4511258602142334, + "learning_rate": 0.0001, + "loss": 95.0545, + "step": 730 + }, + { + "epoch": 1.1918677885826574, + "grad_norm": 2.3094682693481445, + "learning_rate": 0.0001, + "loss": 97.004, + "step": 740 + }, + { + "epoch": 1.207991132161032, + "grad_norm": 2.1313347816467285, + "learning_rate": 0.0001, + "loss": 94.8819, + "step": 750 + }, + { + "epoch": 1.2241144757394065, + "grad_norm": 1.614182710647583, + "learning_rate": 0.0001, + "loss": 95.5339, + "step": 760 + }, + { + "epoch": 1.240237819317781, + "grad_norm": 2.029953956604004, + "learning_rate": 0.0001, + "loss": 93.7303, + "step": 770 + }, + { + "epoch": 1.2563611628961555, + "grad_norm": 1.7871663570404053, + "learning_rate": 0.0001, + "loss": 93.8354, + "step": 780 + }, + { + "epoch": 1.2724845064745303, + "grad_norm": 1.72455632686615, + "learning_rate": 0.0001, + "loss": 94.7348, + "step": 790 + }, + { + "epoch": 1.2886078500529048, + "grad_norm": 1.9776639938354492, + "learning_rate": 0.0001, + "loss": 93.5957, + "step": 800 + }, + { + "epoch": 1.3047311936312793, + "grad_norm": 1.7187533378601074, + "learning_rate": 0.0001, + "loss": 92.9583, + "step": 810 + }, + { + "epoch": 1.3208545372096538, + "grad_norm": 1.5949162244796753, + "learning_rate": 0.0001, + "loss": 93.3088, + "step": 820 + }, + { + "epoch": 1.3369778807880284, + "grad_norm": 1.4616206884384155, + "learning_rate": 0.0001, + "loss": 93.2125, + "step": 830 + }, + { + "epoch": 1.353101224366403, + "grad_norm": 2.832163095474243, + "learning_rate": 0.0001, + "loss": 90.3301, + "step": 840 + }, + { + "epoch": 1.3692245679447774, + "grad_norm": 2.174501419067383, + "learning_rate": 0.0001, + "loss": 91.6988, + "step": 850 + }, + { + "epoch": 1.3853479115231522, + "grad_norm": 1.2156339883804321, + "learning_rate": 0.0001, + "loss": 92.7419, + "step": 860 + }, + { + "epoch": 1.4014712551015267, + "grad_norm": 1.608261227607727, + "learning_rate": 0.0001, + "loss": 89.8255, + "step": 870 + }, + { + "epoch": 1.4175945986799012, + "grad_norm": 1.6671923398971558, + "learning_rate": 0.0001, + "loss": 90.9683, + "step": 880 + }, + { + "epoch": 1.4337179422582758, + "grad_norm": 1.2346433401107788, + "learning_rate": 0.0001, + "loss": 90.6092, + "step": 890 + }, + { + "epoch": 1.4498412858366505, + "grad_norm": 1.561085820198059, + "learning_rate": 0.0001, + "loss": 88.0842, + "step": 900 + }, + { + "epoch": 1.465964629415025, + "grad_norm": 1.5475201606750488, + "learning_rate": 0.0001, + "loss": 89.8786, + "step": 910 + }, + { + "epoch": 1.4820879729933996, + "grad_norm": 1.6044800281524658, + "learning_rate": 0.0001, + "loss": 88.0364, + "step": 920 + }, + { + "epoch": 1.498211316571774, + "grad_norm": 1.4081908464431763, + "learning_rate": 0.0001, + "loss": 88.8799, + "step": 930 + }, + { + "epoch": 1.5143346601501486, + "grad_norm": 2.341053009033203, + "learning_rate": 0.0001, + "loss": 88.2023, + "step": 940 + }, + { + "epoch": 1.5304580037285231, + "grad_norm": 1.3976799249649048, + "learning_rate": 0.0001, + "loss": 86.6493, + "step": 950 + }, + { + "epoch": 1.5465813473068977, + "grad_norm": 1.2485483884811401, + "learning_rate": 0.0001, + "loss": 88.9224, + "step": 960 + }, + { + "epoch": 1.5627046908852722, + "grad_norm": 1.6824235916137695, + "learning_rate": 0.0001, + "loss": 89.6484, + "step": 970 + }, + { + "epoch": 1.578828034463647, + "grad_norm": 1.64790940284729, + "learning_rate": 0.0001, + "loss": 87.2479, + "step": 980 + }, + { + "epoch": 1.5949513780420215, + "grad_norm": 1.2093443870544434, + "learning_rate": 0.0001, + "loss": 86.8241, + "step": 990 + }, + { + "epoch": 1.611074721620396, + "grad_norm": 1.5306812524795532, + "learning_rate": 0.0001, + "loss": 88.0006, + "step": 1000 + }, + { + "epoch": 1.6271980651987707, + "grad_norm": 1.2465763092041016, + "learning_rate": 0.0001, + "loss": 86.8143, + "step": 1010 + }, + { + "epoch": 1.6433214087771453, + "grad_norm": 1.5224069356918335, + "learning_rate": 0.0001, + "loss": 87.0102, + "step": 1020 + }, + { + "epoch": 1.6594447523555198, + "grad_norm": 1.312179684638977, + "learning_rate": 0.0001, + "loss": 88.7395, + "step": 1030 + }, + { + "epoch": 1.6755680959338943, + "grad_norm": 1.2343158721923828, + "learning_rate": 0.0001, + "loss": 87.2868, + "step": 1040 + }, + { + "epoch": 1.6916914395122689, + "grad_norm": 1.2778265476226807, + "learning_rate": 0.0001, + "loss": 87.043, + "step": 1050 + }, + { + "epoch": 1.7078147830906434, + "grad_norm": 1.2799627780914307, + "learning_rate": 0.0001, + "loss": 87.7843, + "step": 1060 + }, + { + "epoch": 1.723938126669018, + "grad_norm": 1.3688952922821045, + "learning_rate": 0.0001, + "loss": 85.8985, + "step": 1070 + }, + { + "epoch": 1.7400614702473924, + "grad_norm": 1.4317796230316162, + "learning_rate": 0.0001, + "loss": 85.5408, + "step": 1080 + }, + { + "epoch": 1.756184813825767, + "grad_norm": 1.164801001548767, + "learning_rate": 0.0001, + "loss": 85.8484, + "step": 1090 + }, + { + "epoch": 1.7723081574041417, + "grad_norm": 1.3929173946380615, + "learning_rate": 0.0001, + "loss": 86.9031, + "step": 1100 + }, + { + "epoch": 1.7884315009825162, + "grad_norm": 2.116119861602783, + "learning_rate": 0.0001, + "loss": 86.5343, + "step": 1110 + }, + { + "epoch": 1.8045548445608908, + "grad_norm": 1.4117276668548584, + "learning_rate": 0.0001, + "loss": 85.2678, + "step": 1120 + }, + { + "epoch": 1.8206781881392655, + "grad_norm": 1.3213504552841187, + "learning_rate": 0.0001, + "loss": 85.8935, + "step": 1130 + }, + { + "epoch": 1.83680153171764, + "grad_norm": 1.3589770793914795, + "learning_rate": 0.0001, + "loss": 85.032, + "step": 1140 + }, + { + "epoch": 1.8529248752960146, + "grad_norm": 1.3336007595062256, + "learning_rate": 0.0001, + "loss": 84.0993, + "step": 1150 + }, + { + "epoch": 1.869048218874389, + "grad_norm": 1.1081451177597046, + "learning_rate": 0.0001, + "loss": 84.2225, + "step": 1160 + }, + { + "epoch": 1.8851715624527636, + "grad_norm": 1.482038974761963, + "learning_rate": 0.0001, + "loss": 82.1439, + "step": 1170 + }, + { + "epoch": 1.9012949060311382, + "grad_norm": 1.1571478843688965, + "learning_rate": 0.0001, + "loss": 82.5045, + "step": 1180 + }, + { + "epoch": 1.9174182496095127, + "grad_norm": 1.2665468454360962, + "learning_rate": 0.0001, + "loss": 82.8761, + "step": 1190 + }, + { + "epoch": 1.9335415931878872, + "grad_norm": 1.0771509408950806, + "learning_rate": 0.0001, + "loss": 81.5162, + "step": 1200 + }, + { + "epoch": 1.949664936766262, + "grad_norm": 1.2992299795150757, + "learning_rate": 0.0001, + "loss": 82.4038, + "step": 1210 + }, + { + "epoch": 1.9657882803446365, + "grad_norm": 1.2946668863296509, + "learning_rate": 0.0001, + "loss": 83.4837, + "step": 1220 + }, + { + "epoch": 1.981911623923011, + "grad_norm": 1.3505686521530151, + "learning_rate": 0.0001, + "loss": 81.8993, + "step": 1230 + }, + { + "epoch": 1.9980349675013858, + "grad_norm": 1.1247715950012207, + "learning_rate": 0.0001, + "loss": 82.9006, + "step": 1240 + }, + { + "epoch": 2.0128986748627, + "grad_norm": 1.5652127265930176, + "learning_rate": 0.0001, + "loss": 75.8347, + "step": 1250 + }, + { + "epoch": 2.0290220184410743, + "grad_norm": 7.311964988708496, + "learning_rate": 0.0001, + "loss": 80.5435, + "step": 1260 + }, + { + "epoch": 2.045145362019449, + "grad_norm": 1.2229893207550049, + "learning_rate": 0.0001, + "loss": 83.3277, + "step": 1270 + }, + { + "epoch": 2.0612687055978234, + "grad_norm": 1.1387693881988525, + "learning_rate": 0.0001, + "loss": 81.3252, + "step": 1280 + }, + { + "epoch": 2.077392049176198, + "grad_norm": 1.307201623916626, + "learning_rate": 0.0001, + "loss": 82.3311, + "step": 1290 + }, + { + "epoch": 2.0935153927545724, + "grad_norm": 1.1502896547317505, + "learning_rate": 0.0001, + "loss": 81.1682, + "step": 1300 + }, + { + "epoch": 2.109638736332947, + "grad_norm": 0.9291583299636841, + "learning_rate": 0.0001, + "loss": 81.7017, + "step": 1310 + }, + { + "epoch": 2.1257620799113215, + "grad_norm": 1.0334666967391968, + "learning_rate": 0.0001, + "loss": 81.718, + "step": 1320 + }, + { + "epoch": 2.141885423489696, + "grad_norm": 1.1099056005477905, + "learning_rate": 0.0001, + "loss": 81.2232, + "step": 1330 + }, + { + "epoch": 2.1580087670680705, + "grad_norm": 1.289478063583374, + "learning_rate": 0.0001, + "loss": 81.8736, + "step": 1340 + }, + { + "epoch": 2.174132110646445, + "grad_norm": 1.1634794473648071, + "learning_rate": 0.0001, + "loss": 79.9434, + "step": 1350 + }, + { + "epoch": 2.19025545422482, + "grad_norm": 0.9563440680503845, + "learning_rate": 0.0001, + "loss": 80.628, + "step": 1360 + }, + { + "epoch": 2.2063787978031946, + "grad_norm": 1.1268844604492188, + "learning_rate": 0.0001, + "loss": 82.1389, + "step": 1370 + }, + { + "epoch": 2.222502141381569, + "grad_norm": 1.1050443649291992, + "learning_rate": 0.0001, + "loss": 78.9023, + "step": 1380 + }, + { + "epoch": 2.2386254849599436, + "grad_norm": 1.1936142444610596, + "learning_rate": 0.0001, + "loss": 81.2118, + "step": 1390 + }, + { + "epoch": 2.254748828538318, + "grad_norm": 1.7825465202331543, + "learning_rate": 0.0001, + "loss": 80.8804, + "step": 1400 + }, + { + "epoch": 2.2708721721166927, + "grad_norm": 2.437251329421997, + "learning_rate": 0.0001, + "loss": 79.3807, + "step": 1410 + }, + { + "epoch": 2.286995515695067, + "grad_norm": 1.0604854822158813, + "learning_rate": 0.0001, + "loss": 79.5881, + "step": 1420 + }, + { + "epoch": 2.3031188592734417, + "grad_norm": 1.2285529375076294, + "learning_rate": 0.0001, + "loss": 79.5517, + "step": 1430 + }, + { + "epoch": 2.3192422028518163, + "grad_norm": 1.1916660070419312, + "learning_rate": 0.0001, + "loss": 80.5887, + "step": 1440 + }, + { + "epoch": 2.335365546430191, + "grad_norm": 1.224304437637329, + "learning_rate": 0.0001, + "loss": 79.8109, + "step": 1450 + }, + { + "epoch": 2.3514888900085653, + "grad_norm": 0.9228448867797852, + "learning_rate": 0.0001, + "loss": 79.2787, + "step": 1460 + }, + { + "epoch": 2.3676122335869403, + "grad_norm": 1.2085226774215698, + "learning_rate": 0.0001, + "loss": 78.4983, + "step": 1470 + }, + { + "epoch": 2.383735577165315, + "grad_norm": 2.424083948135376, + "learning_rate": 0.0001, + "loss": 79.8657, + "step": 1480 + }, + { + "epoch": 2.3998589207436893, + "grad_norm": 1.0277849435806274, + "learning_rate": 0.0001, + "loss": 80.6915, + "step": 1490 + }, + { + "epoch": 2.415982264322064, + "grad_norm": 1.0226075649261475, + "learning_rate": 0.0001, + "loss": 78.9751, + "step": 1500 + }, + { + "epoch": 2.4321056079004384, + "grad_norm": 0.8860388398170471, + "learning_rate": 0.0001, + "loss": 78.3915, + "step": 1510 + }, + { + "epoch": 2.448228951478813, + "grad_norm": 1.265637755393982, + "learning_rate": 0.0001, + "loss": 78.8737, + "step": 1520 + }, + { + "epoch": 2.4643522950571874, + "grad_norm": 1.2518186569213867, + "learning_rate": 0.0001, + "loss": 76.5169, + "step": 1530 + }, + { + "epoch": 2.480475638635562, + "grad_norm": 1.074277400970459, + "learning_rate": 0.0001, + "loss": 77.9156, + "step": 1540 + }, + { + "epoch": 2.4965989822139365, + "grad_norm": 1.0666711330413818, + "learning_rate": 0.0001, + "loss": 78.4103, + "step": 1550 + }, + { + "epoch": 2.512722325792311, + "grad_norm": 0.9492248892784119, + "learning_rate": 0.0001, + "loss": 76.7393, + "step": 1560 + }, + { + "epoch": 2.5288456693706856, + "grad_norm": 1.127238154411316, + "learning_rate": 0.0001, + "loss": 79.729, + "step": 1570 + }, + { + "epoch": 2.5449690129490605, + "grad_norm": 1.1520992517471313, + "learning_rate": 0.0001, + "loss": 78.5618, + "step": 1580 + }, + { + "epoch": 2.5610923565274346, + "grad_norm": 0.8632035255432129, + "learning_rate": 0.0001, + "loss": 79.032, + "step": 1590 + }, + { + "epoch": 2.5772157001058096, + "grad_norm": 1.0506329536437988, + "learning_rate": 0.0001, + "loss": 77.6204, + "step": 1600 + }, + { + "epoch": 2.593339043684184, + "grad_norm": 1.019895315170288, + "learning_rate": 0.0001, + "loss": 79.2634, + "step": 1610 + }, + { + "epoch": 2.6094623872625586, + "grad_norm": 1.1685571670532227, + "learning_rate": 0.0001, + "loss": 78.8371, + "step": 1620 + }, + { + "epoch": 2.625585730840933, + "grad_norm": 1.147546648979187, + "learning_rate": 0.0001, + "loss": 78.2113, + "step": 1630 + }, + { + "epoch": 2.6417090744193077, + "grad_norm": 1.7336841821670532, + "learning_rate": 0.0001, + "loss": 78.7866, + "step": 1640 + }, + { + "epoch": 2.657832417997682, + "grad_norm": 1.2150614261627197, + "learning_rate": 0.0001, + "loss": 78.5621, + "step": 1650 + }, + { + "epoch": 2.6739557615760567, + "grad_norm": 1.2257503271102905, + "learning_rate": 0.0001, + "loss": 77.5835, + "step": 1660 + }, + { + "epoch": 2.6900791051544313, + "grad_norm": 1.0375311374664307, + "learning_rate": 0.0001, + "loss": 77.7509, + "step": 1670 + }, + { + "epoch": 2.706202448732806, + "grad_norm": 1.2551004886627197, + "learning_rate": 0.0001, + "loss": 77.8615, + "step": 1680 + }, + { + "epoch": 2.7223257923111808, + "grad_norm": 1.0464223623275757, + "learning_rate": 0.0001, + "loss": 77.2837, + "step": 1690 + }, + { + "epoch": 2.738449135889555, + "grad_norm": 1.0254806280136108, + "learning_rate": 0.0001, + "loss": 76.9616, + "step": 1700 + }, + { + "epoch": 2.75457247946793, + "grad_norm": 1.1665682792663574, + "learning_rate": 0.0001, + "loss": 77.9108, + "step": 1710 + }, + { + "epoch": 2.7706958230463044, + "grad_norm": 1.1349453926086426, + "learning_rate": 0.0001, + "loss": 78.7636, + "step": 1720 + }, + { + "epoch": 2.786819166624679, + "grad_norm": 1.4961057901382446, + "learning_rate": 0.0001, + "loss": 76.3862, + "step": 1730 + }, + { + "epoch": 2.8029425102030534, + "grad_norm": 1.10721755027771, + "learning_rate": 0.0001, + "loss": 77.4105, + "step": 1740 + }, + { + "epoch": 2.819065853781428, + "grad_norm": 0.9878576397895813, + "learning_rate": 0.0001, + "loss": 78.4855, + "step": 1750 + }, + { + "epoch": 2.8351891973598025, + "grad_norm": 1.1722687482833862, + "learning_rate": 0.0001, + "loss": 75.1589, + "step": 1760 + }, + { + "epoch": 2.851312540938177, + "grad_norm": 1.6624362468719482, + "learning_rate": 0.0001, + "loss": 77.1647, + "step": 1770 + }, + { + "epoch": 2.8674358845165515, + "grad_norm": 0.9542295932769775, + "learning_rate": 0.0001, + "loss": 77.1656, + "step": 1780 + }, + { + "epoch": 2.883559228094926, + "grad_norm": 2.328542947769165, + "learning_rate": 0.0001, + "loss": 76.4686, + "step": 1790 + }, + { + "epoch": 2.899682571673301, + "grad_norm": 1.2532029151916504, + "learning_rate": 0.0001, + "loss": 76.1727, + "step": 1800 + }, + { + "epoch": 2.915805915251675, + "grad_norm": 0.9604716897010803, + "learning_rate": 0.0001, + "loss": 74.3132, + "step": 1810 + }, + { + "epoch": 2.93192925883005, + "grad_norm": 0.9182595610618591, + "learning_rate": 0.0001, + "loss": 77.8689, + "step": 1820 + }, + { + "epoch": 2.9480526024084246, + "grad_norm": 0.9623042941093445, + "learning_rate": 0.0001, + "loss": 77.7531, + "step": 1830 + }, + { + "epoch": 2.964175945986799, + "grad_norm": 1.0500119924545288, + "learning_rate": 0.0001, + "loss": 77.6007, + "step": 1840 + }, + { + "epoch": 2.9802992895651736, + "grad_norm": 1.0874103307724, + "learning_rate": 0.0001, + "loss": 77.042, + "step": 1850 + }, + { + "epoch": 2.996422633143548, + "grad_norm": 0.9955475330352783, + "learning_rate": 0.0001, + "loss": 75.468, + "step": 1860 + }, + { + "epoch": 3.011286340504862, + "grad_norm": 1.4281760454177856, + "learning_rate": 0.0001, + "loss": 69.5972, + "step": 1870 + }, + { + "epoch": 3.0274096840832367, + "grad_norm": 1.0292123556137085, + "learning_rate": 0.0001, + "loss": 76.0186, + "step": 1880 + }, + { + "epoch": 3.0435330276616113, + "grad_norm": 0.9535816311836243, + "learning_rate": 0.0001, + "loss": 76.0979, + "step": 1890 + }, + { + "epoch": 3.059656371239986, + "grad_norm": 0.9099674820899963, + "learning_rate": 0.0001, + "loss": 75.9641, + "step": 1900 + }, + { + "epoch": 3.0757797148183603, + "grad_norm": 0.9568380117416382, + "learning_rate": 0.0001, + "loss": 75.0656, + "step": 1910 + }, + { + "epoch": 3.091903058396735, + "grad_norm": 0.9701060652732849, + "learning_rate": 0.0001, + "loss": 75.1422, + "step": 1920 + }, + { + "epoch": 3.1080264019751094, + "grad_norm": 1.3879098892211914, + "learning_rate": 0.0001, + "loss": 76.0731, + "step": 1930 + }, + { + "epoch": 3.1241497455534843, + "grad_norm": 0.9782084226608276, + "learning_rate": 0.0001, + "loss": 75.9666, + "step": 1940 + }, + { + "epoch": 3.140273089131859, + "grad_norm": 1.0856964588165283, + "learning_rate": 0.0001, + "loss": 75.0217, + "step": 1950 + }, + { + "epoch": 3.1563964327102334, + "grad_norm": 0.8506444692611694, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 1960 + }, + { + "epoch": 3.172519776288608, + "grad_norm": 1.044980525970459, + "learning_rate": 0.0001, + "loss": 75.3471, + "step": 1970 + }, + { + "epoch": 3.1886431198669825, + "grad_norm": 0.8432291746139526, + "learning_rate": 0.0001, + "loss": 75.3563, + "step": 1980 + }, + { + "epoch": 3.204766463445357, + "grad_norm": 0.9651834964752197, + "learning_rate": 0.0001, + "loss": 76.3991, + "step": 1990 + }, + { + "epoch": 3.2208898070237315, + "grad_norm": 0.957943320274353, + "learning_rate": 0.0001, + "loss": 77.2613, + "step": 2000 + }, + { + "epoch": 3.237013150602106, + "grad_norm": 0.9495745301246643, + "learning_rate": 0.0001, + "loss": 73.3801, + "step": 2010 + }, + { + "epoch": 3.2531364941804806, + "grad_norm": 1.182983160018921, + "learning_rate": 0.0001, + "loss": 74.5695, + "step": 2020 + }, + { + "epoch": 3.269259837758855, + "grad_norm": 0.828452467918396, + "learning_rate": 0.0001, + "loss": 75.4313, + "step": 2030 + }, + { + "epoch": 3.2853831813372296, + "grad_norm": 0.8340104818344116, + "learning_rate": 0.0001, + "loss": 75.1086, + "step": 2040 + }, + { + "epoch": 3.301506524915604, + "grad_norm": 1.1700762510299683, + "learning_rate": 0.0001, + "loss": 75.7496, + "step": 2050 + }, + { + "epoch": 3.317629868493979, + "grad_norm": 0.8709110617637634, + "learning_rate": 0.0001, + "loss": 73.8403, + "step": 2060 + }, + { + "epoch": 3.3337532120723536, + "grad_norm": 0.8928954601287842, + "learning_rate": 0.0001, + "loss": 75.6134, + "step": 2070 + }, + { + "epoch": 3.349876555650728, + "grad_norm": 0.9787353277206421, + "learning_rate": 0.0001, + "loss": 75.8116, + "step": 2080 + }, + { + "epoch": 3.3659998992291027, + "grad_norm": 0.9515123963356018, + "learning_rate": 0.0001, + "loss": 75.5024, + "step": 2090 + }, + { + "epoch": 3.3821232428074772, + "grad_norm": 0.9529262185096741, + "learning_rate": 0.0001, + "loss": 74.1553, + "step": 2100 + }, + { + "epoch": 3.3982465863858518, + "grad_norm": 0.8934710621833801, + "learning_rate": 0.0001, + "loss": 74.1501, + "step": 2110 + }, + { + "epoch": 3.4143699299642263, + "grad_norm": 1.089447021484375, + "learning_rate": 0.0001, + "loss": 74.9123, + "step": 2120 + }, + { + "epoch": 3.430493273542601, + "grad_norm": 1.2569622993469238, + "learning_rate": 0.0001, + "loss": 74.3962, + "step": 2130 + }, + { + "epoch": 3.4466166171209753, + "grad_norm": 1.020094394683838, + "learning_rate": 0.0001, + "loss": 74.6617, + "step": 2140 + }, + { + "epoch": 3.46273996069935, + "grad_norm": 1.1579667329788208, + "learning_rate": 0.0001, + "loss": 74.2745, + "step": 2150 + }, + { + "epoch": 3.4788633042777244, + "grad_norm": 1.448935627937317, + "learning_rate": 0.0001, + "loss": 71.5285, + "step": 2160 + }, + { + "epoch": 3.4949866478560994, + "grad_norm": 1.0030494928359985, + "learning_rate": 0.0001, + "loss": 74.6666, + "step": 2170 + }, + { + "epoch": 3.511109991434474, + "grad_norm": 0.9010888934135437, + "learning_rate": 0.0001, + "loss": 74.9355, + "step": 2180 + }, + { + "epoch": 3.5272333350128484, + "grad_norm": 1.1595124006271362, + "learning_rate": 0.0001, + "loss": 74.5049, + "step": 2190 + }, + { + "epoch": 3.543356678591223, + "grad_norm": 1.066405177116394, + "learning_rate": 0.0001, + "loss": 74.6336, + "step": 2200 + }, + { + "epoch": 3.5594800221695975, + "grad_norm": 1.2301596403121948, + "learning_rate": 0.0001, + "loss": 74.6664, + "step": 2210 + }, + { + "epoch": 3.575603365747972, + "grad_norm": 1.0073511600494385, + "learning_rate": 0.0001, + "loss": 73.1405, + "step": 2220 + }, + { + "epoch": 3.5917267093263465, + "grad_norm": 0.9913995862007141, + "learning_rate": 0.0001, + "loss": 75.7443, + "step": 2230 + }, + { + "epoch": 3.607850052904721, + "grad_norm": 0.9555648565292358, + "learning_rate": 0.0001, + "loss": 74.8036, + "step": 2240 + }, + { + "epoch": 3.6239733964830956, + "grad_norm": 0.9350649118423462, + "learning_rate": 0.0001, + "loss": 73.5783, + "step": 2250 + }, + { + "epoch": 3.64009674006147, + "grad_norm": 0.935840368270874, + "learning_rate": 0.0001, + "loss": 75.1694, + "step": 2260 + }, + { + "epoch": 3.6562200836398446, + "grad_norm": 0.9303803443908691, + "learning_rate": 0.0001, + "loss": 72.8282, + "step": 2270 + }, + { + "epoch": 3.6723434272182196, + "grad_norm": 0.937164843082428, + "learning_rate": 0.0001, + "loss": 70.3305, + "step": 2280 + }, + { + "epoch": 3.6884667707965937, + "grad_norm": 1.027329444885254, + "learning_rate": 0.0001, + "loss": 73.748, + "step": 2290 + }, + { + "epoch": 3.7045901143749687, + "grad_norm": 1.0915215015411377, + "learning_rate": 0.0001, + "loss": 72.2265, + "step": 2300 + }, + { + "epoch": 3.720713457953343, + "grad_norm": 1.1031261682510376, + "learning_rate": 0.0001, + "loss": 71.2795, + "step": 2310 + }, + { + "epoch": 3.7368368015317177, + "grad_norm": 1.1466140747070312, + "learning_rate": 0.0001, + "loss": 72.0116, + "step": 2320 + }, + { + "epoch": 3.7529601451100922, + "grad_norm": 1.0441662073135376, + "learning_rate": 0.0001, + "loss": 73.3735, + "step": 2330 + }, + { + "epoch": 3.7690834886884668, + "grad_norm": 0.8610432744026184, + "learning_rate": 0.0001, + "loss": 73.3433, + "step": 2340 + }, + { + "epoch": 3.7852068322668413, + "grad_norm": 0.885720431804657, + "learning_rate": 0.0001, + "loss": 73.575, + "step": 2350 + }, + { + "epoch": 3.801330175845216, + "grad_norm": 1.0023950338363647, + "learning_rate": 0.0001, + "loss": 73.4166, + "step": 2360 + }, + { + "epoch": 3.8174535194235903, + "grad_norm": 1.3820304870605469, + "learning_rate": 0.0001, + "loss": 72.9599, + "step": 2370 + }, + { + "epoch": 3.833576863001965, + "grad_norm": 0.9645710587501526, + "learning_rate": 0.0001, + "loss": 72.3922, + "step": 2380 + }, + { + "epoch": 3.84970020658034, + "grad_norm": 1.273369312286377, + "learning_rate": 0.0001, + "loss": 72.0158, + "step": 2390 + }, + { + "epoch": 3.865823550158714, + "grad_norm": 0.859863817691803, + "learning_rate": 0.0001, + "loss": 74.6642, + "step": 2400 + }, + { + "epoch": 3.881946893737089, + "grad_norm": 0.9396682381629944, + "learning_rate": 0.0001, + "loss": 71.9003, + "step": 2410 + }, + { + "epoch": 3.8980702373154634, + "grad_norm": 0.9976151585578918, + "learning_rate": 0.0001, + "loss": 73.0716, + "step": 2420 + }, + { + "epoch": 3.914193580893838, + "grad_norm": 1.0730767250061035, + "learning_rate": 0.0001, + "loss": 72.7847, + "step": 2430 + }, + { + "epoch": 3.9303169244722125, + "grad_norm": 1.002018690109253, + "learning_rate": 0.0001, + "loss": 71.7309, + "step": 2440 + }, + { + "epoch": 3.946440268050587, + "grad_norm": 0.9102753400802612, + "learning_rate": 0.0001, + "loss": 71.616, + "step": 2450 + }, + { + "epoch": 3.9625636116289615, + "grad_norm": 0.9827467799186707, + "learning_rate": 0.0001, + "loss": 72.22, + "step": 2460 + }, + { + "epoch": 3.978686955207336, + "grad_norm": 1.2781893014907837, + "learning_rate": 0.0001, + "loss": 73.2048, + "step": 2470 + }, + { + "epoch": 3.9948102987857106, + "grad_norm": 0.9042410850524902, + "learning_rate": 0.0001, + "loss": 72.9843, + "step": 2480 + }, + { + "epoch": 4.009674006147025, + "grad_norm": 0.8662117123603821, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 2490 + }, + { + "epoch": 4.0257973497254, + "grad_norm": 0.9448756575584412, + "learning_rate": 0.0001, + "loss": 73.36, + "step": 2500 + }, + { + "epoch": 4.041920693303774, + "grad_norm": 0.9054566621780396, + "learning_rate": 0.0001, + "loss": 73.94, + "step": 2510 + }, + { + "epoch": 4.058044036882149, + "grad_norm": 0.957398533821106, + "learning_rate": 0.0001, + "loss": 73.2019, + "step": 2520 + }, + { + "epoch": 4.074167380460523, + "grad_norm": 0.9444408416748047, + "learning_rate": 0.0001, + "loss": 70.6553, + "step": 2530 + }, + { + "epoch": 4.090290724038898, + "grad_norm": 0.8716063499450684, + "learning_rate": 0.0001, + "loss": 70.1048, + "step": 2540 + }, + { + "epoch": 4.106414067617272, + "grad_norm": 0.949249267578125, + "learning_rate": 0.0001, + "loss": 72.4453, + "step": 2550 + }, + { + "epoch": 4.122537411195647, + "grad_norm": 0.9843282699584961, + "learning_rate": 0.0001, + "loss": 72.1725, + "step": 2560 + }, + { + "epoch": 4.138660754774021, + "grad_norm": 0.8977480530738831, + "learning_rate": 0.0001, + "loss": 72.5272, + "step": 2570 + }, + { + "epoch": 4.154784098352396, + "grad_norm": 0.9957827925682068, + "learning_rate": 0.0001, + "loss": 72.289, + "step": 2580 + }, + { + "epoch": 4.170907441930771, + "grad_norm": 1.1358791589736938, + "learning_rate": 0.0001, + "loss": 72.1949, + "step": 2590 + }, + { + "epoch": 4.187030785509145, + "grad_norm": 0.9953081607818604, + "learning_rate": 0.0001, + "loss": 72.6004, + "step": 2600 + }, + { + "epoch": 4.20315412908752, + "grad_norm": 1.019061803817749, + "learning_rate": 0.0001, + "loss": 72.3772, + "step": 2610 + }, + { + "epoch": 4.219277472665894, + "grad_norm": 0.9062562584877014, + "learning_rate": 0.0001, + "loss": 72.0566, + "step": 2620 + }, + { + "epoch": 4.235400816244269, + "grad_norm": 1.0877611637115479, + "learning_rate": 0.0001, + "loss": 72.854, + "step": 2630 + }, + { + "epoch": 4.251524159822643, + "grad_norm": 1.2419648170471191, + "learning_rate": 0.0001, + "loss": 72.6498, + "step": 2640 + }, + { + "epoch": 4.267647503401018, + "grad_norm": 0.9507225751876831, + "learning_rate": 0.0001, + "loss": 68.9583, + "step": 2650 + }, + { + "epoch": 4.283770846979392, + "grad_norm": 1.121191382408142, + "learning_rate": 0.0001, + "loss": 72.8464, + "step": 2660 + }, + { + "epoch": 4.299894190557767, + "grad_norm": 1.0543361902236938, + "learning_rate": 0.0001, + "loss": 71.9882, + "step": 2670 + }, + { + "epoch": 4.316017534136141, + "grad_norm": 0.8622955083847046, + "learning_rate": 0.0001, + "loss": 71.7373, + "step": 2680 + }, + { + "epoch": 4.332140877714516, + "grad_norm": 0.911253809928894, + "learning_rate": 0.0001, + "loss": 71.1968, + "step": 2690 + }, + { + "epoch": 4.34826422129289, + "grad_norm": 4.405472755432129, + "learning_rate": 0.0001, + "loss": 70.4128, + "step": 2700 + }, + { + "epoch": 4.364387564871265, + "grad_norm": 0.9142985939979553, + "learning_rate": 0.0001, + "loss": 69.7548, + "step": 2710 + }, + { + "epoch": 4.38051090844964, + "grad_norm": 1.1022522449493408, + "learning_rate": 0.0001, + "loss": 71.8282, + "step": 2720 + }, + { + "epoch": 4.396634252028014, + "grad_norm": 1.018386721611023, + "learning_rate": 0.0001, + "loss": 71.0853, + "step": 2730 + }, + { + "epoch": 4.412757595606389, + "grad_norm": 1.0483211278915405, + "learning_rate": 0.0001, + "loss": 72.1478, + "step": 2740 + }, + { + "epoch": 4.428880939184763, + "grad_norm": 1.026861548423767, + "learning_rate": 0.0001, + "loss": 71.8947, + "step": 2750 + }, + { + "epoch": 4.445004282763138, + "grad_norm": 1.1858552694320679, + "learning_rate": 0.0001, + "loss": 71.7016, + "step": 2760 + }, + { + "epoch": 4.461127626341512, + "grad_norm": 1.1212221384048462, + "learning_rate": 0.0001, + "loss": 71.4454, + "step": 2770 + }, + { + "epoch": 4.477250969919887, + "grad_norm": 0.8062102794647217, + "learning_rate": 0.0001, + "loss": 73.2672, + "step": 2780 + }, + { + "epoch": 4.493374313498261, + "grad_norm": 1.104483962059021, + "learning_rate": 0.0001, + "loss": 70.548, + "step": 2790 + }, + { + "epoch": 4.509497657076636, + "grad_norm": 0.942057728767395, + "learning_rate": 0.0001, + "loss": 72.3584, + "step": 2800 + }, + { + "epoch": 4.525621000655011, + "grad_norm": 0.9028333425521851, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 2810 + }, + { + "epoch": 4.541744344233385, + "grad_norm": 0.9096795916557312, + "learning_rate": 0.0001, + "loss": 70.6596, + "step": 2820 + }, + { + "epoch": 4.55786768781176, + "grad_norm": 0.9038829207420349, + "learning_rate": 0.0001, + "loss": 72.6148, + "step": 2830 + }, + { + "epoch": 4.573991031390134, + "grad_norm": 1.8511613607406616, + "learning_rate": 0.0001, + "loss": 71.7205, + "step": 2840 + }, + { + "epoch": 4.590114374968509, + "grad_norm": 1.3492343425750732, + "learning_rate": 0.0001, + "loss": 72.1492, + "step": 2850 + }, + { + "epoch": 4.6062377185468835, + "grad_norm": 1.1469528675079346, + "learning_rate": 0.0001, + "loss": 70.5892, + "step": 2860 + }, + { + "epoch": 4.622361062125258, + "grad_norm": 1.1374379396438599, + "learning_rate": 0.0001, + "loss": 70.061, + "step": 2870 + }, + { + "epoch": 4.6384844057036325, + "grad_norm": 0.9597565531730652, + "learning_rate": 0.0001, + "loss": 71.0869, + "step": 2880 + }, + { + "epoch": 4.6546077492820075, + "grad_norm": 0.8356521129608154, + "learning_rate": 0.0001, + "loss": 72.1891, + "step": 2890 + }, + { + "epoch": 4.670731092860382, + "grad_norm": 0.8882647752761841, + "learning_rate": 0.0001, + "loss": 71.3845, + "step": 2900 + }, + { + "epoch": 4.6868544364387565, + "grad_norm": 1.1904003620147705, + "learning_rate": 0.0001, + "loss": 72.3086, + "step": 2910 + }, + { + "epoch": 4.702977780017131, + "grad_norm": 0.8010085821151733, + "learning_rate": 0.0001, + "loss": 71.3051, + "step": 2920 + }, + { + "epoch": 4.719101123595506, + "grad_norm": 0.7572135329246521, + "learning_rate": 0.0001, + "loss": 72.5605, + "step": 2930 + }, + { + "epoch": 4.735224467173881, + "grad_norm": 1.0866942405700684, + "learning_rate": 0.0001, + "loss": 70.0935, + "step": 2940 + }, + { + "epoch": 4.751347810752255, + "grad_norm": 0.9358721971511841, + "learning_rate": 0.0001, + "loss": 71.0391, + "step": 2950 + }, + { + "epoch": 4.76747115433063, + "grad_norm": 1.0276161432266235, + "learning_rate": 0.0001, + "loss": 69.8462, + "step": 2960 + }, + { + "epoch": 4.783594497909004, + "grad_norm": 0.9573063850402832, + "learning_rate": 0.0001, + "loss": 69.7089, + "step": 2970 + }, + { + "epoch": 4.799717841487379, + "grad_norm": 1.0638892650604248, + "learning_rate": 0.0001, + "loss": 71.8376, + "step": 2980 + }, + { + "epoch": 4.815841185065753, + "grad_norm": 0.9894681572914124, + "learning_rate": 0.0001, + "loss": 70.1764, + "step": 2990 + }, + { + "epoch": 4.831964528644128, + "grad_norm": 1.1265769004821777, + "learning_rate": 0.0001, + "loss": 71.7918, + "step": 3000 + }, + { + "epoch": 4.848087872222502, + "grad_norm": 2.2792513370513916, + "learning_rate": 0.0001, + "loss": 70.4341, + "step": 3010 + }, + { + "epoch": 4.864211215800877, + "grad_norm": 0.8469329476356506, + "learning_rate": 0.0001, + "loss": 72.1137, + "step": 3020 + }, + { + "epoch": 4.880334559379252, + "grad_norm": 0.9333013296127319, + "learning_rate": 0.0001, + "loss": 71.0789, + "step": 3030 + }, + { + "epoch": 4.896457902957626, + "grad_norm": 0.7888994216918945, + "learning_rate": 0.0001, + "loss": 70.8197, + "step": 3040 + }, + { + "epoch": 4.912581246536, + "grad_norm": 0.9194874167442322, + "learning_rate": 0.0001, + "loss": 72.071, + "step": 3050 + }, + { + "epoch": 4.928704590114375, + "grad_norm": 1.1252421140670776, + "learning_rate": 0.0001, + "loss": 71.6943, + "step": 3060 + }, + { + "epoch": 4.94482793369275, + "grad_norm": 1.1801071166992188, + "learning_rate": 0.0001, + "loss": 70.8937, + "step": 3070 + }, + { + "epoch": 4.960951277271124, + "grad_norm": 0.9162415266036987, + "learning_rate": 0.0001, + "loss": 70.024, + "step": 3080 + }, + { + "epoch": 4.977074620849499, + "grad_norm": 0.8833363652229309, + "learning_rate": 0.0001, + "loss": 71.3835, + "step": 3090 + }, + { + "epoch": 4.993197964427873, + "grad_norm": 0.9985883831977844, + "learning_rate": 0.0001, + "loss": 71.635, + "step": 3100 + }, + { + "epoch": 5.0080616717891875, + "grad_norm": 1.097298264503479, + "learning_rate": 0.0001, + "loss": 65.3542, + "step": 3110 + }, + { + "epoch": 5.024185015367562, + "grad_norm": 0.9703903198242188, + "learning_rate": 0.0001, + "loss": 69.8587, + "step": 3120 + }, + { + "epoch": 5.0403083589459365, + "grad_norm": 0.8894070386886597, + "learning_rate": 0.0001, + "loss": 71.2868, + "step": 3130 + }, + { + "epoch": 5.056431702524311, + "grad_norm": 0.6984127163887024, + "learning_rate": 0.0001, + "loss": 69.0425, + "step": 3140 + }, + { + "epoch": 5.072555046102686, + "grad_norm": 0.8607762455940247, + "learning_rate": 0.0001, + "loss": 69.5354, + "step": 3150 + }, + { + "epoch": 5.08867838968106, + "grad_norm": 1.1215914487838745, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 3160 + }, + { + "epoch": 5.104801733259435, + "grad_norm": 0.9114409685134888, + "learning_rate": 0.0001, + "loss": 68.4186, + "step": 3170 + }, + { + "epoch": 5.12092507683781, + "grad_norm": 1.010400652885437, + "learning_rate": 0.0001, + "loss": 68.3965, + "step": 3180 + }, + { + "epoch": 5.137048420416184, + "grad_norm": 0.8741958141326904, + "learning_rate": 0.0001, + "loss": 70.6728, + "step": 3190 + }, + { + "epoch": 5.153171763994559, + "grad_norm": 0.8760207891464233, + "learning_rate": 0.0001, + "loss": 72.0704, + "step": 3200 + }, + { + "epoch": 5.169295107572933, + "grad_norm": 1.328247308731079, + "learning_rate": 0.0001, + "loss": 69.9082, + "step": 3210 + }, + { + "epoch": 5.185418451151308, + "grad_norm": 0.8263765573501587, + "learning_rate": 0.0001, + "loss": 70.3539, + "step": 3220 + }, + { + "epoch": 5.201541794729682, + "grad_norm": 0.9358834028244019, + "learning_rate": 0.0001, + "loss": 70.4221, + "step": 3230 + }, + { + "epoch": 5.217665138308057, + "grad_norm": 1.0769531726837158, + "learning_rate": 0.0001, + "loss": 68.8167, + "step": 3240 + }, + { + "epoch": 5.233788481886431, + "grad_norm": 1.1438790559768677, + "learning_rate": 0.0001, + "loss": 68.9288, + "step": 3250 + }, + { + "epoch": 5.249911825464806, + "grad_norm": 0.9534218311309814, + "learning_rate": 0.0001, + "loss": 70.3277, + "step": 3260 + }, + { + "epoch": 5.26603516904318, + "grad_norm": 0.88532954454422, + "learning_rate": 0.0001, + "loss": 67.5781, + "step": 3270 + }, + { + "epoch": 5.282158512621555, + "grad_norm": 0.7986133694648743, + "learning_rate": 0.0001, + "loss": 69.759, + "step": 3280 + }, + { + "epoch": 5.29828185619993, + "grad_norm": 0.9100566506385803, + "learning_rate": 0.0001, + "loss": 68.6368, + "step": 3290 + }, + { + "epoch": 5.314405199778304, + "grad_norm": 0.8861745595932007, + "learning_rate": 0.0001, + "loss": 70.5756, + "step": 3300 + }, + { + "epoch": 5.330528543356679, + "grad_norm": 1.0729680061340332, + "learning_rate": 0.0001, + "loss": 70.3178, + "step": 3310 + }, + { + "epoch": 5.346651886935053, + "grad_norm": 1.0288280248641968, + "learning_rate": 0.0001, + "loss": 69.422, + "step": 3320 + }, + { + "epoch": 5.362775230513428, + "grad_norm": 0.8335140347480774, + "learning_rate": 0.0001, + "loss": 69.8281, + "step": 3330 + }, + { + "epoch": 5.378898574091802, + "grad_norm": 0.9789717197418213, + "learning_rate": 0.0001, + "loss": 71.1532, + "step": 3340 + }, + { + "epoch": 5.395021917670177, + "grad_norm": 0.9884677529335022, + "learning_rate": 0.0001, + "loss": 69.5673, + "step": 3350 + }, + { + "epoch": 5.411145261248551, + "grad_norm": 1.050154447555542, + "learning_rate": 0.0001, + "loss": 70.8015, + "step": 3360 + }, + { + "epoch": 5.427268604826926, + "grad_norm": 0.9645342230796814, + "learning_rate": 0.0001, + "loss": 69.1615, + "step": 3370 + }, + { + "epoch": 5.4433919484053, + "grad_norm": 0.8383159041404724, + "learning_rate": 0.0001, + "loss": 70.6299, + "step": 3380 + }, + { + "epoch": 5.459515291983675, + "grad_norm": 0.9250556826591492, + "learning_rate": 0.0001, + "loss": 69.231, + "step": 3390 + }, + { + "epoch": 5.475638635562049, + "grad_norm": 0.8108687400817871, + "learning_rate": 0.0001, + "loss": 70.5858, + "step": 3400 + }, + { + "epoch": 5.491761979140424, + "grad_norm": 0.8748818039894104, + "learning_rate": 0.0001, + "loss": 69.9001, + "step": 3410 + }, + { + "epoch": 5.507885322718799, + "grad_norm": 0.7411381006240845, + "learning_rate": 0.0001, + "loss": 68.3248, + "step": 3420 + }, + { + "epoch": 5.524008666297173, + "grad_norm": 1.0753326416015625, + "learning_rate": 0.0001, + "loss": 69.5831, + "step": 3430 + }, + { + "epoch": 5.540132009875548, + "grad_norm": 0.875254213809967, + "learning_rate": 0.0001, + "loss": 69.7516, + "step": 3440 + }, + { + "epoch": 5.556255353453922, + "grad_norm": 0.7637733817100525, + "learning_rate": 0.0001, + "loss": 68.0078, + "step": 3450 + }, + { + "epoch": 5.572378697032297, + "grad_norm": 0.968203604221344, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 3460 + }, + { + "epoch": 5.588502040610671, + "grad_norm": 0.8903141617774963, + "learning_rate": 0.0001, + "loss": 68.5835, + "step": 3470 + }, + { + "epoch": 5.604625384189046, + "grad_norm": 0.9536195397377014, + "learning_rate": 0.0001, + "loss": 71.0259, + "step": 3480 + }, + { + "epoch": 5.62074872776742, + "grad_norm": 0.8951714038848877, + "learning_rate": 0.0001, + "loss": 69.4422, + "step": 3490 + }, + { + "epoch": 5.636872071345795, + "grad_norm": 0.9639522433280945, + "learning_rate": 0.0001, + "loss": 66.5173, + "step": 3500 + }, + { + "epoch": 5.65299541492417, + "grad_norm": 1.0022730827331543, + "learning_rate": 0.0001, + "loss": 68.3443, + "step": 3510 + }, + { + "epoch": 5.669118758502544, + "grad_norm": 1.0654487609863281, + "learning_rate": 0.0001, + "loss": 69.0835, + "step": 3520 + }, + { + "epoch": 5.685242102080919, + "grad_norm": 0.9934629797935486, + "learning_rate": 0.0001, + "loss": 69.2278, + "step": 3530 + }, + { + "epoch": 5.7013654456592935, + "grad_norm": 0.8773566484451294, + "learning_rate": 0.0001, + "loss": 68.9339, + "step": 3540 + }, + { + "epoch": 5.7174887892376685, + "grad_norm": 0.7756998538970947, + "learning_rate": 0.0001, + "loss": 69.2683, + "step": 3550 + }, + { + "epoch": 5.7336121328160425, + "grad_norm": 0.8819359540939331, + "learning_rate": 0.0001, + "loss": 68.5071, + "step": 3560 + }, + { + "epoch": 5.7497354763944175, + "grad_norm": 1.1069825887680054, + "learning_rate": 0.0001, + "loss": 68.5236, + "step": 3570 + }, + { + "epoch": 5.765858819972792, + "grad_norm": 0.8006312847137451, + "learning_rate": 0.0001, + "loss": 68.7192, + "step": 3580 + }, + { + "epoch": 5.781982163551167, + "grad_norm": 1.0278719663619995, + "learning_rate": 0.0001, + "loss": 69.2545, + "step": 3590 + }, + { + "epoch": 5.798105507129541, + "grad_norm": 0.9205041527748108, + "learning_rate": 0.0001, + "loss": 69.7698, + "step": 3600 + }, + { + "epoch": 5.814228850707916, + "grad_norm": 1.0769038200378418, + "learning_rate": 0.0001, + "loss": 67.6568, + "step": 3610 + }, + { + "epoch": 5.83035219428629, + "grad_norm": 1.0565663576126099, + "learning_rate": 0.0001, + "loss": 68.9166, + "step": 3620 + }, + { + "epoch": 5.846475537864665, + "grad_norm": 0.9008663296699524, + "learning_rate": 0.0001, + "loss": 69.5837, + "step": 3630 + }, + { + "epoch": 5.86259888144304, + "grad_norm": 0.8369411826133728, + "learning_rate": 0.0001, + "loss": 68.5661, + "step": 3640 + }, + { + "epoch": 5.878722225021414, + "grad_norm": 1.0472379922866821, + "learning_rate": 0.0001, + "loss": 68.3122, + "step": 3650 + }, + { + "epoch": 5.894845568599789, + "grad_norm": 0.965277373790741, + "learning_rate": 0.0001, + "loss": 69.8062, + "step": 3660 + }, + { + "epoch": 5.910968912178163, + "grad_norm": 0.8936495780944824, + "learning_rate": 0.0001, + "loss": 67.1016, + "step": 3670 + }, + { + "epoch": 5.927092255756538, + "grad_norm": 0.8559162020683289, + "learning_rate": 0.0001, + "loss": 69.9986, + "step": 3680 + }, + { + "epoch": 5.943215599334912, + "grad_norm": 1.4815773963928223, + "learning_rate": 0.0001, + "loss": 66.7648, + "step": 3690 + }, + { + "epoch": 5.959338942913287, + "grad_norm": 0.9034469723701477, + "learning_rate": 0.0001, + "loss": 68.0767, + "step": 3700 + }, + { + "epoch": 5.975462286491661, + "grad_norm": 1.838799238204956, + "learning_rate": 0.0001, + "loss": 66.5848, + "step": 3710 + }, + { + "epoch": 5.991585630070036, + "grad_norm": 0.9274787902832031, + "learning_rate": 0.0001, + "loss": 70.0396, + "step": 3720 + }, + { + "epoch": 6.0064493374313495, + "grad_norm": 1.016974687576294, + "learning_rate": 0.0001, + "loss": 62.2268, + "step": 3730 + }, + { + "epoch": 6.022572681009724, + "grad_norm": 1.0522642135620117, + "learning_rate": 0.0001, + "loss": 69.3381, + "step": 3740 + }, + { + "epoch": 6.0386960245880985, + "grad_norm": 1.1157186031341553, + "learning_rate": 0.0001, + "loss": 68.8969, + "step": 3750 + }, + { + "epoch": 6.0548193681664735, + "grad_norm": 0.9913288950920105, + "learning_rate": 0.0001, + "loss": 67.3634, + "step": 3760 + }, + { + "epoch": 6.0709427117448485, + "grad_norm": 0.8823490142822266, + "learning_rate": 0.0001, + "loss": 70.978, + "step": 3770 + }, + { + "epoch": 6.0870660553232225, + "grad_norm": 0.853837788105011, + "learning_rate": 0.0001, + "loss": 68.9924, + "step": 3780 + }, + { + "epoch": 6.1031893989015975, + "grad_norm": 0.7937766909599304, + "learning_rate": 0.0001, + "loss": 68.1994, + "step": 3790 + }, + { + "epoch": 6.119312742479972, + "grad_norm": 1.7638057470321655, + "learning_rate": 0.0001, + "loss": 68.5078, + "step": 3800 + }, + { + "epoch": 6.135436086058347, + "grad_norm": 1.0542887449264526, + "learning_rate": 0.0001, + "loss": 68.9362, + "step": 3810 + }, + { + "epoch": 6.151559429636721, + "grad_norm": 1.06019926071167, + "learning_rate": 0.0001, + "loss": 67.2493, + "step": 3820 + }, + { + "epoch": 6.167682773215096, + "grad_norm": 0.9765111804008484, + "learning_rate": 0.0001, + "loss": 70.4797, + "step": 3830 + }, + { + "epoch": 6.18380611679347, + "grad_norm": 1.006372332572937, + "learning_rate": 0.0001, + "loss": 69.4233, + "step": 3840 + }, + { + "epoch": 6.199929460371845, + "grad_norm": 0.8274241089820862, + "learning_rate": 0.0001, + "loss": 67.9907, + "step": 3850 + }, + { + "epoch": 6.216052803950219, + "grad_norm": 0.9905765056610107, + "learning_rate": 0.0001, + "loss": 69.6988, + "step": 3860 + }, + { + "epoch": 6.232176147528594, + "grad_norm": 1.0065921545028687, + "learning_rate": 0.0001, + "loss": 68.7678, + "step": 3870 + }, + { + "epoch": 6.248299491106969, + "grad_norm": 0.8845255970954895, + "learning_rate": 0.0001, + "loss": 68.3039, + "step": 3880 + }, + { + "epoch": 6.264422834685343, + "grad_norm": 0.9030968546867371, + "learning_rate": 0.0001, + "loss": 68.671, + "step": 3890 + }, + { + "epoch": 6.280546178263718, + "grad_norm": 0.8650258183479309, + "learning_rate": 0.0001, + "loss": 68.5309, + "step": 3900 + }, + { + "epoch": 6.296669521842092, + "grad_norm": 0.8961272835731506, + "learning_rate": 0.0001, + "loss": 67.5611, + "step": 3910 + }, + { + "epoch": 6.312792865420467, + "grad_norm": 1.1702407598495483, + "learning_rate": 0.0001, + "loss": 68.2768, + "step": 3920 + }, + { + "epoch": 6.328916208998841, + "grad_norm": 0.8598962426185608, + "learning_rate": 0.0001, + "loss": 69.1961, + "step": 3930 + }, + { + "epoch": 6.345039552577216, + "grad_norm": 0.8714122176170349, + "learning_rate": 0.0001, + "loss": 66.8628, + "step": 3940 + }, + { + "epoch": 6.36116289615559, + "grad_norm": 0.7757895588874817, + "learning_rate": 0.0001, + "loss": 68.6908, + "step": 3950 + }, + { + "epoch": 6.377286239733965, + "grad_norm": 0.9189490675926208, + "learning_rate": 0.0001, + "loss": 67.8685, + "step": 3960 + }, + { + "epoch": 6.393409583312339, + "grad_norm": 0.8161299228668213, + "learning_rate": 0.0001, + "loss": 66.816, + "step": 3970 + }, + { + "epoch": 6.409532926890714, + "grad_norm": 0.8559791445732117, + "learning_rate": 0.0001, + "loss": 68.4302, + "step": 3980 + }, + { + "epoch": 6.425656270469089, + "grad_norm": 0.8087984919548035, + "learning_rate": 0.0001, + "loss": 66.2646, + "step": 3990 + }, + { + "epoch": 6.441779614047463, + "grad_norm": 0.7733110785484314, + "learning_rate": 0.0001, + "loss": 69.2769, + "step": 4000 + }, + { + "epoch": 6.457902957625838, + "grad_norm": 0.9335984587669373, + "learning_rate": 0.0001, + "loss": 67.1199, + "step": 4010 + }, + { + "epoch": 6.474026301204212, + "grad_norm": 0.9546418786048889, + "learning_rate": 0.0001, + "loss": 68.8706, + "step": 4020 + }, + { + "epoch": 6.490149644782587, + "grad_norm": 0.9670593738555908, + "learning_rate": 0.0001, + "loss": 68.2329, + "step": 4030 + }, + { + "epoch": 6.506272988360961, + "grad_norm": 0.8428588509559631, + "learning_rate": 0.0001, + "loss": 68.0105, + "step": 4040 + }, + { + "epoch": 6.522396331939336, + "grad_norm": 0.9755622744560242, + "learning_rate": 0.0001, + "loss": 68.656, + "step": 4050 + }, + { + "epoch": 6.53851967551771, + "grad_norm": 0.8972346186637878, + "learning_rate": 0.0001, + "loss": 69.3665, + "step": 4060 + }, + { + "epoch": 6.554643019096085, + "grad_norm": 0.9716668128967285, + "learning_rate": 0.0001, + "loss": 68.1053, + "step": 4070 + }, + { + "epoch": 6.570766362674459, + "grad_norm": 0.8147910833358765, + "learning_rate": 0.0001, + "loss": 67.9778, + "step": 4080 + }, + { + "epoch": 6.586889706252834, + "grad_norm": 0.8304234147071838, + "learning_rate": 0.0001, + "loss": 67.7069, + "step": 4090 + }, + { + "epoch": 6.603013049831208, + "grad_norm": 0.9356856346130371, + "learning_rate": 0.0001, + "loss": 68.1942, + "step": 4100 + }, + { + "epoch": 6.619136393409583, + "grad_norm": 1.017134189605713, + "learning_rate": 0.0001, + "loss": 67.4211, + "step": 4110 + }, + { + "epoch": 6.635259736987958, + "grad_norm": 1.142558217048645, + "learning_rate": 0.0001, + "loss": 68.4413, + "step": 4120 + }, + { + "epoch": 6.651383080566332, + "grad_norm": 0.9071153998374939, + "learning_rate": 0.0001, + "loss": 69.3666, + "step": 4130 + }, + { + "epoch": 6.667506424144707, + "grad_norm": 0.9620397686958313, + "learning_rate": 0.0001, + "loss": 68.208, + "step": 4140 + }, + { + "epoch": 6.683629767723081, + "grad_norm": 0.8282128572463989, + "learning_rate": 0.0001, + "loss": 67.1702, + "step": 4150 + }, + { + "epoch": 6.699753111301456, + "grad_norm": 0.8838199973106384, + "learning_rate": 0.0001, + "loss": 69.0074, + "step": 4160 + }, + { + "epoch": 6.71587645487983, + "grad_norm": 0.8864356279373169, + "learning_rate": 0.0001, + "loss": 67.8052, + "step": 4170 + }, + { + "epoch": 6.731999798458205, + "grad_norm": 0.9373385906219482, + "learning_rate": 0.0001, + "loss": 67.446, + "step": 4180 + }, + { + "epoch": 6.7481231420365795, + "grad_norm": 0.8308956027030945, + "learning_rate": 0.0001, + "loss": 68.716, + "step": 4190 + }, + { + "epoch": 6.7642464856149545, + "grad_norm": 0.8031372427940369, + "learning_rate": 0.0001, + "loss": 68.0482, + "step": 4200 + }, + { + "epoch": 6.780369829193329, + "grad_norm": 0.9258586764335632, + "learning_rate": 0.0001, + "loss": 68.5607, + "step": 4210 + }, + { + "epoch": 6.7964931727717035, + "grad_norm": 0.8147327899932861, + "learning_rate": 0.0001, + "loss": 67.7265, + "step": 4220 + }, + { + "epoch": 6.8126165163500785, + "grad_norm": 0.8717535734176636, + "learning_rate": 0.0001, + "loss": 67.8097, + "step": 4230 + }, + { + "epoch": 6.828739859928453, + "grad_norm": 0.9876209497451782, + "learning_rate": 0.0001, + "loss": 67.9068, + "step": 4240 + }, + { + "epoch": 6.8448632035068275, + "grad_norm": 0.718376874923706, + "learning_rate": 0.0001, + "loss": 68.1762, + "step": 4250 + }, + { + "epoch": 6.860986547085202, + "grad_norm": 0.8264320492744446, + "learning_rate": 0.0001, + "loss": 67.8391, + "step": 4260 + }, + { + "epoch": 6.877109890663577, + "grad_norm": 0.7478179335594177, + "learning_rate": 0.0001, + "loss": 67.6635, + "step": 4270 + }, + { + "epoch": 6.893233234241951, + "grad_norm": 0.8394957184791565, + "learning_rate": 0.0001, + "loss": 68.6073, + "step": 4280 + }, + { + "epoch": 6.909356577820326, + "grad_norm": 0.8534071445465088, + "learning_rate": 0.0001, + "loss": 69.6206, + "step": 4290 + }, + { + "epoch": 6.9254799213987, + "grad_norm": 0.9137582778930664, + "learning_rate": 0.0001, + "loss": 67.4362, + "step": 4300 + }, + { + "epoch": 6.941603264977075, + "grad_norm": 0.8869971036911011, + "learning_rate": 0.0001, + "loss": 67.6811, + "step": 4310 + }, + { + "epoch": 6.957726608555449, + "grad_norm": 0.889025866985321, + "learning_rate": 0.0001, + "loss": 65.3354, + "step": 4320 + }, + { + "epoch": 6.973849952133824, + "grad_norm": 0.7968549728393555, + "learning_rate": 0.0001, + "loss": 66.192, + "step": 4330 + }, + { + "epoch": 6.989973295712199, + "grad_norm": 1.0077433586120605, + "learning_rate": 0.0001, + "loss": 67.5216, + "step": 4340 + }, + { + "epoch": 7.004837003073512, + "grad_norm": 0.7900649309158325, + "learning_rate": 0.0001, + "loss": 62.2421, + "step": 4350 + }, + { + "epoch": 7.020960346651887, + "grad_norm": 1.0160410404205322, + "learning_rate": 0.0001, + "loss": 68.0978, + "step": 4360 + }, + { + "epoch": 7.037083690230261, + "grad_norm": 0.841387927532196, + "learning_rate": 0.0001, + "loss": 68.1008, + "step": 4370 + }, + { + "epoch": 7.053207033808636, + "grad_norm": 0.87884122133255, + "learning_rate": 0.0001, + "loss": 66.4667, + "step": 4380 + }, + { + "epoch": 7.06933037738701, + "grad_norm": 0.8580564856529236, + "learning_rate": 0.0001, + "loss": 67.2002, + "step": 4390 + }, + { + "epoch": 7.085453720965385, + "grad_norm": 0.7560063600540161, + "learning_rate": 0.0001, + "loss": 67.6743, + "step": 4400 + }, + { + "epoch": 7.1015770645437595, + "grad_norm": 0.855478048324585, + "learning_rate": 0.0001, + "loss": 67.822, + "step": 4410 + }, + { + "epoch": 7.1177004081221344, + "grad_norm": 0.7994911670684814, + "learning_rate": 0.0001, + "loss": 66.0094, + "step": 4420 + }, + { + "epoch": 7.1338237517005085, + "grad_norm": 0.8728616833686829, + "learning_rate": 0.0001, + "loss": 66.405, + "step": 4430 + }, + { + "epoch": 7.1499470952788835, + "grad_norm": 0.8434081673622131, + "learning_rate": 0.0001, + "loss": 67.0535, + "step": 4440 + }, + { + "epoch": 7.166070438857258, + "grad_norm": 0.8543094396591187, + "learning_rate": 0.0001, + "loss": 66.2417, + "step": 4450 + }, + { + "epoch": 7.182193782435633, + "grad_norm": 0.7209319472312927, + "learning_rate": 0.0001, + "loss": 67.3958, + "step": 4460 + }, + { + "epoch": 7.1983171260140075, + "grad_norm": 0.6735154390335083, + "learning_rate": 0.0001, + "loss": 67.4086, + "step": 4470 + }, + { + "epoch": 7.214440469592382, + "grad_norm": 0.9126104116439819, + "learning_rate": 0.0001, + "loss": 68.4191, + "step": 4480 + }, + { + "epoch": 7.230563813170757, + "grad_norm": 0.9334300756454468, + "learning_rate": 0.0001, + "loss": 68.3247, + "step": 4490 + }, + { + "epoch": 7.246687156749131, + "grad_norm": 1.5956038236618042, + "learning_rate": 0.0001, + "loss": 68.2779, + "step": 4500 + }, + { + "epoch": 7.262810500327506, + "grad_norm": 1.5012775659561157, + "learning_rate": 0.0001, + "loss": 68.3533, + "step": 4510 + }, + { + "epoch": 7.27893384390588, + "grad_norm": 1.1376632452011108, + "learning_rate": 0.0001, + "loss": 65.3627, + "step": 4520 + }, + { + "epoch": 7.295057187484255, + "grad_norm": 0.8597291111946106, + "learning_rate": 0.0001, + "loss": 67.8653, + "step": 4530 + }, + { + "epoch": 7.311180531062629, + "grad_norm": 0.7645048499107361, + "learning_rate": 0.0001, + "loss": 65.4592, + "step": 4540 + }, + { + "epoch": 7.327303874641004, + "grad_norm": 0.7717318534851074, + "learning_rate": 0.0001, + "loss": 66.902, + "step": 4550 + }, + { + "epoch": 7.343427218219378, + "grad_norm": 0.8884950876235962, + "learning_rate": 0.0001, + "loss": 67.1858, + "step": 4560 + }, + { + "epoch": 7.359550561797753, + "grad_norm": 0.7826898694038391, + "learning_rate": 0.0001, + "loss": 68.4673, + "step": 4570 + }, + { + "epoch": 7.375673905376128, + "grad_norm": 0.7484182119369507, + "learning_rate": 0.0001, + "loss": 67.4859, + "step": 4580 + }, + { + "epoch": 7.391797248954502, + "grad_norm": 0.918194591999054, + "learning_rate": 0.0001, + "loss": 67.7393, + "step": 4590 + }, + { + "epoch": 7.407920592532877, + "grad_norm": 1.0696896314620972, + "learning_rate": 0.0001, + "loss": 67.0101, + "step": 4600 + }, + { + "epoch": 7.424043936111251, + "grad_norm": 0.8190692067146301, + "learning_rate": 0.0001, + "loss": 68.5358, + "step": 4610 + }, + { + "epoch": 7.440167279689626, + "grad_norm": 0.9063783884048462, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 4620 + }, + { + "epoch": 7.456290623268, + "grad_norm": 0.7588100433349609, + "learning_rate": 0.0001, + "loss": 68.0066, + "step": 4630 + }, + { + "epoch": 7.472413966846375, + "grad_norm": 0.897281289100647, + "learning_rate": 0.0001, + "loss": 67.2499, + "step": 4640 + }, + { + "epoch": 7.488537310424749, + "grad_norm": 0.9505569338798523, + "learning_rate": 0.0001, + "loss": 67.5452, + "step": 4650 + }, + { + "epoch": 7.504660654003124, + "grad_norm": 0.8409159183502197, + "learning_rate": 0.0001, + "loss": 66.6293, + "step": 4660 + }, + { + "epoch": 7.520783997581498, + "grad_norm": 0.9878574013710022, + "learning_rate": 0.0001, + "loss": 65.3379, + "step": 4670 + }, + { + "epoch": 7.536907341159873, + "grad_norm": 1.0410685539245605, + "learning_rate": 0.0001, + "loss": 65.8028, + "step": 4680 + }, + { + "epoch": 7.553030684738248, + "grad_norm": 0.8417725563049316, + "learning_rate": 0.0001, + "loss": 65.4283, + "step": 4690 + }, + { + "epoch": 7.569154028316622, + "grad_norm": 0.7660863399505615, + "learning_rate": 0.0001, + "loss": 66.6765, + "step": 4700 + }, + { + "epoch": 7.585277371894997, + "grad_norm": 0.8074233531951904, + "learning_rate": 0.0001, + "loss": 67.1082, + "step": 4710 + }, + { + "epoch": 7.601400715473371, + "grad_norm": 0.7653326988220215, + "learning_rate": 0.0001, + "loss": 66.028, + "step": 4720 + }, + { + "epoch": 7.617524059051746, + "grad_norm": 1.1764041185379028, + "learning_rate": 0.0001, + "loss": 65.2587, + "step": 4730 + }, + { + "epoch": 7.63364740263012, + "grad_norm": 1.7604950666427612, + "learning_rate": 0.0001, + "loss": 66.0543, + "step": 4740 + }, + { + "epoch": 7.649770746208495, + "grad_norm": 0.941438615322113, + "learning_rate": 0.0001, + "loss": 65.6954, + "step": 4750 + }, + { + "epoch": 7.665894089786869, + "grad_norm": 0.740069568157196, + "learning_rate": 0.0001, + "loss": 67.9701, + "step": 4760 + }, + { + "epoch": 7.682017433365244, + "grad_norm": 0.9867332577705383, + "learning_rate": 0.0001, + "loss": 66.1445, + "step": 4770 + }, + { + "epoch": 7.698140776943618, + "grad_norm": 0.8156166672706604, + "learning_rate": 0.0001, + "loss": 67.2647, + "step": 4780 + }, + { + "epoch": 7.714264120521993, + "grad_norm": 0.7689579129219055, + "learning_rate": 0.0001, + "loss": 67.0961, + "step": 4790 + }, + { + "epoch": 7.730387464100367, + "grad_norm": 0.8159700036048889, + "learning_rate": 0.0001, + "loss": 66.4974, + "step": 4800 + }, + { + "epoch": 7.746510807678742, + "grad_norm": 0.8554222583770752, + "learning_rate": 0.0001, + "loss": 64.5057, + "step": 4810 + }, + { + "epoch": 7.762634151257117, + "grad_norm": 0.856629490852356, + "learning_rate": 0.0001, + "loss": 67.7819, + "step": 4820 + }, + { + "epoch": 7.778757494835491, + "grad_norm": 1.157179832458496, + "learning_rate": 0.0001, + "loss": 66.0122, + "step": 4830 + }, + { + "epoch": 7.794880838413866, + "grad_norm": 0.9597711563110352, + "learning_rate": 0.0001, + "loss": 66.8248, + "step": 4840 + }, + { + "epoch": 7.8110041819922404, + "grad_norm": 0.9175744652748108, + "learning_rate": 0.0001, + "loss": 65.7998, + "step": 4850 + }, + { + "epoch": 7.827127525570615, + "grad_norm": 0.8467298150062561, + "learning_rate": 0.0001, + "loss": 67.5363, + "step": 4860 + }, + { + "epoch": 7.8432508691489895, + "grad_norm": 0.863922655582428, + "learning_rate": 0.0001, + "loss": 67.7219, + "step": 4870 + }, + { + "epoch": 7.8593742127273645, + "grad_norm": 0.9896290302276611, + "learning_rate": 0.0001, + "loss": 66.7098, + "step": 4880 + }, + { + "epoch": 7.8754975563057386, + "grad_norm": 0.8828520774841309, + "learning_rate": 0.0001, + "loss": 66.0192, + "step": 4890 + }, + { + "epoch": 7.8916208998841135, + "grad_norm": 0.9093656539916992, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 4900 + }, + { + "epoch": 7.9077442434624885, + "grad_norm": 0.8665091395378113, + "learning_rate": 0.0001, + "loss": 66.5851, + "step": 4910 + }, + { + "epoch": 7.923867587040863, + "grad_norm": 0.8027398586273193, + "learning_rate": 0.0001, + "loss": 64.8607, + "step": 4920 + }, + { + "epoch": 7.9399909306192376, + "grad_norm": 0.7808436751365662, + "learning_rate": 0.0001, + "loss": 65.052, + "step": 4930 + }, + { + "epoch": 7.956114274197612, + "grad_norm": 0.7246070504188538, + "learning_rate": 0.0001, + "loss": 64.789, + "step": 4940 + }, + { + "epoch": 7.972237617775987, + "grad_norm": 1.0405027866363525, + "learning_rate": 0.0001, + "loss": 66.7735, + "step": 4950 + }, + { + "epoch": 7.988360961354361, + "grad_norm": 0.812921404838562, + "learning_rate": 0.0001, + "loss": 64.1286, + "step": 4960 + }, + { + "epoch": 8.003224668715674, + "grad_norm": 0.884215772151947, + "learning_rate": 0.0001, + "loss": 61.0812, + "step": 4970 + }, + { + "epoch": 8.01934801229405, + "grad_norm": 0.8519641757011414, + "learning_rate": 0.0001, + "loss": 66.2784, + "step": 4980 + }, + { + "epoch": 8.035471355872424, + "grad_norm": 0.8331199884414673, + "learning_rate": 0.0001, + "loss": 67.1405, + "step": 4990 + }, + { + "epoch": 8.0515946994508, + "grad_norm": 0.8568746447563171, + "learning_rate": 0.0001, + "loss": 66.7835, + "step": 5000 + }, + { + "epoch": 8.067718043029172, + "grad_norm": 1.170482873916626, + "learning_rate": 0.0001, + "loss": 64.2269, + "step": 5010 + }, + { + "epoch": 8.083841386607547, + "grad_norm": 0.9171113967895508, + "learning_rate": 0.0001, + "loss": 66.5567, + "step": 5020 + }, + { + "epoch": 8.099964730185922, + "grad_norm": 0.8988714814186096, + "learning_rate": 0.0001, + "loss": 66.0471, + "step": 5030 + }, + { + "epoch": 8.116088073764297, + "grad_norm": 0.7651671171188354, + "learning_rate": 0.0001, + "loss": 65.7185, + "step": 5040 + }, + { + "epoch": 8.132211417342672, + "grad_norm": 0.6890843510627747, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 5050 + }, + { + "epoch": 8.148334760921045, + "grad_norm": 0.7683818340301514, + "learning_rate": 0.0001, + "loss": 65.895, + "step": 5060 + }, + { + "epoch": 8.16445810449942, + "grad_norm": 0.9398573637008667, + "learning_rate": 0.0001, + "loss": 64.674, + "step": 5070 + }, + { + "epoch": 8.180581448077795, + "grad_norm": 0.7708531022071838, + "learning_rate": 0.0001, + "loss": 65.0637, + "step": 5080 + }, + { + "epoch": 8.19670479165617, + "grad_norm": 0.8840665221214294, + "learning_rate": 0.0001, + "loss": 65.6449, + "step": 5090 + }, + { + "epoch": 8.212828135234544, + "grad_norm": 0.8390524983406067, + "learning_rate": 0.0001, + "loss": 65.4738, + "step": 5100 + }, + { + "epoch": 8.228951478812919, + "grad_norm": 1.7559128999710083, + "learning_rate": 0.0001, + "loss": 68.911, + "step": 5110 + }, + { + "epoch": 8.245074822391294, + "grad_norm": 0.8293817043304443, + "learning_rate": 0.0001, + "loss": 65.2601, + "step": 5120 + }, + { + "epoch": 8.261198165969668, + "grad_norm": 0.8947123289108276, + "learning_rate": 0.0001, + "loss": 65.8611, + "step": 5130 + }, + { + "epoch": 8.277321509548042, + "grad_norm": 0.8526618480682373, + "learning_rate": 0.0001, + "loss": 65.559, + "step": 5140 + }, + { + "epoch": 8.293444853126417, + "grad_norm": 1.7423408031463623, + "learning_rate": 0.0001, + "loss": 64.5007, + "step": 5150 + }, + { + "epoch": 8.309568196704792, + "grad_norm": 1.0512502193450928, + "learning_rate": 0.0001, + "loss": 66.2593, + "step": 5160 + }, + { + "epoch": 8.325691540283167, + "grad_norm": 0.8129869103431702, + "learning_rate": 0.0001, + "loss": 64.7989, + "step": 5170 + }, + { + "epoch": 8.341814883861542, + "grad_norm": 0.852199912071228, + "learning_rate": 0.0001, + "loss": 65.9627, + "step": 5180 + }, + { + "epoch": 8.357938227439915, + "grad_norm": 0.8932474851608276, + "learning_rate": 0.0001, + "loss": 64.1035, + "step": 5190 + }, + { + "epoch": 8.37406157101829, + "grad_norm": 1.2631597518920898, + "learning_rate": 0.0001, + "loss": 65.3406, + "step": 5200 + }, + { + "epoch": 8.390184914596665, + "grad_norm": 0.9062619209289551, + "learning_rate": 0.0001, + "loss": 65.1841, + "step": 5210 + }, + { + "epoch": 8.40630825817504, + "grad_norm": 0.9153940081596375, + "learning_rate": 0.0001, + "loss": 66.3852, + "step": 5220 + }, + { + "epoch": 8.422431601753413, + "grad_norm": 0.8056164979934692, + "learning_rate": 0.0001, + "loss": 66.511, + "step": 5230 + }, + { + "epoch": 8.438554945331788, + "grad_norm": 0.7506036162376404, + "learning_rate": 0.0001, + "loss": 65.275, + "step": 5240 + }, + { + "epoch": 8.454678288910163, + "grad_norm": 0.7994081377983093, + "learning_rate": 0.0001, + "loss": 64.9946, + "step": 5250 + }, + { + "epoch": 8.470801632488538, + "grad_norm": 0.9887873530387878, + "learning_rate": 0.0001, + "loss": 67.2057, + "step": 5260 + }, + { + "epoch": 8.486924976066913, + "grad_norm": 0.8495044112205505, + "learning_rate": 0.0001, + "loss": 66.2481, + "step": 5270 + }, + { + "epoch": 8.503048319645286, + "grad_norm": 0.7353820204734802, + "learning_rate": 0.0001, + "loss": 65.6163, + "step": 5280 + }, + { + "epoch": 8.519171663223661, + "grad_norm": 0.795216977596283, + "learning_rate": 0.0001, + "loss": 64.808, + "step": 5290 + }, + { + "epoch": 8.535295006802036, + "grad_norm": 0.8121851682662964, + "learning_rate": 0.0001, + "loss": 65.9755, + "step": 5300 + }, + { + "epoch": 8.55141835038041, + "grad_norm": 0.8164243698120117, + "learning_rate": 0.0001, + "loss": 66.0583, + "step": 5310 + }, + { + "epoch": 8.567541693958784, + "grad_norm": 0.9893137216567993, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 5320 + }, + { + "epoch": 8.583665037537159, + "grad_norm": 1.3616434335708618, + "learning_rate": 0.0001, + "loss": 64.1444, + "step": 5330 + }, + { + "epoch": 8.599788381115534, + "grad_norm": 0.7002740502357483, + "learning_rate": 0.0001, + "loss": 65.0906, + "step": 5340 + }, + { + "epoch": 8.615911724693909, + "grad_norm": 0.838424563407898, + "learning_rate": 0.0001, + "loss": 65.4578, + "step": 5350 + }, + { + "epoch": 8.632035068272282, + "grad_norm": 0.988902747631073, + "learning_rate": 0.0001, + "loss": 64.3405, + "step": 5360 + }, + { + "epoch": 8.648158411850657, + "grad_norm": 0.7941252589225769, + "learning_rate": 0.0001, + "loss": 67.0146, + "step": 5370 + }, + { + "epoch": 8.664281755429032, + "grad_norm": 0.7727184295654297, + "learning_rate": 0.0001, + "loss": 64.8011, + "step": 5380 + }, + { + "epoch": 8.680405099007407, + "grad_norm": 0.9514367580413818, + "learning_rate": 0.0001, + "loss": 65.8165, + "step": 5390 + }, + { + "epoch": 8.69652844258578, + "grad_norm": 0.870544970035553, + "learning_rate": 0.0001, + "loss": 65.803, + "step": 5400 + }, + { + "epoch": 8.712651786164155, + "grad_norm": 0.7348082065582275, + "learning_rate": 0.0001, + "loss": 65.8409, + "step": 5410 + }, + { + "epoch": 8.72877512974253, + "grad_norm": 0.7543814778327942, + "learning_rate": 0.0001, + "loss": 65.5958, + "step": 5420 + }, + { + "epoch": 8.744898473320905, + "grad_norm": 0.7520368099212646, + "learning_rate": 0.0001, + "loss": 65.3323, + "step": 5430 + }, + { + "epoch": 8.76102181689928, + "grad_norm": 0.7610031366348267, + "learning_rate": 0.0001, + "loss": 65.4386, + "step": 5440 + }, + { + "epoch": 8.777145160477653, + "grad_norm": 0.8854930400848389, + "learning_rate": 0.0001, + "loss": 66.181, + "step": 5450 + }, + { + "epoch": 8.793268504056028, + "grad_norm": 0.7659342885017395, + "learning_rate": 0.0001, + "loss": 65.2586, + "step": 5460 + }, + { + "epoch": 8.809391847634403, + "grad_norm": 0.8592572212219238, + "learning_rate": 0.0001, + "loss": 66.3306, + "step": 5470 + }, + { + "epoch": 8.825515191212778, + "grad_norm": 0.8169172406196594, + "learning_rate": 0.0001, + "loss": 65.8283, + "step": 5480 + }, + { + "epoch": 8.841638534791151, + "grad_norm": 0.7380614876747131, + "learning_rate": 0.0001, + "loss": 65.8711, + "step": 5490 + }, + { + "epoch": 8.857761878369526, + "grad_norm": 0.9993331432342529, + "learning_rate": 0.0001, + "loss": 64.9378, + "step": 5500 + }, + { + "epoch": 8.873885221947901, + "grad_norm": 0.6252862811088562, + "learning_rate": 0.0001, + "loss": 64.1655, + "step": 5510 + }, + { + "epoch": 8.890008565526276, + "grad_norm": 0.8441852331161499, + "learning_rate": 0.0001, + "loss": 64.9795, + "step": 5520 + }, + { + "epoch": 8.906131909104651, + "grad_norm": 0.7075499892234802, + "learning_rate": 0.0001, + "loss": 65.5843, + "step": 5530 + }, + { + "epoch": 8.922255252683025, + "grad_norm": 0.7374144196510315, + "learning_rate": 0.0001, + "loss": 67.5064, + "step": 5540 + }, + { + "epoch": 8.9383785962614, + "grad_norm": 0.8073632121086121, + "learning_rate": 0.0001, + "loss": 67.0182, + "step": 5550 + }, + { + "epoch": 8.954501939839774, + "grad_norm": 0.7576195001602173, + "learning_rate": 0.0001, + "loss": 65.9139, + "step": 5560 + }, + { + "epoch": 8.97062528341815, + "grad_norm": 1.6479626893997192, + "learning_rate": 0.0001, + "loss": 66.3517, + "step": 5570 + }, + { + "epoch": 8.986748626996523, + "grad_norm": 0.846513569355011, + "learning_rate": 0.0001, + "loss": 64.2908, + "step": 5580 + }, + { + "epoch": 9.001612334357837, + "grad_norm": 0.9551037549972534, + "learning_rate": 0.0001, + "loss": 61.7312, + "step": 5590 + }, + { + "epoch": 9.017735677936212, + "grad_norm": 0.8250304460525513, + "learning_rate": 0.0001, + "loss": 66.3351, + "step": 5600 + }, + { + "epoch": 9.033859021514587, + "grad_norm": 0.7932960391044617, + "learning_rate": 0.0001, + "loss": 66.7753, + "step": 5610 + }, + { + "epoch": 9.04998236509296, + "grad_norm": 0.9839881658554077, + "learning_rate": 0.0001, + "loss": 64.5146, + "step": 5620 + }, + { + "epoch": 9.066105708671335, + "grad_norm": 0.7337850332260132, + "learning_rate": 0.0001, + "loss": 65.0544, + "step": 5630 + }, + { + "epoch": 9.08222905224971, + "grad_norm": 0.7496634721755981, + "learning_rate": 0.0001, + "loss": 64.1451, + "step": 5640 + }, + { + "epoch": 9.098352395828085, + "grad_norm": 0.9457297921180725, + "learning_rate": 0.0001, + "loss": 65.3036, + "step": 5650 + }, + { + "epoch": 9.11447573940646, + "grad_norm": 0.8315348625183105, + "learning_rate": 0.0001, + "loss": 66.0585, + "step": 5660 + }, + { + "epoch": 9.130599082984833, + "grad_norm": 0.9666195511817932, + "learning_rate": 0.0001, + "loss": 64.6802, + "step": 5670 + }, + { + "epoch": 9.146722426563208, + "grad_norm": 0.7416576147079468, + "learning_rate": 0.0001, + "loss": 65.1557, + "step": 5680 + }, + { + "epoch": 9.162845770141583, + "grad_norm": 0.7723799347877502, + "learning_rate": 0.0001, + "loss": 66.2871, + "step": 5690 + }, + { + "epoch": 9.178969113719958, + "grad_norm": 0.7461305856704712, + "learning_rate": 0.0001, + "loss": 65.5795, + "step": 5700 + }, + { + "epoch": 9.195092457298331, + "grad_norm": 0.6663411259651184, + "learning_rate": 0.0001, + "loss": 66.4094, + "step": 5710 + }, + { + "epoch": 9.211215800876706, + "grad_norm": 0.9340134859085083, + "learning_rate": 0.0001, + "loss": 63.5753, + "step": 5720 + }, + { + "epoch": 9.227339144455081, + "grad_norm": 0.7643963694572449, + "learning_rate": 0.0001, + "loss": 65.3359, + "step": 5730 + }, + { + "epoch": 9.243462488033456, + "grad_norm": 0.6931570768356323, + "learning_rate": 0.0001, + "loss": 65.0755, + "step": 5740 + }, + { + "epoch": 9.259585831611831, + "grad_norm": 0.9096988439559937, + "learning_rate": 0.0001, + "loss": 64.7983, + "step": 5750 + }, + { + "epoch": 9.275709175190205, + "grad_norm": 0.8360128402709961, + "learning_rate": 0.0001, + "loss": 64.3604, + "step": 5760 + }, + { + "epoch": 9.29183251876858, + "grad_norm": 0.9302591681480408, + "learning_rate": 0.0001, + "loss": 66.2754, + "step": 5770 + }, + { + "epoch": 9.307955862346954, + "grad_norm": 0.7000735998153687, + "learning_rate": 0.0001, + "loss": 65.2241, + "step": 5780 + }, + { + "epoch": 9.32407920592533, + "grad_norm": 0.7321175932884216, + "learning_rate": 0.0001, + "loss": 65.3558, + "step": 5790 + }, + { + "epoch": 9.340202549503703, + "grad_norm": 0.7948715090751648, + "learning_rate": 0.0001, + "loss": 64.2793, + "step": 5800 + }, + { + "epoch": 9.356325893082078, + "grad_norm": 0.7676129341125488, + "learning_rate": 0.0001, + "loss": 66.4778, + "step": 5810 + }, + { + "epoch": 9.372449236660453, + "grad_norm": 0.977236270904541, + "learning_rate": 0.0001, + "loss": 64.5587, + "step": 5820 + }, + { + "epoch": 9.388572580238828, + "grad_norm": 0.86808842420578, + "learning_rate": 0.0001, + "loss": 65.8026, + "step": 5830 + }, + { + "epoch": 9.4046959238172, + "grad_norm": 0.7455324530601501, + "learning_rate": 0.0001, + "loss": 65.066, + "step": 5840 + }, + { + "epoch": 9.420819267395576, + "grad_norm": 0.6784212589263916, + "learning_rate": 0.0001, + "loss": 65.4167, + "step": 5850 + }, + { + "epoch": 9.43694261097395, + "grad_norm": 1.033731460571289, + "learning_rate": 0.0001, + "loss": 66.4502, + "step": 5860 + }, + { + "epoch": 9.453065954552326, + "grad_norm": 0.7946107983589172, + "learning_rate": 0.0001, + "loss": 63.7567, + "step": 5870 + }, + { + "epoch": 9.4691892981307, + "grad_norm": 0.7826364636421204, + "learning_rate": 0.0001, + "loss": 65.4484, + "step": 5880 + }, + { + "epoch": 9.485312641709074, + "grad_norm": 0.7605016827583313, + "learning_rate": 0.0001, + "loss": 66.8459, + "step": 5890 + }, + { + "epoch": 9.501435985287449, + "grad_norm": 0.7383126020431519, + "learning_rate": 0.0001, + "loss": 64.6873, + "step": 5900 + }, + { + "epoch": 9.517559328865824, + "grad_norm": 0.8355686664581299, + "learning_rate": 0.0001, + "loss": 63.5145, + "step": 5910 + }, + { + "epoch": 9.533682672444199, + "grad_norm": 0.8827091455459595, + "learning_rate": 0.0001, + "loss": 67.2207, + "step": 5920 + }, + { + "epoch": 9.549806016022572, + "grad_norm": 0.6988142728805542, + "learning_rate": 0.0001, + "loss": 64.2289, + "step": 5930 + }, + { + "epoch": 9.565929359600947, + "grad_norm": 0.7609000205993652, + "learning_rate": 0.0001, + "loss": 63.9324, + "step": 5940 + }, + { + "epoch": 9.582052703179322, + "grad_norm": 0.8053853511810303, + "learning_rate": 0.0001, + "loss": 62.5564, + "step": 5950 + }, + { + "epoch": 9.598176046757697, + "grad_norm": 0.7684671878814697, + "learning_rate": 0.0001, + "loss": 66.2669, + "step": 5960 + }, + { + "epoch": 9.614299390336072, + "grad_norm": 0.8411065340042114, + "learning_rate": 0.0001, + "loss": 64.5087, + "step": 5970 + }, + { + "epoch": 9.630422733914445, + "grad_norm": 0.963498592376709, + "learning_rate": 0.0001, + "loss": 68.1736, + "step": 5980 + }, + { + "epoch": 9.64654607749282, + "grad_norm": 0.8715981841087341, + "learning_rate": 0.0001, + "loss": 64.1686, + "step": 5990 + }, + { + "epoch": 9.662669421071195, + "grad_norm": 0.8217800855636597, + "learning_rate": 0.0001, + "loss": 64.2211, + "step": 6000 + }, + { + "epoch": 9.67879276464957, + "grad_norm": 0.7824742197990417, + "learning_rate": 0.0001, + "loss": 65.9052, + "step": 6010 + }, + { + "epoch": 9.694916108227943, + "grad_norm": 0.7440009117126465, + "learning_rate": 0.0001, + "loss": 63.3749, + "step": 6020 + }, + { + "epoch": 9.711039451806318, + "grad_norm": 0.8759479522705078, + "learning_rate": 0.0001, + "loss": 63.4488, + "step": 6030 + }, + { + "epoch": 9.727162795384693, + "grad_norm": 3.662144184112549, + "learning_rate": 0.0001, + "loss": 62.8787, + "step": 6040 + }, + { + "epoch": 9.743286138963068, + "grad_norm": 1.007149577140808, + "learning_rate": 0.0001, + "loss": 66.0053, + "step": 6050 + }, + { + "epoch": 9.759409482541441, + "grad_norm": 0.8369242548942566, + "learning_rate": 0.0001, + "loss": 67.233, + "step": 6060 + }, + { + "epoch": 9.775532826119816, + "grad_norm": 0.8160434365272522, + "learning_rate": 0.0001, + "loss": 65.2089, + "step": 6070 + }, + { + "epoch": 9.791656169698191, + "grad_norm": 0.8482319712638855, + "learning_rate": 0.0001, + "loss": 65.0718, + "step": 6080 + }, + { + "epoch": 9.807779513276566, + "grad_norm": 0.8800129890441895, + "learning_rate": 0.0001, + "loss": 65.145, + "step": 6090 + }, + { + "epoch": 9.82390285685494, + "grad_norm": 0.8911405801773071, + "learning_rate": 0.0001, + "loss": 65.7952, + "step": 6100 + }, + { + "epoch": 9.840026200433314, + "grad_norm": 0.8108358979225159, + "learning_rate": 0.0001, + "loss": 65.1928, + "step": 6110 + }, + { + "epoch": 9.85614954401169, + "grad_norm": 0.8919743895530701, + "learning_rate": 0.0001, + "loss": 65.8129, + "step": 6120 + }, + { + "epoch": 9.872272887590064, + "grad_norm": 0.73152756690979, + "learning_rate": 0.0001, + "loss": 65.7141, + "step": 6130 + }, + { + "epoch": 9.88839623116844, + "grad_norm": 0.8583903312683105, + "learning_rate": 0.0001, + "loss": 64.8231, + "step": 6140 + }, + { + "epoch": 9.904519574746812, + "grad_norm": 0.798543393611908, + "learning_rate": 0.0001, + "loss": 64.5073, + "step": 6150 + }, + { + "epoch": 9.920642918325187, + "grad_norm": 0.8657358288764954, + "learning_rate": 0.0001, + "loss": 64.1207, + "step": 6160 + }, + { + "epoch": 9.936766261903562, + "grad_norm": 0.9324677586555481, + "learning_rate": 0.0001, + "loss": 64.7129, + "step": 6170 + }, + { + "epoch": 9.952889605481937, + "grad_norm": 0.7371154427528381, + "learning_rate": 0.0001, + "loss": 64.0251, + "step": 6180 + }, + { + "epoch": 9.96901294906031, + "grad_norm": 0.8273237347602844, + "learning_rate": 0.0001, + "loss": 65.8633, + "step": 6190 + }, + { + "epoch": 9.985136292638686, + "grad_norm": 0.7219365239143372, + "learning_rate": 0.0001, + "loss": 63.4029, + "step": 6200 + }, + { + "epoch": 10.0, + "grad_norm": 0.76242995262146, + "learning_rate": 0.0001, + "loss": 60.9994, + "step": 6210 + }, + { + "epoch": 10.016123343578375, + "grad_norm": 0.8677287697792053, + "learning_rate": 0.0001, + "loss": 65.9687, + "step": 6220 + }, + { + "epoch": 10.03224668715675, + "grad_norm": 0.7073521018028259, + "learning_rate": 0.0001, + "loss": 63.0083, + "step": 6230 + }, + { + "epoch": 10.048370030735123, + "grad_norm": 0.794535756111145, + "learning_rate": 0.0001, + "loss": 63.0616, + "step": 6240 + }, + { + "epoch": 10.064493374313498, + "grad_norm": 0.6974631547927856, + "learning_rate": 0.0001, + "loss": 64.6579, + "step": 6250 + }, + { + "epoch": 10.080616717891873, + "grad_norm": 0.8279150128364563, + "learning_rate": 0.0001, + "loss": 65.1069, + "step": 6260 + }, + { + "epoch": 10.096740061470248, + "grad_norm": 0.8682932257652283, + "learning_rate": 0.0001, + "loss": 64.5019, + "step": 6270 + }, + { + "epoch": 10.112863405048621, + "grad_norm": 1.0158190727233887, + "learning_rate": 0.0001, + "loss": 65.8854, + "step": 6280 + }, + { + "epoch": 10.128986748626996, + "grad_norm": 0.6911011934280396, + "learning_rate": 0.0001, + "loss": 62.4879, + "step": 6290 + }, + { + "epoch": 10.145110092205371, + "grad_norm": 0.7394022941589355, + "learning_rate": 0.0001, + "loss": 64.2604, + "step": 6300 + }, + { + "epoch": 10.161233435783746, + "grad_norm": 0.7941621541976929, + "learning_rate": 0.0001, + "loss": 64.5133, + "step": 6310 + }, + { + "epoch": 10.17735677936212, + "grad_norm": 1.0238994359970093, + "learning_rate": 0.0001, + "loss": 63.8594, + "step": 6320 + }, + { + "epoch": 10.193480122940494, + "grad_norm": 0.7332146763801575, + "learning_rate": 0.0001, + "loss": 66.5434, + "step": 6330 + }, + { + "epoch": 10.20960346651887, + "grad_norm": 0.9375078082084656, + "learning_rate": 0.0001, + "loss": 63.4793, + "step": 6340 + }, + { + "epoch": 10.225726810097244, + "grad_norm": 0.715931236743927, + "learning_rate": 0.0001, + "loss": 65.8812, + "step": 6350 + }, + { + "epoch": 10.24185015367562, + "grad_norm": 1.291536808013916, + "learning_rate": 0.0001, + "loss": 64.9125, + "step": 6360 + }, + { + "epoch": 10.257973497253992, + "grad_norm": 0.8942781686782837, + "learning_rate": 0.0001, + "loss": 67.1706, + "step": 6370 + }, + { + "epoch": 10.274096840832367, + "grad_norm": 0.8272124528884888, + "learning_rate": 0.0001, + "loss": 62.1721, + "step": 6380 + }, + { + "epoch": 10.290220184410742, + "grad_norm": 0.8191152215003967, + "learning_rate": 0.0001, + "loss": 64.8931, + "step": 6390 + }, + { + "epoch": 10.306343527989117, + "grad_norm": 1.7500241994857788, + "learning_rate": 0.0001, + "loss": 62.9693, + "step": 6400 + }, + { + "epoch": 10.32246687156749, + "grad_norm": 1.0159430503845215, + "learning_rate": 0.0001, + "loss": 63.0889, + "step": 6410 + }, + { + "epoch": 10.338590215145866, + "grad_norm": 0.7300348281860352, + "learning_rate": 0.0001, + "loss": 65.1742, + "step": 6420 + }, + { + "epoch": 10.35471355872424, + "grad_norm": 1.3132387399673462, + "learning_rate": 0.0001, + "loss": 64.8867, + "step": 6430 + }, + { + "epoch": 10.370836902302615, + "grad_norm": 0.7502451539039612, + "learning_rate": 0.0001, + "loss": 66.5094, + "step": 6440 + }, + { + "epoch": 10.38696024588099, + "grad_norm": 0.9408686757087708, + "learning_rate": 0.0001, + "loss": 65.1515, + "step": 6450 + }, + { + "epoch": 10.403083589459364, + "grad_norm": 0.769598126411438, + "learning_rate": 0.0001, + "loss": 65.1478, + "step": 6460 + }, + { + "epoch": 10.419206933037739, + "grad_norm": 0.9086162447929382, + "learning_rate": 0.0001, + "loss": 63.5263, + "step": 6470 + }, + { + "epoch": 10.435330276616114, + "grad_norm": 0.7899322509765625, + "learning_rate": 0.0001, + "loss": 63.3939, + "step": 6480 + }, + { + "epoch": 10.451453620194489, + "grad_norm": 0.9135434031486511, + "learning_rate": 0.0001, + "loss": 63.867, + "step": 6490 + }, + { + "epoch": 10.467576963772862, + "grad_norm": 0.9312737584114075, + "learning_rate": 0.0001, + "loss": 63.6939, + "step": 6500 + }, + { + "epoch": 10.483700307351237, + "grad_norm": 0.861764132976532, + "learning_rate": 0.0001, + "loss": 63.4523, + "step": 6510 + }, + { + "epoch": 10.499823650929612, + "grad_norm": 0.7056366801261902, + "learning_rate": 0.0001, + "loss": 63.2889, + "step": 6520 + }, + { + "epoch": 10.515946994507987, + "grad_norm": 0.7007710337638855, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 6530 + }, + { + "epoch": 10.53207033808636, + "grad_norm": 0.7091413736343384, + "learning_rate": 0.0001, + "loss": 63.4185, + "step": 6540 + }, + { + "epoch": 10.548193681664735, + "grad_norm": 0.7147234678268433, + "learning_rate": 0.0001, + "loss": 65.0726, + "step": 6550 + }, + { + "epoch": 10.56431702524311, + "grad_norm": 0.7954673767089844, + "learning_rate": 0.0001, + "loss": 65.0334, + "step": 6560 + }, + { + "epoch": 10.580440368821485, + "grad_norm": 0.725352942943573, + "learning_rate": 0.0001, + "loss": 62.6663, + "step": 6570 + }, + { + "epoch": 10.59656371239986, + "grad_norm": 0.9563508629798889, + "learning_rate": 0.0001, + "loss": 64.0705, + "step": 6580 + }, + { + "epoch": 10.612687055978233, + "grad_norm": 0.7778734564781189, + "learning_rate": 0.0001, + "loss": 63.7619, + "step": 6590 + }, + { + "epoch": 10.628810399556608, + "grad_norm": 0.8414369821548462, + "learning_rate": 0.0001, + "loss": 63.9238, + "step": 6600 + }, + { + "epoch": 10.644933743134983, + "grad_norm": 0.8070562481880188, + "learning_rate": 0.0001, + "loss": 63.9591, + "step": 6610 + }, + { + "epoch": 10.661057086713358, + "grad_norm": 0.6493734121322632, + "learning_rate": 0.0001, + "loss": 63.7237, + "step": 6620 + }, + { + "epoch": 10.677180430291731, + "grad_norm": 0.8263372778892517, + "learning_rate": 0.0001, + "loss": 64.2262, + "step": 6630 + }, + { + "epoch": 10.693303773870106, + "grad_norm": 1.0090227127075195, + "learning_rate": 0.0001, + "loss": 63.5469, + "step": 6640 + }, + { + "epoch": 10.709427117448481, + "grad_norm": 0.998319685459137, + "learning_rate": 0.0001, + "loss": 62.9113, + "step": 6650 + }, + { + "epoch": 10.725550461026856, + "grad_norm": 1.006489634513855, + "learning_rate": 0.0001, + "loss": 66.1458, + "step": 6660 + }, + { + "epoch": 10.741673804605231, + "grad_norm": 0.9575757384300232, + "learning_rate": 0.0001, + "loss": 63.0913, + "step": 6670 + }, + { + "epoch": 10.757797148183604, + "grad_norm": 0.6832238435745239, + "learning_rate": 0.0001, + "loss": 66.7501, + "step": 6680 + }, + { + "epoch": 10.773920491761979, + "grad_norm": 0.8075717687606812, + "learning_rate": 0.0001, + "loss": 64.2839, + "step": 6690 + }, + { + "epoch": 10.790043835340354, + "grad_norm": 0.9402624368667603, + "learning_rate": 0.0001, + "loss": 64.9296, + "step": 6700 + }, + { + "epoch": 10.806167178918729, + "grad_norm": 0.7681158185005188, + "learning_rate": 0.0001, + "loss": 63.5686, + "step": 6710 + }, + { + "epoch": 10.822290522497102, + "grad_norm": 0.8451285362243652, + "learning_rate": 0.0001, + "loss": 64.5404, + "step": 6720 + }, + { + "epoch": 10.838413866075477, + "grad_norm": 0.7278769016265869, + "learning_rate": 0.0001, + "loss": 64.1341, + "step": 6730 + }, + { + "epoch": 10.854537209653852, + "grad_norm": 0.8524075746536255, + "learning_rate": 0.0001, + "loss": 64.0483, + "step": 6740 + }, + { + "epoch": 10.870660553232227, + "grad_norm": 0.703598141670227, + "learning_rate": 0.0001, + "loss": 64.0072, + "step": 6750 + }, + { + "epoch": 10.8867838968106, + "grad_norm": 0.7740384936332703, + "learning_rate": 0.0001, + "loss": 63.4171, + "step": 6760 + }, + { + "epoch": 10.902907240388975, + "grad_norm": 0.8302356004714966, + "learning_rate": 0.0001, + "loss": 65.5159, + "step": 6770 + }, + { + "epoch": 10.91903058396735, + "grad_norm": 1.0289733409881592, + "learning_rate": 0.0001, + "loss": 64.9785, + "step": 6780 + }, + { + "epoch": 10.935153927545725, + "grad_norm": 0.6721987128257751, + "learning_rate": 0.0001, + "loss": 63.965, + "step": 6790 + }, + { + "epoch": 10.951277271124098, + "grad_norm": 0.7614948749542236, + "learning_rate": 0.0001, + "loss": 64.1375, + "step": 6800 + }, + { + "epoch": 10.967400614702473, + "grad_norm": 0.7342216372489929, + "learning_rate": 0.0001, + "loss": 64.4342, + "step": 6810 + }, + { + "epoch": 10.983523958280848, + "grad_norm": 0.8808079957962036, + "learning_rate": 0.0001, + "loss": 65.2864, + "step": 6820 + }, + { + "epoch": 10.999647301859223, + "grad_norm": 0.818477988243103, + "learning_rate": 0.0001, + "loss": 63.9931, + "step": 6830 + }, + { + "epoch": 11.014511009220538, + "grad_norm": 0.7867474555969238, + "learning_rate": 0.0001, + "loss": 59.2426, + "step": 6840 + }, + { + "epoch": 11.030634352798911, + "grad_norm": 0.6163678765296936, + "learning_rate": 0.0001, + "loss": 65.0149, + "step": 6850 + }, + { + "epoch": 11.046757696377286, + "grad_norm": 0.8334609866142273, + "learning_rate": 0.0001, + "loss": 63.5017, + "step": 6860 + }, + { + "epoch": 11.062881039955661, + "grad_norm": 0.7335264086723328, + "learning_rate": 0.0001, + "loss": 64.6307, + "step": 6870 + }, + { + "epoch": 11.079004383534036, + "grad_norm": 0.7723846435546875, + "learning_rate": 0.0001, + "loss": 64.316, + "step": 6880 + }, + { + "epoch": 11.09512772711241, + "grad_norm": 0.7972145080566406, + "learning_rate": 0.0001, + "loss": 63.1469, + "step": 6890 + }, + { + "epoch": 11.111251070690784, + "grad_norm": 0.7696133852005005, + "learning_rate": 0.0001, + "loss": 64.0338, + "step": 6900 + }, + { + "epoch": 11.127374414269159, + "grad_norm": 0.8972133994102478, + "learning_rate": 0.0001, + "loss": 65.5727, + "step": 6910 + }, + { + "epoch": 11.143497757847534, + "grad_norm": 0.6776295900344849, + "learning_rate": 0.0001, + "loss": 66.176, + "step": 6920 + }, + { + "epoch": 11.159621101425909, + "grad_norm": 0.7345874905586243, + "learning_rate": 0.0001, + "loss": 64.0949, + "step": 6930 + }, + { + "epoch": 11.175744445004282, + "grad_norm": 0.7988374829292297, + "learning_rate": 0.0001, + "loss": 64.7721, + "step": 6940 + }, + { + "epoch": 11.191867788582657, + "grad_norm": 0.7552387714385986, + "learning_rate": 0.0001, + "loss": 63.1053, + "step": 6950 + }, + { + "epoch": 11.207991132161032, + "grad_norm": 0.6388214230537415, + "learning_rate": 0.0001, + "loss": 63.7871, + "step": 6960 + }, + { + "epoch": 11.224114475739407, + "grad_norm": 0.6837960481643677, + "learning_rate": 0.0001, + "loss": 63.906, + "step": 6970 + }, + { + "epoch": 11.24023781931778, + "grad_norm": 0.7050679326057434, + "learning_rate": 0.0001, + "loss": 64.2803, + "step": 6980 + }, + { + "epoch": 11.256361162896155, + "grad_norm": 0.8619478940963745, + "learning_rate": 0.0001, + "loss": 62.8786, + "step": 6990 + }, + { + "epoch": 11.27248450647453, + "grad_norm": 0.9759813547134399, + "learning_rate": 0.0001, + "loss": 65.2299, + "step": 7000 + }, + { + "epoch": 11.288607850052905, + "grad_norm": 0.7584514021873474, + "learning_rate": 0.0001, + "loss": 61.4178, + "step": 7010 + }, + { + "epoch": 11.304731193631278, + "grad_norm": 0.8467874526977539, + "learning_rate": 0.0001, + "loss": 65.6229, + "step": 7020 + }, + { + "epoch": 11.320854537209653, + "grad_norm": 0.7121654152870178, + "learning_rate": 0.0001, + "loss": 63.9061, + "step": 7030 + }, + { + "epoch": 11.336977880788028, + "grad_norm": 0.8885557055473328, + "learning_rate": 0.0001, + "loss": 65.1436, + "step": 7040 + }, + { + "epoch": 11.353101224366403, + "grad_norm": 0.6320177912712097, + "learning_rate": 0.0001, + "loss": 62.8535, + "step": 7050 + }, + { + "epoch": 11.369224567944778, + "grad_norm": 0.9287629723548889, + "learning_rate": 0.0001, + "loss": 62.7407, + "step": 7060 + }, + { + "epoch": 11.385347911523152, + "grad_norm": 0.7705608606338501, + "learning_rate": 0.0001, + "loss": 63.469, + "step": 7070 + }, + { + "epoch": 11.401471255101526, + "grad_norm": 0.6687246561050415, + "learning_rate": 0.0001, + "loss": 62.722, + "step": 7080 + }, + { + "epoch": 11.417594598679901, + "grad_norm": 0.7774648666381836, + "learning_rate": 0.0001, + "loss": 63.1145, + "step": 7090 + }, + { + "epoch": 11.433717942258276, + "grad_norm": 0.7808363437652588, + "learning_rate": 0.0001, + "loss": 63.9342, + "step": 7100 + }, + { + "epoch": 11.44984128583665, + "grad_norm": 0.8033562302589417, + "learning_rate": 0.0001, + "loss": 65.7876, + "step": 7110 + }, + { + "epoch": 11.465964629415025, + "grad_norm": 0.8143486976623535, + "learning_rate": 0.0001, + "loss": 64.8782, + "step": 7120 + }, + { + "epoch": 11.4820879729934, + "grad_norm": 0.7855561971664429, + "learning_rate": 0.0001, + "loss": 63.4944, + "step": 7130 + }, + { + "epoch": 11.498211316571775, + "grad_norm": 0.7525351047515869, + "learning_rate": 0.0001, + "loss": 62.4088, + "step": 7140 + }, + { + "epoch": 11.514334660150148, + "grad_norm": 0.7193494439125061, + "learning_rate": 0.0001, + "loss": 64.2386, + "step": 7150 + }, + { + "epoch": 11.530458003728523, + "grad_norm": 0.892281174659729, + "learning_rate": 0.0001, + "loss": 63.8075, + "step": 7160 + }, + { + "epoch": 11.546581347306898, + "grad_norm": 0.720287561416626, + "learning_rate": 0.0001, + "loss": 63.3259, + "step": 7170 + }, + { + "epoch": 11.562704690885273, + "grad_norm": 1.4404689073562622, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 7180 + }, + { + "epoch": 11.578828034463648, + "grad_norm": 0.9082726240158081, + "learning_rate": 0.0001, + "loss": 64.6877, + "step": 7190 + }, + { + "epoch": 11.59495137804202, + "grad_norm": 0.7137670516967773, + "learning_rate": 0.0001, + "loss": 63.4832, + "step": 7200 + }, + { + "epoch": 11.611074721620396, + "grad_norm": 0.6848503947257996, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 7210 + }, + { + "epoch": 11.62719806519877, + "grad_norm": 0.7483687996864319, + "learning_rate": 0.0001, + "loss": 63.294, + "step": 7220 + }, + { + "epoch": 11.643321408777146, + "grad_norm": 0.935321569442749, + "learning_rate": 0.0001, + "loss": 62.8669, + "step": 7230 + }, + { + "epoch": 11.659444752355519, + "grad_norm": 0.8359326124191284, + "learning_rate": 0.0001, + "loss": 64.4482, + "step": 7240 + }, + { + "epoch": 11.675568095933894, + "grad_norm": 0.7838515043258667, + "learning_rate": 0.0001, + "loss": 64.1975, + "step": 7250 + }, + { + "epoch": 11.691691439512269, + "grad_norm": 0.9128443598747253, + "learning_rate": 0.0001, + "loss": 63.1822, + "step": 7260 + }, + { + "epoch": 11.707814783090644, + "grad_norm": 1.0545419454574585, + "learning_rate": 0.0001, + "loss": 63.3112, + "step": 7270 + }, + { + "epoch": 11.723938126669019, + "grad_norm": 0.8572486639022827, + "learning_rate": 0.0001, + "loss": 64.9605, + "step": 7280 + }, + { + "epoch": 11.740061470247392, + "grad_norm": 1.0894116163253784, + "learning_rate": 0.0001, + "loss": 64.3671, + "step": 7290 + }, + { + "epoch": 11.756184813825767, + "grad_norm": 0.7500355839729309, + "learning_rate": 0.0001, + "loss": 62.6381, + "step": 7300 + }, + { + "epoch": 11.772308157404142, + "grad_norm": 0.7315185070037842, + "learning_rate": 0.0001, + "loss": 66.1438, + "step": 7310 + }, + { + "epoch": 11.788431500982517, + "grad_norm": 0.8123629093170166, + "learning_rate": 0.0001, + "loss": 62.6012, + "step": 7320 + }, + { + "epoch": 11.80455484456089, + "grad_norm": 0.8072763085365295, + "learning_rate": 0.0001, + "loss": 63.4424, + "step": 7330 + }, + { + "epoch": 11.820678188139265, + "grad_norm": 1.1077874898910522, + "learning_rate": 0.0001, + "loss": 63.2812, + "step": 7340 + }, + { + "epoch": 11.83680153171764, + "grad_norm": 0.7116726636886597, + "learning_rate": 0.0001, + "loss": 63.6077, + "step": 7350 + }, + { + "epoch": 11.852924875296015, + "grad_norm": 0.8817790746688843, + "learning_rate": 0.0001, + "loss": 63.1938, + "step": 7360 + }, + { + "epoch": 11.86904821887439, + "grad_norm": 1.021730661392212, + "learning_rate": 0.0001, + "loss": 65.2215, + "step": 7370 + }, + { + "epoch": 11.885171562452763, + "grad_norm": 0.9871250987052917, + "learning_rate": 0.0001, + "loss": 64.6468, + "step": 7380 + }, + { + "epoch": 11.901294906031138, + "grad_norm": 1.3591430187225342, + "learning_rate": 0.0001, + "loss": 61.5546, + "step": 7390 + }, + { + "epoch": 11.917418249609513, + "grad_norm": 0.8114227056503296, + "learning_rate": 0.0001, + "loss": 61.8215, + "step": 7400 + }, + { + "epoch": 11.933541593187888, + "grad_norm": 0.7151288986206055, + "learning_rate": 0.0001, + "loss": 61.4856, + "step": 7410 + }, + { + "epoch": 11.949664936766261, + "grad_norm": 0.7494287490844727, + "learning_rate": 0.0001, + "loss": 63.6451, + "step": 7420 + }, + { + "epoch": 11.965788280344636, + "grad_norm": 0.7377745509147644, + "learning_rate": 0.0001, + "loss": 63.002, + "step": 7430 + }, + { + "epoch": 11.981911623923011, + "grad_norm": 0.6534659266471863, + "learning_rate": 0.0001, + "loss": 63.6404, + "step": 7440 + }, + { + "epoch": 11.998034967501386, + "grad_norm": 0.7548736333847046, + "learning_rate": 0.0001, + "loss": 64.1986, + "step": 7450 + }, + { + "epoch": 12.012898674862699, + "grad_norm": 0.7227553129196167, + "learning_rate": 0.0001, + "loss": 56.8327, + "step": 7460 + }, + { + "epoch": 12.029022018441074, + "grad_norm": 0.6348695158958435, + "learning_rate": 0.0001, + "loss": 62.4472, + "step": 7470 + }, + { + "epoch": 12.045145362019449, + "grad_norm": 0.8249921202659607, + "learning_rate": 0.0001, + "loss": 65.1877, + "step": 7480 + }, + { + "epoch": 12.061268705597824, + "grad_norm": 0.7152777314186096, + "learning_rate": 0.0001, + "loss": 64.2609, + "step": 7490 + }, + { + "epoch": 12.077392049176197, + "grad_norm": 0.7399184703826904, + "learning_rate": 0.0001, + "loss": 63.3997, + "step": 7500 + }, + { + "epoch": 12.093515392754572, + "grad_norm": 0.6972354650497437, + "learning_rate": 0.0001, + "loss": 61.3573, + "step": 7510 + }, + { + "epoch": 12.109638736332947, + "grad_norm": 0.7666720151901245, + "learning_rate": 0.0001, + "loss": 63.8438, + "step": 7520 + }, + { + "epoch": 12.125762079911322, + "grad_norm": 0.7309302687644958, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 7530 + }, + { + "epoch": 12.141885423489697, + "grad_norm": 1.3805177211761475, + "learning_rate": 0.0001, + "loss": 64.68, + "step": 7540 + }, + { + "epoch": 12.15800876706807, + "grad_norm": 1.0315922498703003, + "learning_rate": 0.0001, + "loss": 63.6372, + "step": 7550 + }, + { + "epoch": 12.174132110646445, + "grad_norm": 1.2012155055999756, + "learning_rate": 0.0001, + "loss": 62.5765, + "step": 7560 + }, + { + "epoch": 12.19025545422482, + "grad_norm": 0.7769846320152283, + "learning_rate": 0.0001, + "loss": 63.9737, + "step": 7570 + }, + { + "epoch": 12.206378797803195, + "grad_norm": 0.6587379574775696, + "learning_rate": 0.0001, + "loss": 63.382, + "step": 7580 + }, + { + "epoch": 12.222502141381568, + "grad_norm": 0.8309435248374939, + "learning_rate": 0.0001, + "loss": 64.4027, + "step": 7590 + }, + { + "epoch": 12.238625484959943, + "grad_norm": 0.8221393823623657, + "learning_rate": 0.0001, + "loss": 62.8561, + "step": 7600 + }, + { + "epoch": 12.254748828538318, + "grad_norm": 0.8629311323165894, + "learning_rate": 0.0001, + "loss": 64.3673, + "step": 7610 + }, + { + "epoch": 12.270872172116693, + "grad_norm": 0.698113739490509, + "learning_rate": 0.0001, + "loss": 63.6153, + "step": 7620 + }, + { + "epoch": 12.286995515695068, + "grad_norm": 0.72906094789505, + "learning_rate": 0.0001, + "loss": 64.8439, + "step": 7630 + }, + { + "epoch": 12.303118859273441, + "grad_norm": 0.7402384877204895, + "learning_rate": 0.0001, + "loss": 64.2103, + "step": 7640 + }, + { + "epoch": 12.319242202851816, + "grad_norm": 0.9972009062767029, + "learning_rate": 0.0001, + "loss": 63.803, + "step": 7650 + }, + { + "epoch": 12.335365546430191, + "grad_norm": 0.7852274775505066, + "learning_rate": 0.0001, + "loss": 64.8144, + "step": 7660 + }, + { + "epoch": 12.351488890008566, + "grad_norm": 0.7433444857597351, + "learning_rate": 0.0001, + "loss": 63.0123, + "step": 7670 + }, + { + "epoch": 12.36761223358694, + "grad_norm": 0.8548759818077087, + "learning_rate": 0.0001, + "loss": 63.2215, + "step": 7680 + }, + { + "epoch": 12.383735577165314, + "grad_norm": 0.6759565472602844, + "learning_rate": 0.0001, + "loss": 62.1246, + "step": 7690 + }, + { + "epoch": 12.39985892074369, + "grad_norm": 0.9079536199569702, + "learning_rate": 0.0001, + "loss": 64.1692, + "step": 7700 + }, + { + "epoch": 12.415982264322064, + "grad_norm": 0.6890071034431458, + "learning_rate": 0.0001, + "loss": 62.5602, + "step": 7710 + }, + { + "epoch": 12.432105607900438, + "grad_norm": 0.8096348643302917, + "learning_rate": 0.0001, + "loss": 61.0103, + "step": 7720 + }, + { + "epoch": 12.448228951478812, + "grad_norm": 0.7359873056411743, + "learning_rate": 0.0001, + "loss": 62.1914, + "step": 7730 + }, + { + "epoch": 12.464352295057187, + "grad_norm": 1.0701091289520264, + "learning_rate": 0.0001, + "loss": 61.7295, + "step": 7740 + }, + { + "epoch": 12.480475638635562, + "grad_norm": 0.791928768157959, + "learning_rate": 0.0001, + "loss": 62.6812, + "step": 7750 + }, + { + "epoch": 12.496598982213937, + "grad_norm": 0.7719446420669556, + "learning_rate": 0.0001, + "loss": 61.9114, + "step": 7760 + }, + { + "epoch": 12.51272232579231, + "grad_norm": 0.8720338344573975, + "learning_rate": 0.0001, + "loss": 64.8107, + "step": 7770 + }, + { + "epoch": 12.528845669370686, + "grad_norm": 0.9999920129776001, + "learning_rate": 0.0001, + "loss": 61.198, + "step": 7780 + }, + { + "epoch": 12.54496901294906, + "grad_norm": 0.710754930973053, + "learning_rate": 0.0001, + "loss": 64.1462, + "step": 7790 + }, + { + "epoch": 12.561092356527435, + "grad_norm": 0.7401363849639893, + "learning_rate": 0.0001, + "loss": 61.8988, + "step": 7800 + }, + { + "epoch": 12.577215700105809, + "grad_norm": 0.9183729290962219, + "learning_rate": 0.0001, + "loss": 62.6392, + "step": 7810 + }, + { + "epoch": 12.593339043684184, + "grad_norm": 0.7341399788856506, + "learning_rate": 0.0001, + "loss": 62.7068, + "step": 7820 + }, + { + "epoch": 12.609462387262559, + "grad_norm": 0.8066722750663757, + "learning_rate": 0.0001, + "loss": 64.0625, + "step": 7830 + }, + { + "epoch": 12.625585730840934, + "grad_norm": 0.6088835597038269, + "learning_rate": 0.0001, + "loss": 64.3208, + "step": 7840 + }, + { + "epoch": 12.641709074419307, + "grad_norm": 0.6969302892684937, + "learning_rate": 0.0001, + "loss": 62.5817, + "step": 7850 + }, + { + "epoch": 12.657832417997682, + "grad_norm": 0.82635098695755, + "learning_rate": 0.0001, + "loss": 62.0061, + "step": 7860 + }, + { + "epoch": 12.673955761576057, + "grad_norm": 0.7016922235488892, + "learning_rate": 0.0001, + "loss": 62.5166, + "step": 7870 + }, + { + "epoch": 12.690079105154432, + "grad_norm": 0.8035777807235718, + "learning_rate": 0.0001, + "loss": 63.6543, + "step": 7880 + }, + { + "epoch": 12.706202448732807, + "grad_norm": 0.8573809862136841, + "learning_rate": 0.0001, + "loss": 62.5258, + "step": 7890 + }, + { + "epoch": 12.72232579231118, + "grad_norm": 0.9497472643852234, + "learning_rate": 0.0001, + "loss": 63.1133, + "step": 7900 + }, + { + "epoch": 12.738449135889555, + "grad_norm": 0.7494457960128784, + "learning_rate": 0.0001, + "loss": 63.8157, + "step": 7910 + }, + { + "epoch": 12.75457247946793, + "grad_norm": 0.7657570242881775, + "learning_rate": 0.0001, + "loss": 62.6255, + "step": 7920 + }, + { + "epoch": 12.770695823046305, + "grad_norm": 0.7848380208015442, + "learning_rate": 0.0001, + "loss": 62.9095, + "step": 7930 + }, + { + "epoch": 12.786819166624678, + "grad_norm": 0.8386877775192261, + "learning_rate": 0.0001, + "loss": 63.2254, + "step": 7940 + }, + { + "epoch": 12.802942510203053, + "grad_norm": 0.6774776577949524, + "learning_rate": 0.0001, + "loss": 64.3929, + "step": 7950 + }, + { + "epoch": 12.819065853781428, + "grad_norm": 0.798547625541687, + "learning_rate": 0.0001, + "loss": 62.9884, + "step": 7960 + }, + { + "epoch": 12.835189197359803, + "grad_norm": 0.9340888261795044, + "learning_rate": 0.0001, + "loss": 62.6197, + "step": 7970 + }, + { + "epoch": 12.851312540938178, + "grad_norm": 0.8211432695388794, + "learning_rate": 0.0001, + "loss": 62.7756, + "step": 7980 + }, + { + "epoch": 12.867435884516551, + "grad_norm": 0.7612001299858093, + "learning_rate": 0.0001, + "loss": 62.0792, + "step": 7990 + }, + { + "epoch": 12.883559228094926, + "grad_norm": 0.7743386626243591, + "learning_rate": 0.0001, + "loss": 63.9203, + "step": 8000 + }, + { + "epoch": 12.899682571673301, + "grad_norm": 0.7419672608375549, + "learning_rate": 0.0001, + "loss": 62.1705, + "step": 8010 + }, + { + "epoch": 12.915805915251676, + "grad_norm": 0.6556256413459778, + "learning_rate": 0.0001, + "loss": 64.5863, + "step": 8020 + }, + { + "epoch": 12.93192925883005, + "grad_norm": 1.136175274848938, + "learning_rate": 0.0001, + "loss": 63.2619, + "step": 8030 + }, + { + "epoch": 12.948052602408424, + "grad_norm": 0.8631499409675598, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8040 + }, + { + "epoch": 12.9641759459868, + "grad_norm": 0.8386172652244568, + "learning_rate": 0.0001, + "loss": 63.0598, + "step": 8050 + }, + { + "epoch": 12.980299289565174, + "grad_norm": 0.8078306913375854, + "learning_rate": 0.0001, + "loss": 62.6877, + "step": 8060 + }, + { + "epoch": 12.996422633143549, + "grad_norm": 0.8486189842224121, + "learning_rate": 0.0001, + "loss": 62.8643, + "step": 8070 + }, + { + "epoch": 13.011286340504862, + "grad_norm": 0.8717893958091736, + "learning_rate": 0.0001, + "loss": 57.6947, + "step": 8080 + }, + { + "epoch": 13.027409684083237, + "grad_norm": 1.2023789882659912, + "learning_rate": 0.0001, + "loss": 60.4728, + "step": 8090 + }, + { + "epoch": 13.043533027661612, + "grad_norm": 0.6879557967185974, + "learning_rate": 0.0001, + "loss": 61.7643, + "step": 8100 + }, + { + "epoch": 13.059656371239987, + "grad_norm": 1.0918916463851929, + "learning_rate": 0.0001, + "loss": 65.3682, + "step": 8110 + }, + { + "epoch": 13.07577971481836, + "grad_norm": 0.85753333568573, + "learning_rate": 0.0001, + "loss": 63.0815, + "step": 8120 + }, + { + "epoch": 13.091903058396735, + "grad_norm": 0.6568583846092224, + "learning_rate": 0.0001, + "loss": 64.7167, + "step": 8130 + }, + { + "epoch": 13.10802640197511, + "grad_norm": 0.6178144812583923, + "learning_rate": 0.0001, + "loss": 62.4294, + "step": 8140 + }, + { + "epoch": 13.124149745553485, + "grad_norm": 0.7930784821510315, + "learning_rate": 0.0001, + "loss": 64.542, + "step": 8150 + }, + { + "epoch": 13.140273089131858, + "grad_norm": 0.7675037980079651, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8160 + }, + { + "epoch": 13.156396432710233, + "grad_norm": 0.786997377872467, + "learning_rate": 0.0001, + "loss": 64.1495, + "step": 8170 + }, + { + "epoch": 13.172519776288608, + "grad_norm": 0.7752560377120972, + "learning_rate": 0.0001, + "loss": 61.6483, + "step": 8180 + }, + { + "epoch": 13.188643119866983, + "grad_norm": 0.8455203771591187, + "learning_rate": 0.0001, + "loss": 63.3551, + "step": 8190 + }, + { + "epoch": 13.204766463445356, + "grad_norm": 0.7223097681999207, + "learning_rate": 0.0001, + "loss": 63.1473, + "step": 8200 + }, + { + "epoch": 13.220889807023731, + "grad_norm": 0.7688442468643188, + "learning_rate": 0.0001, + "loss": 62.6723, + "step": 8210 + }, + { + "epoch": 13.237013150602106, + "grad_norm": 0.7730395793914795, + "learning_rate": 0.0001, + "loss": 64.3424, + "step": 8220 + }, + { + "epoch": 13.253136494180481, + "grad_norm": 0.8479727506637573, + "learning_rate": 0.0001, + "loss": 61.4336, + "step": 8230 + }, + { + "epoch": 13.269259837758856, + "grad_norm": 0.9070560932159424, + "learning_rate": 0.0001, + "loss": 63.1221, + "step": 8240 + }, + { + "epoch": 13.28538318133723, + "grad_norm": 0.8924600481987, + "learning_rate": 0.0001, + "loss": 63.9172, + "step": 8250 + }, + { + "epoch": 13.301506524915604, + "grad_norm": 1.0688443183898926, + "learning_rate": 0.0001, + "loss": 63.7944, + "step": 8260 + }, + { + "epoch": 13.31762986849398, + "grad_norm": 0.6866253018379211, + "learning_rate": 0.0001, + "loss": 65.5847, + "step": 8270 + }, + { + "epoch": 13.333753212072354, + "grad_norm": 0.711171567440033, + "learning_rate": 0.0001, + "loss": 62.698, + "step": 8280 + }, + { + "epoch": 13.349876555650727, + "grad_norm": 0.8128448724746704, + "learning_rate": 0.0001, + "loss": 61.5992, + "step": 8290 + }, + { + "epoch": 13.365999899229102, + "grad_norm": 0.7483755350112915, + "learning_rate": 0.0001, + "loss": 63.782, + "step": 8300 + }, + { + "epoch": 13.382123242807477, + "grad_norm": 0.7872599363327026, + "learning_rate": 0.0001, + "loss": 65.4922, + "step": 8310 + }, + { + "epoch": 13.398246586385852, + "grad_norm": 0.6861543655395508, + "learning_rate": 0.0001, + "loss": 62.6946, + "step": 8320 + }, + { + "epoch": 13.414369929964227, + "grad_norm": 0.7192599773406982, + "learning_rate": 0.0001, + "loss": 62.894, + "step": 8330 + }, + { + "epoch": 13.4304932735426, + "grad_norm": 0.7665286064147949, + "learning_rate": 0.0001, + "loss": 62.6253, + "step": 8340 + }, + { + "epoch": 13.446616617120975, + "grad_norm": 0.7909309267997742, + "learning_rate": 0.0001, + "loss": 62.5355, + "step": 8350 + }, + { + "epoch": 13.46273996069935, + "grad_norm": 0.7339541912078857, + "learning_rate": 0.0001, + "loss": 62.8514, + "step": 8360 + }, + { + "epoch": 13.478863304277725, + "grad_norm": 0.9893004298210144, + "learning_rate": 0.0001, + "loss": 63.7751, + "step": 8370 + }, + { + "epoch": 13.494986647856098, + "grad_norm": 0.9109547734260559, + "learning_rate": 0.0001, + "loss": 62.6697, + "step": 8380 + }, + { + "epoch": 13.511109991434473, + "grad_norm": 0.6807034015655518, + "learning_rate": 0.0001, + "loss": 62.1418, + "step": 8390 + }, + { + "epoch": 13.527233335012848, + "grad_norm": 0.915533185005188, + "learning_rate": 0.0001, + "loss": 61.7645, + "step": 8400 + }, + { + "epoch": 13.543356678591223, + "grad_norm": 0.8662852048873901, + "learning_rate": 0.0001, + "loss": 62.5546, + "step": 8410 + }, + { + "epoch": 13.559480022169598, + "grad_norm": 0.8678365349769592, + "learning_rate": 0.0001, + "loss": 63.8795, + "step": 8420 + }, + { + "epoch": 13.575603365747972, + "grad_norm": 0.6808590888977051, + "learning_rate": 0.0001, + "loss": 64.158, + "step": 8430 + }, + { + "epoch": 13.591726709326347, + "grad_norm": 0.7896480560302734, + "learning_rate": 0.0001, + "loss": 62.1052, + "step": 8440 + }, + { + "epoch": 13.607850052904721, + "grad_norm": 0.8973979949951172, + "learning_rate": 0.0001, + "loss": 64.478, + "step": 8450 + }, + { + "epoch": 13.623973396483096, + "grad_norm": 0.8228474259376526, + "learning_rate": 0.0001, + "loss": 61.2416, + "step": 8460 + }, + { + "epoch": 13.64009674006147, + "grad_norm": 0.9414492845535278, + "learning_rate": 0.0001, + "loss": 62.816, + "step": 8470 + }, + { + "epoch": 13.656220083639845, + "grad_norm": 0.9369436502456665, + "learning_rate": 0.0001, + "loss": 63.4951, + "step": 8480 + }, + { + "epoch": 13.67234342721822, + "grad_norm": 0.7152880430221558, + "learning_rate": 0.0001, + "loss": 62.4737, + "step": 8490 + }, + { + "epoch": 13.688466770796595, + "grad_norm": 0.7364594340324402, + "learning_rate": 0.0001, + "loss": 63.8688, + "step": 8500 + }, + { + "epoch": 13.704590114374968, + "grad_norm": 0.8499143123626709, + "learning_rate": 0.0001, + "loss": 64.5654, + "step": 8510 + }, + { + "epoch": 13.720713457953343, + "grad_norm": 0.7699156999588013, + "learning_rate": 0.0001, + "loss": 63.3726, + "step": 8520 + }, + { + "epoch": 13.736836801531718, + "grad_norm": 0.9146804809570312, + "learning_rate": 0.0001, + "loss": 63.6565, + "step": 8530 + }, + { + "epoch": 13.752960145110093, + "grad_norm": 0.773318350315094, + "learning_rate": 0.0001, + "loss": 65.0719, + "step": 8540 + }, + { + "epoch": 13.769083488688466, + "grad_norm": 0.6158450245857239, + "learning_rate": 0.0001, + "loss": 60.7291, + "step": 8550 + }, + { + "epoch": 13.78520683226684, + "grad_norm": 0.8053523302078247, + "learning_rate": 0.0001, + "loss": 62.5622, + "step": 8560 + }, + { + "epoch": 13.801330175845216, + "grad_norm": 0.8983457684516907, + "learning_rate": 0.0001, + "loss": 64.2921, + "step": 8570 + }, + { + "epoch": 13.81745351942359, + "grad_norm": 0.8214695453643799, + "learning_rate": 0.0001, + "loss": 62.2649, + "step": 8580 + }, + { + "epoch": 13.833576863001966, + "grad_norm": 0.8079172372817993, + "learning_rate": 0.0001, + "loss": 64.0828, + "step": 8590 + }, + { + "epoch": 13.849700206580339, + "grad_norm": 1.126568078994751, + "learning_rate": 0.0001, + "loss": 62.3927, + "step": 8600 + }, + { + "epoch": 13.865823550158714, + "grad_norm": 0.7597793936729431, + "learning_rate": 0.0001, + "loss": 62.5922, + "step": 8610 + }, + { + "epoch": 13.881946893737089, + "grad_norm": 0.671531617641449, + "learning_rate": 0.0001, + "loss": 61.9885, + "step": 8620 + }, + { + "epoch": 13.898070237315464, + "grad_norm": 0.823423445224762, + "learning_rate": 0.0001, + "loss": 62.0632, + "step": 8630 + }, + { + "epoch": 13.914193580893837, + "grad_norm": 0.7694658637046814, + "learning_rate": 0.0001, + "loss": 64.5485, + "step": 8640 + }, + { + "epoch": 13.930316924472212, + "grad_norm": 0.7394612431526184, + "learning_rate": 0.0001, + "loss": 61.9626, + "step": 8650 + }, + { + "epoch": 13.946440268050587, + "grad_norm": 0.724054217338562, + "learning_rate": 0.0001, + "loss": 62.3761, + "step": 8660 + }, + { + "epoch": 13.962563611628962, + "grad_norm": 0.7160128951072693, + "learning_rate": 0.0001, + "loss": 60.5142, + "step": 8670 + }, + { + "epoch": 13.978686955207337, + "grad_norm": 1.0045746564865112, + "learning_rate": 0.0001, + "loss": 62.7059, + "step": 8680 + }, + { + "epoch": 13.99481029878571, + "grad_norm": 0.6980776190757751, + "learning_rate": 0.0001, + "loss": 62.9427, + "step": 8690 + }, + { + "epoch": 14.009674006147025, + "grad_norm": 0.810961127281189, + "learning_rate": 0.0001, + "loss": 58.7552, + "step": 8700 + }, + { + "epoch": 14.0257973497254, + "grad_norm": 0.9210551381111145, + "learning_rate": 0.0001, + "loss": 63.7412, + "step": 8710 + }, + { + "epoch": 14.041920693303775, + "grad_norm": 0.7805785536766052, + "learning_rate": 0.0001, + "loss": 61.477, + "step": 8720 + }, + { + "epoch": 14.058044036882148, + "grad_norm": 0.8528208136558533, + "learning_rate": 0.0001, + "loss": 62.6746, + "step": 8730 + }, + { + "epoch": 14.074167380460523, + "grad_norm": 0.7467619776725769, + "learning_rate": 0.0001, + "loss": 63.8744, + "step": 8740 + }, + { + "epoch": 14.090290724038898, + "grad_norm": 0.6434402465820312, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 8750 + }, + { + "epoch": 14.106414067617273, + "grad_norm": 0.7217757701873779, + "learning_rate": 0.0001, + "loss": 64.0428, + "step": 8760 + }, + { + "epoch": 14.122537411195646, + "grad_norm": 0.8665125966072083, + "learning_rate": 0.0001, + "loss": 62.2772, + "step": 8770 + }, + { + "epoch": 14.13866075477402, + "grad_norm": 0.7868321537971497, + "learning_rate": 0.0001, + "loss": 62.0509, + "step": 8780 + }, + { + "epoch": 14.154784098352396, + "grad_norm": 0.8716760873794556, + "learning_rate": 0.0001, + "loss": 61.6021, + "step": 8790 + }, + { + "epoch": 14.17090744193077, + "grad_norm": 1.0922521352767944, + "learning_rate": 0.0001, + "loss": 64.4197, + "step": 8800 + }, + { + "epoch": 14.187030785509146, + "grad_norm": 0.6545577049255371, + "learning_rate": 0.0001, + "loss": 62.9259, + "step": 8810 + }, + { + "epoch": 14.203154129087519, + "grad_norm": 0.7634063959121704, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 8820 + }, + { + "epoch": 14.219277472665894, + "grad_norm": 0.9075300097465515, + "learning_rate": 0.0001, + "loss": 62.9128, + "step": 8830 + }, + { + "epoch": 14.235400816244269, + "grad_norm": 3.475320816040039, + "learning_rate": 0.0001, + "loss": 62.4841, + "step": 8840 + }, + { + "epoch": 14.251524159822644, + "grad_norm": 0.6839455962181091, + "learning_rate": 0.0001, + "loss": 61.5898, + "step": 8850 + }, + { + "epoch": 14.267647503401017, + "grad_norm": 0.8024231791496277, + "learning_rate": 0.0001, + "loss": 61.1875, + "step": 8860 + }, + { + "epoch": 14.283770846979392, + "grad_norm": 0.8948174118995667, + "learning_rate": 0.0001, + "loss": 62.3386, + "step": 8870 + }, + { + "epoch": 14.299894190557767, + "grad_norm": 0.7571985721588135, + "learning_rate": 0.0001, + "loss": 60.579, + "step": 8880 + }, + { + "epoch": 14.316017534136142, + "grad_norm": 1.432371973991394, + "learning_rate": 0.0001, + "loss": 62.4405, + "step": 8890 + }, + { + "epoch": 14.332140877714515, + "grad_norm": 0.6696875691413879, + "learning_rate": 0.0001, + "loss": 62.9984, + "step": 8900 + }, + { + "epoch": 14.34826422129289, + "grad_norm": 0.749396026134491, + "learning_rate": 0.0001, + "loss": 61.492, + "step": 8910 + }, + { + "epoch": 14.364387564871265, + "grad_norm": 0.664193332195282, + "learning_rate": 0.0001, + "loss": 62.9893, + "step": 8920 + }, + { + "epoch": 14.38051090844964, + "grad_norm": 0.7032697796821594, + "learning_rate": 0.0001, + "loss": 61.1682, + "step": 8930 + }, + { + "epoch": 14.396634252028015, + "grad_norm": 0.813119649887085, + "learning_rate": 0.0001, + "loss": 62.906, + "step": 8940 + }, + { + "epoch": 14.412757595606388, + "grad_norm": 0.7699596881866455, + "learning_rate": 0.0001, + "loss": 64.8087, + "step": 8950 + }, + { + "epoch": 14.428880939184763, + "grad_norm": 0.6424354910850525, + "learning_rate": 0.0001, + "loss": 62.2418, + "step": 8960 + }, + { + "epoch": 14.445004282763138, + "grad_norm": 0.8741731643676758, + "learning_rate": 0.0001, + "loss": 62.656, + "step": 8970 + }, + { + "epoch": 14.461127626341513, + "grad_norm": 0.710617184638977, + "learning_rate": 0.0001, + "loss": 60.8979, + "step": 8980 + }, + { + "epoch": 14.477250969919886, + "grad_norm": 0.6493995785713196, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 8990 + }, + { + "epoch": 14.493374313498261, + "grad_norm": 0.6655835509300232, + "learning_rate": 0.0001, + "loss": 64.151, + "step": 9000 + }, + { + "epoch": 14.509497657076636, + "grad_norm": 0.8140881657600403, + "learning_rate": 0.0001, + "loss": 62.4447, + "step": 9010 + }, + { + "epoch": 14.525621000655011, + "grad_norm": 0.7288498878479004, + "learning_rate": 0.0001, + "loss": 62.5189, + "step": 9020 + }, + { + "epoch": 14.541744344233386, + "grad_norm": 0.6972017884254456, + "learning_rate": 0.0001, + "loss": 61.5441, + "step": 9030 + }, + { + "epoch": 14.55786768781176, + "grad_norm": 0.7444366216659546, + "learning_rate": 0.0001, + "loss": 62.3233, + "step": 9040 + }, + { + "epoch": 14.573991031390134, + "grad_norm": 0.6602845191955566, + "learning_rate": 0.0001, + "loss": 61.4176, + "step": 9050 + }, + { + "epoch": 14.59011437496851, + "grad_norm": 0.7149770855903625, + "learning_rate": 0.0001, + "loss": 62.159, + "step": 9060 + }, + { + "epoch": 14.606237718546884, + "grad_norm": 0.6975772976875305, + "learning_rate": 0.0001, + "loss": 63.1857, + "step": 9070 + }, + { + "epoch": 14.622361062125258, + "grad_norm": 0.7835262417793274, + "learning_rate": 0.0001, + "loss": 61.6795, + "step": 9080 + }, + { + "epoch": 14.638484405703633, + "grad_norm": 5.790656566619873, + "learning_rate": 0.0001, + "loss": 62.7087, + "step": 9090 + }, + { + "epoch": 14.654607749282007, + "grad_norm": 1.0082288980484009, + "learning_rate": 0.0001, + "loss": 61.6808, + "step": 9100 + }, + { + "epoch": 14.670731092860382, + "grad_norm": 0.7758098840713501, + "learning_rate": 0.0001, + "loss": 61.0734, + "step": 9110 + }, + { + "epoch": 14.686854436438756, + "grad_norm": 0.7841722965240479, + "learning_rate": 0.0001, + "loss": 62.8882, + "step": 9120 + }, + { + "epoch": 14.70297778001713, + "grad_norm": 0.7695890069007874, + "learning_rate": 0.0001, + "loss": 60.7811, + "step": 9130 + }, + { + "epoch": 14.719101123595506, + "grad_norm": 0.7444761395454407, + "learning_rate": 0.0001, + "loss": 61.692, + "step": 9140 + }, + { + "epoch": 14.73522446717388, + "grad_norm": 0.7036821246147156, + "learning_rate": 0.0001, + "loss": 63.1698, + "step": 9150 + }, + { + "epoch": 14.751347810752256, + "grad_norm": 0.7891719937324524, + "learning_rate": 0.0001, + "loss": 63.8575, + "step": 9160 + }, + { + "epoch": 14.767471154330629, + "grad_norm": 0.6371109485626221, + "learning_rate": 0.0001, + "loss": 61.3358, + "step": 9170 + }, + { + "epoch": 14.783594497909004, + "grad_norm": 0.6551769375801086, + "learning_rate": 0.0001, + "loss": 62.2705, + "step": 9180 + }, + { + "epoch": 14.799717841487379, + "grad_norm": 0.6922310590744019, + "learning_rate": 0.0001, + "loss": 61.6442, + "step": 9190 + }, + { + "epoch": 14.815841185065754, + "grad_norm": 0.9615464210510254, + "learning_rate": 0.0001, + "loss": 60.932, + "step": 9200 + }, + { + "epoch": 14.831964528644127, + "grad_norm": 0.6367932558059692, + "learning_rate": 0.0001, + "loss": 61.3475, + "step": 9210 + }, + { + "epoch": 14.848087872222502, + "grad_norm": 0.5634505748748779, + "learning_rate": 0.0001, + "loss": 61.7562, + "step": 9220 + }, + { + "epoch": 14.864211215800877, + "grad_norm": 0.7021980285644531, + "learning_rate": 0.0001, + "loss": 62.0672, + "step": 9230 + }, + { + "epoch": 14.880334559379252, + "grad_norm": 0.8832064867019653, + "learning_rate": 0.0001, + "loss": 60.9882, + "step": 9240 + }, + { + "epoch": 14.896457902957625, + "grad_norm": 0.9043561220169067, + "learning_rate": 0.0001, + "loss": 63.0968, + "step": 9250 + }, + { + "epoch": 14.912581246536, + "grad_norm": 0.8958032131195068, + "learning_rate": 0.0001, + "loss": 61.5431, + "step": 9260 + }, + { + "epoch": 14.928704590114375, + "grad_norm": 0.7181615829467773, + "learning_rate": 0.0001, + "loss": 62.3626, + "step": 9270 + }, + { + "epoch": 14.94482793369275, + "grad_norm": 0.7541075348854065, + "learning_rate": 0.0001, + "loss": 61.7186, + "step": 9280 + }, + { + "epoch": 14.960951277271125, + "grad_norm": 1.5627169609069824, + "learning_rate": 0.0001, + "loss": 61.654, + "step": 9290 + }, + { + "epoch": 14.977074620849498, + "grad_norm": 0.7126879096031189, + "learning_rate": 0.0001, + "loss": 61.6806, + "step": 9300 + }, + { + "epoch": 14.993197964427873, + "grad_norm": 0.7397864460945129, + "learning_rate": 0.0001, + "loss": 61.2669, + "step": 9310 + }, + { + "epoch": 15.008061671789187, + "grad_norm": 0.7282766699790955, + "learning_rate": 0.0001, + "loss": 56.2289, + "step": 9320 + }, + { + "epoch": 15.024185015367562, + "grad_norm": 0.8346580862998962, + "learning_rate": 0.0001, + "loss": 61.4335, + "step": 9330 + }, + { + "epoch": 15.040308358945936, + "grad_norm": 0.9117271304130554, + "learning_rate": 0.0001, + "loss": 60.4636, + "step": 9340 + }, + { + "epoch": 15.05643170252431, + "grad_norm": 0.8483272194862366, + "learning_rate": 0.0001, + "loss": 61.3003, + "step": 9350 + }, + { + "epoch": 15.072555046102686, + "grad_norm": 1.1898932456970215, + "learning_rate": 0.0001, + "loss": 62.7169, + "step": 9360 + }, + { + "epoch": 15.08867838968106, + "grad_norm": 0.7258167862892151, + "learning_rate": 0.0001, + "loss": 63.5285, + "step": 9370 + }, + { + "epoch": 15.104801733259436, + "grad_norm": 0.7810865044593811, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 9380 + }, + { + "epoch": 15.120925076837809, + "grad_norm": 1.0359688997268677, + "learning_rate": 0.0001, + "loss": 62.5435, + "step": 9390 + }, + { + "epoch": 15.137048420416184, + "grad_norm": 0.7371546030044556, + "learning_rate": 0.0001, + "loss": 60.5583, + "step": 9400 + }, + { + "epoch": 15.153171763994559, + "grad_norm": 0.7578926086425781, + "learning_rate": 0.0001, + "loss": 61.2853, + "step": 9410 + }, + { + "epoch": 15.169295107572934, + "grad_norm": 0.6161713004112244, + "learning_rate": 0.0001, + "loss": 62.9725, + "step": 9420 + }, + { + "epoch": 15.185418451151307, + "grad_norm": 0.8325598835945129, + "learning_rate": 0.0001, + "loss": 62.857, + "step": 9430 + }, + { + "epoch": 15.201541794729682, + "grad_norm": 1.1438454389572144, + "learning_rate": 0.0001, + "loss": 61.8167, + "step": 9440 + }, + { + "epoch": 15.217665138308057, + "grad_norm": 1.1820837259292603, + "learning_rate": 0.0001, + "loss": 60.9899, + "step": 9450 + }, + { + "epoch": 15.233788481886432, + "grad_norm": 0.6057641506195068, + "learning_rate": 0.0001, + "loss": 62.5144, + "step": 9460 + }, + { + "epoch": 15.249911825464805, + "grad_norm": 0.6661948561668396, + "learning_rate": 0.0001, + "loss": 62.0835, + "step": 9470 + }, + { + "epoch": 15.26603516904318, + "grad_norm": 0.7623168230056763, + "learning_rate": 0.0001, + "loss": 61.9491, + "step": 9480 + }, + { + "epoch": 15.282158512621555, + "grad_norm": 0.6784029006958008, + "learning_rate": 0.0001, + "loss": 63.6006, + "step": 9490 + }, + { + "epoch": 15.29828185619993, + "grad_norm": 0.9433781504631042, + "learning_rate": 0.0001, + "loss": 61.6677, + "step": 9500 + }, + { + "epoch": 15.314405199778305, + "grad_norm": 0.6544079184532166, + "learning_rate": 0.0001, + "loss": 63.1439, + "step": 9510 + }, + { + "epoch": 15.330528543356678, + "grad_norm": 0.7979472279548645, + "learning_rate": 0.0001, + "loss": 62.719, + "step": 9520 + }, + { + "epoch": 15.346651886935053, + "grad_norm": 0.6641252040863037, + "learning_rate": 0.0001, + "loss": 63.2543, + "step": 9530 + }, + { + "epoch": 15.362775230513428, + "grad_norm": 0.7780525088310242, + "learning_rate": 0.0001, + "loss": 62.5659, + "step": 9540 + }, + { + "epoch": 15.378898574091803, + "grad_norm": 0.842349648475647, + "learning_rate": 0.0001, + "loss": 60.041, + "step": 9550 + }, + { + "epoch": 15.395021917670176, + "grad_norm": 0.8991017937660217, + "learning_rate": 0.0001, + "loss": 62.9075, + "step": 9560 + }, + { + "epoch": 15.411145261248551, + "grad_norm": 0.6734052300453186, + "learning_rate": 0.0001, + "loss": 60.8647, + "step": 9570 + }, + { + "epoch": 15.427268604826926, + "grad_norm": 0.7585113644599915, + "learning_rate": 0.0001, + "loss": 62.105, + "step": 9580 + }, + { + "epoch": 15.443391948405301, + "grad_norm": 0.7915840148925781, + "learning_rate": 0.0001, + "loss": 62.2741, + "step": 9590 + }, + { + "epoch": 15.459515291983674, + "grad_norm": 0.8676218390464783, + "learning_rate": 0.0001, + "loss": 62.4548, + "step": 9600 + }, + { + "epoch": 15.47563863556205, + "grad_norm": 0.7698650360107422, + "learning_rate": 0.0001, + "loss": 61.3412, + "step": 9610 + }, + { + "epoch": 15.491761979140424, + "grad_norm": 0.6600908637046814, + "learning_rate": 0.0001, + "loss": 61.1561, + "step": 9620 + }, + { + "epoch": 15.5078853227188, + "grad_norm": 0.5995001792907715, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 9630 + }, + { + "epoch": 15.524008666297174, + "grad_norm": 0.7258557677268982, + "learning_rate": 0.0001, + "loss": 62.4659, + "step": 9640 + }, + { + "epoch": 15.540132009875547, + "grad_norm": 0.6562618613243103, + "learning_rate": 0.0001, + "loss": 63.288, + "step": 9650 + }, + { + "epoch": 15.556255353453922, + "grad_norm": 0.9224995374679565, + "learning_rate": 0.0001, + "loss": 61.4861, + "step": 9660 + }, + { + "epoch": 15.572378697032297, + "grad_norm": 0.7053484916687012, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 9670 + }, + { + "epoch": 15.588502040610672, + "grad_norm": 1.095229148864746, + "learning_rate": 0.0001, + "loss": 62.1542, + "step": 9680 + }, + { + "epoch": 15.604625384189045, + "grad_norm": 0.7563759088516235, + "learning_rate": 0.0001, + "loss": 61.1644, + "step": 9690 + }, + { + "epoch": 15.62074872776742, + "grad_norm": 0.6508835554122925, + "learning_rate": 0.0001, + "loss": 60.1374, + "step": 9700 + }, + { + "epoch": 15.636872071345795, + "grad_norm": 0.664799153804779, + "learning_rate": 0.0001, + "loss": 60.2331, + "step": 9710 + }, + { + "epoch": 15.65299541492417, + "grad_norm": 0.6525771617889404, + "learning_rate": 0.0001, + "loss": 60.6142, + "step": 9720 + }, + { + "epoch": 15.669118758502545, + "grad_norm": 0.7395381331443787, + "learning_rate": 0.0001, + "loss": 63.5623, + "step": 9730 + }, + { + "epoch": 15.685242102080919, + "grad_norm": 0.7063724398612976, + "learning_rate": 0.0001, + "loss": 63.7133, + "step": 9740 + }, + { + "epoch": 15.701365445659293, + "grad_norm": 0.7918921113014221, + "learning_rate": 0.0001, + "loss": 62.5957, + "step": 9750 + }, + { + "epoch": 15.717488789237668, + "grad_norm": 0.8135486245155334, + "learning_rate": 0.0001, + "loss": 62.9146, + "step": 9760 + }, + { + "epoch": 15.733612132816043, + "grad_norm": 0.8422091007232666, + "learning_rate": 0.0001, + "loss": 61.1466, + "step": 9770 + }, + { + "epoch": 15.749735476394417, + "grad_norm": 0.6329447627067566, + "learning_rate": 0.0001, + "loss": 61.7247, + "step": 9780 + }, + { + "epoch": 15.765858819972792, + "grad_norm": 1.0627039670944214, + "learning_rate": 0.0001, + "loss": 60.6789, + "step": 9790 + }, + { + "epoch": 15.781982163551167, + "grad_norm": 0.7316949367523193, + "learning_rate": 0.0001, + "loss": 61.9896, + "step": 9800 + }, + { + "epoch": 15.798105507129542, + "grad_norm": 0.6968954205513, + "learning_rate": 0.0001, + "loss": 60.8571, + "step": 9810 + }, + { + "epoch": 15.814228850707915, + "grad_norm": 0.7226986289024353, + "learning_rate": 0.0001, + "loss": 62.2664, + "step": 9820 + }, + { + "epoch": 15.83035219428629, + "grad_norm": 0.6542951464653015, + "learning_rate": 0.0001, + "loss": 61.3316, + "step": 9830 + }, + { + "epoch": 15.846475537864665, + "grad_norm": 0.7662289142608643, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 9840 + }, + { + "epoch": 15.86259888144304, + "grad_norm": 0.7060043215751648, + "learning_rate": 0.0001, + "loss": 62.2647, + "step": 9850 + }, + { + "epoch": 15.878722225021415, + "grad_norm": 0.6948843002319336, + "learning_rate": 0.0001, + "loss": 62.8678, + "step": 9860 + }, + { + "epoch": 15.894845568599788, + "grad_norm": 0.7787327766418457, + "learning_rate": 0.0001, + "loss": 62.9154, + "step": 9870 + }, + { + "epoch": 15.910968912178163, + "grad_norm": 0.7077026963233948, + "learning_rate": 0.0001, + "loss": 62.9537, + "step": 9880 + }, + { + "epoch": 15.927092255756538, + "grad_norm": 0.9957109093666077, + "learning_rate": 0.0001, + "loss": 60.9461, + "step": 9890 + }, + { + "epoch": 15.943215599334913, + "grad_norm": 0.9767161011695862, + "learning_rate": 0.0001, + "loss": 62.5944, + "step": 9900 + }, + { + "epoch": 15.959338942913286, + "grad_norm": 0.6016744375228882, + "learning_rate": 0.0001, + "loss": 62.5889, + "step": 9910 + }, + { + "epoch": 15.975462286491661, + "grad_norm": 0.5464991331100464, + "learning_rate": 0.0001, + "loss": 61.3138, + "step": 9920 + }, + { + "epoch": 15.991585630070036, + "grad_norm": 0.8242359757423401, + "learning_rate": 0.0001, + "loss": 61.8091, + "step": 9930 + }, + { + "epoch": 16.00644933743135, + "grad_norm": 0.7031534314155579, + "learning_rate": 0.0001, + "loss": 57.4847, + "step": 9940 + }, + { + "epoch": 16.022572681009724, + "grad_norm": 0.8429487943649292, + "learning_rate": 0.0001, + "loss": 61.7286, + "step": 9950 + }, + { + "epoch": 16.0386960245881, + "grad_norm": 0.7200850248336792, + "learning_rate": 0.0001, + "loss": 62.6968, + "step": 9960 + }, + { + "epoch": 16.054819368166473, + "grad_norm": 0.7412559390068054, + "learning_rate": 0.0001, + "loss": 62.413, + "step": 9970 + }, + { + "epoch": 16.07094271174485, + "grad_norm": 0.8179170489311218, + "learning_rate": 0.0001, + "loss": 62.7044, + "step": 9980 + }, + { + "epoch": 16.087066055323223, + "grad_norm": 0.6939452886581421, + "learning_rate": 0.0001, + "loss": 61.9062, + "step": 9990 + }, + { + "epoch": 16.1031893989016, + "grad_norm": 0.7145045399665833, + "learning_rate": 0.0001, + "loss": 61.6201, + "step": 10000 + }, + { + "epoch": 16.119312742479973, + "grad_norm": 0.7874398231506348, + "learning_rate": 0.0001, + "loss": 60.4299, + "step": 10010 + }, + { + "epoch": 16.135436086058345, + "grad_norm": 0.6640581488609314, + "learning_rate": 0.0001, + "loss": 62.0195, + "step": 10020 + }, + { + "epoch": 16.15155942963672, + "grad_norm": 0.7903703451156616, + "learning_rate": 0.0001, + "loss": 62.501, + "step": 10030 + }, + { + "epoch": 16.167682773215095, + "grad_norm": 0.6669818162918091, + "learning_rate": 0.0001, + "loss": 59.1489, + "step": 10040 + }, + { + "epoch": 16.18380611679347, + "grad_norm": 0.8380771279335022, + "learning_rate": 0.0001, + "loss": 61.0985, + "step": 10050 + }, + { + "epoch": 16.199929460371845, + "grad_norm": 0.7925304174423218, + "learning_rate": 0.0001, + "loss": 62.9251, + "step": 10060 + }, + { + "epoch": 16.21605280395022, + "grad_norm": 1.2176182270050049, + "learning_rate": 0.0001, + "loss": 61.6772, + "step": 10070 + }, + { + "epoch": 16.232176147528595, + "grad_norm": 0.828171968460083, + "learning_rate": 0.0001, + "loss": 61.3296, + "step": 10080 + }, + { + "epoch": 16.24829949110697, + "grad_norm": 0.6884972453117371, + "learning_rate": 0.0001, + "loss": 61.5371, + "step": 10090 + }, + { + "epoch": 16.264422834685345, + "grad_norm": 0.6455051302909851, + "learning_rate": 0.0001, + "loss": 60.6548, + "step": 10100 + }, + { + "epoch": 16.280546178263716, + "grad_norm": 0.687982976436615, + "learning_rate": 0.0001, + "loss": 62.0357, + "step": 10110 + }, + { + "epoch": 16.29666952184209, + "grad_norm": 0.811744213104248, + "learning_rate": 0.0001, + "loss": 60.5664, + "step": 10120 + }, + { + "epoch": 16.312792865420466, + "grad_norm": 0.7140675187110901, + "learning_rate": 0.0001, + "loss": 62.9491, + "step": 10130 + }, + { + "epoch": 16.32891620899884, + "grad_norm": 0.9068045020103455, + "learning_rate": 0.0001, + "loss": 61.118, + "step": 10140 + }, + { + "epoch": 16.345039552577216, + "grad_norm": 0.9028809666633606, + "learning_rate": 0.0001, + "loss": 61.4308, + "step": 10150 + }, + { + "epoch": 16.36116289615559, + "grad_norm": 0.7502166628837585, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10160 + }, + { + "epoch": 16.377286239733966, + "grad_norm": 0.807391345500946, + "learning_rate": 0.0001, + "loss": 61.6825, + "step": 10170 + }, + { + "epoch": 16.39340958331234, + "grad_norm": 0.762725830078125, + "learning_rate": 0.0001, + "loss": 62.7879, + "step": 10180 + }, + { + "epoch": 16.409532926890712, + "grad_norm": 0.7973660230636597, + "learning_rate": 0.0001, + "loss": 61.0727, + "step": 10190 + }, + { + "epoch": 16.425656270469087, + "grad_norm": 1.004622459411621, + "learning_rate": 0.0001, + "loss": 59.8048, + "step": 10200 + }, + { + "epoch": 16.441779614047462, + "grad_norm": 0.8243395686149597, + "learning_rate": 0.0001, + "loss": 63.9312, + "step": 10210 + }, + { + "epoch": 16.457902957625837, + "grad_norm": 0.7006281614303589, + "learning_rate": 0.0001, + "loss": 61.3766, + "step": 10220 + }, + { + "epoch": 16.474026301204212, + "grad_norm": 0.8672197461128235, + "learning_rate": 0.0001, + "loss": 61.6659, + "step": 10230 + }, + { + "epoch": 16.490149644782587, + "grad_norm": 0.7836477160453796, + "learning_rate": 0.0001, + "loss": 61.3686, + "step": 10240 + }, + { + "epoch": 16.506272988360962, + "grad_norm": 0.5842607617378235, + "learning_rate": 0.0001, + "loss": 62.616, + "step": 10250 + }, + { + "epoch": 16.522396331939337, + "grad_norm": 0.6777102947235107, + "learning_rate": 0.0001, + "loss": 62.1135, + "step": 10260 + }, + { + "epoch": 16.538519675517712, + "grad_norm": 0.856697678565979, + "learning_rate": 0.0001, + "loss": 61.849, + "step": 10270 + }, + { + "epoch": 16.554643019096083, + "grad_norm": 0.7300705313682556, + "learning_rate": 0.0001, + "loss": 60.7678, + "step": 10280 + }, + { + "epoch": 16.57076636267446, + "grad_norm": 0.799802839756012, + "learning_rate": 0.0001, + "loss": 62.4888, + "step": 10290 + }, + { + "epoch": 16.586889706252833, + "grad_norm": 0.791251540184021, + "learning_rate": 0.0001, + "loss": 61.7589, + "step": 10300 + }, + { + "epoch": 16.60301304983121, + "grad_norm": 0.6498433947563171, + "learning_rate": 0.0001, + "loss": 61.087, + "step": 10310 + }, + { + "epoch": 16.619136393409583, + "grad_norm": 0.702925443649292, + "learning_rate": 0.0001, + "loss": 59.3493, + "step": 10320 + }, + { + "epoch": 16.63525973698796, + "grad_norm": 0.8508023619651794, + "learning_rate": 0.0001, + "loss": 61.2188, + "step": 10330 + }, + { + "epoch": 16.651383080566333, + "grad_norm": 0.8325870633125305, + "learning_rate": 0.0001, + "loss": 62.9164, + "step": 10340 + }, + { + "epoch": 16.667506424144708, + "grad_norm": 0.7140557765960693, + "learning_rate": 0.0001, + "loss": 62.5838, + "step": 10350 + }, + { + "epoch": 16.683629767723083, + "grad_norm": 0.5574952363967896, + "learning_rate": 0.0001, + "loss": 59.767, + "step": 10360 + }, + { + "epoch": 16.699753111301455, + "grad_norm": 0.8022965788841248, + "learning_rate": 0.0001, + "loss": 60.2198, + "step": 10370 + }, + { + "epoch": 16.71587645487983, + "grad_norm": 1.1291698217391968, + "learning_rate": 0.0001, + "loss": 61.4908, + "step": 10380 + }, + { + "epoch": 16.731999798458205, + "grad_norm": 0.8651410937309265, + "learning_rate": 0.0001, + "loss": 60.5228, + "step": 10390 + }, + { + "epoch": 16.74812314203658, + "grad_norm": 0.8426535129547119, + "learning_rate": 0.0001, + "loss": 62.3786, + "step": 10400 + }, + { + "epoch": 16.764246485614954, + "grad_norm": 0.6329453587532043, + "learning_rate": 0.0001, + "loss": 60.9621, + "step": 10410 + }, + { + "epoch": 16.78036982919333, + "grad_norm": 0.6745129227638245, + "learning_rate": 0.0001, + "loss": 59.3405, + "step": 10420 + }, + { + "epoch": 16.796493172771704, + "grad_norm": 0.8483593463897705, + "learning_rate": 0.0001, + "loss": 61.1527, + "step": 10430 + }, + { + "epoch": 16.81261651635008, + "grad_norm": 0.8649617433547974, + "learning_rate": 0.0001, + "loss": 61.051, + "step": 10440 + }, + { + "epoch": 16.828739859928454, + "grad_norm": 0.7309728264808655, + "learning_rate": 0.0001, + "loss": 60.3626, + "step": 10450 + }, + { + "epoch": 16.844863203506826, + "grad_norm": 0.6467884182929993, + "learning_rate": 0.0001, + "loss": 63.8254, + "step": 10460 + }, + { + "epoch": 16.8609865470852, + "grad_norm": 0.7628102898597717, + "learning_rate": 0.0001, + "loss": 59.5604, + "step": 10470 + }, + { + "epoch": 16.877109890663576, + "grad_norm": 0.78266841173172, + "learning_rate": 0.0001, + "loss": 60.9387, + "step": 10480 + }, + { + "epoch": 16.89323323424195, + "grad_norm": 0.6936920285224915, + "learning_rate": 0.0001, + "loss": 62.5492, + "step": 10490 + }, + { + "epoch": 16.909356577820326, + "grad_norm": 0.8343939185142517, + "learning_rate": 0.0001, + "loss": 60.9975, + "step": 10500 + }, + { + "epoch": 16.9254799213987, + "grad_norm": 0.798984169960022, + "learning_rate": 0.0001, + "loss": 59.5835, + "step": 10510 + }, + { + "epoch": 16.941603264977076, + "grad_norm": 1.369275450706482, + "learning_rate": 0.0001, + "loss": 61.749, + "step": 10520 + }, + { + "epoch": 16.95772660855545, + "grad_norm": 0.5993408560752869, + "learning_rate": 0.0001, + "loss": 62.1093, + "step": 10530 + }, + { + "epoch": 16.973849952133826, + "grad_norm": 0.8153544664382935, + "learning_rate": 0.0001, + "loss": 61.857, + "step": 10540 + }, + { + "epoch": 16.989973295712197, + "grad_norm": 0.7042689919471741, + "learning_rate": 0.0001, + "loss": 60.5178, + "step": 10550 + }, + { + "epoch": 17.00483700307351, + "grad_norm": 1.1931812763214111, + "learning_rate": 0.0001, + "loss": 56.4815, + "step": 10560 + }, + { + "epoch": 17.020960346651886, + "grad_norm": 0.9051854610443115, + "learning_rate": 0.0001, + "loss": 60.7729, + "step": 10570 + }, + { + "epoch": 17.03708369023026, + "grad_norm": 1.093518614768982, + "learning_rate": 0.0001, + "loss": 60.904, + "step": 10580 + }, + { + "epoch": 17.053207033808636, + "grad_norm": 0.7545284032821655, + "learning_rate": 0.0001, + "loss": 59.7165, + "step": 10590 + }, + { + "epoch": 17.06933037738701, + "grad_norm": 0.8542641401290894, + "learning_rate": 0.0001, + "loss": 60.8947, + "step": 10600 + }, + { + "epoch": 17.085453720965386, + "grad_norm": 0.7393799424171448, + "learning_rate": 0.0001, + "loss": 62.8384, + "step": 10610 + }, + { + "epoch": 17.10157706454376, + "grad_norm": 0.6756435632705688, + "learning_rate": 0.0001, + "loss": 61.164, + "step": 10620 + }, + { + "epoch": 17.117700408122133, + "grad_norm": 0.725150465965271, + "learning_rate": 0.0001, + "loss": 60.3789, + "step": 10630 + }, + { + "epoch": 17.133823751700508, + "grad_norm": 1.1529417037963867, + "learning_rate": 0.0001, + "loss": 62.2465, + "step": 10640 + }, + { + "epoch": 17.149947095278883, + "grad_norm": 0.8266693353652954, + "learning_rate": 0.0001, + "loss": 61.6755, + "step": 10650 + }, + { + "epoch": 17.166070438857258, + "grad_norm": 0.7444040775299072, + "learning_rate": 0.0001, + "loss": 62.7084, + "step": 10660 + }, + { + "epoch": 17.182193782435633, + "grad_norm": 0.8164665102958679, + "learning_rate": 0.0001, + "loss": 61.6032, + "step": 10670 + }, + { + "epoch": 17.198317126014008, + "grad_norm": 0.8215296268463135, + "learning_rate": 0.0001, + "loss": 61.033, + "step": 10680 + }, + { + "epoch": 17.214440469592382, + "grad_norm": 0.7939956188201904, + "learning_rate": 0.0001, + "loss": 60.5016, + "step": 10690 + }, + { + "epoch": 17.230563813170757, + "grad_norm": 0.9533108472824097, + "learning_rate": 0.0001, + "loss": 62.6995, + "step": 10700 + }, + { + "epoch": 17.246687156749132, + "grad_norm": 0.6602820754051208, + "learning_rate": 0.0001, + "loss": 61.2709, + "step": 10710 + }, + { + "epoch": 17.262810500327504, + "grad_norm": 0.8369201421737671, + "learning_rate": 0.0001, + "loss": 63.1138, + "step": 10720 + }, + { + "epoch": 17.27893384390588, + "grad_norm": 0.7431164979934692, + "learning_rate": 0.0001, + "loss": 62.8588, + "step": 10730 + }, + { + "epoch": 17.295057187484254, + "grad_norm": 0.7957271337509155, + "learning_rate": 0.0001, + "loss": 60.5119, + "step": 10740 + }, + { + "epoch": 17.31118053106263, + "grad_norm": 0.9407948851585388, + "learning_rate": 0.0001, + "loss": 61.1877, + "step": 10750 + }, + { + "epoch": 17.327303874641004, + "grad_norm": 0.6641581654548645, + "learning_rate": 0.0001, + "loss": 60.542, + "step": 10760 + }, + { + "epoch": 17.34342721821938, + "grad_norm": 0.7786923050880432, + "learning_rate": 0.0001, + "loss": 62.1582, + "step": 10770 + }, + { + "epoch": 17.359550561797754, + "grad_norm": 0.8150411248207092, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10780 + }, + { + "epoch": 17.37567390537613, + "grad_norm": 0.8315487504005432, + "learning_rate": 0.0001, + "loss": 60.4382, + "step": 10790 + }, + { + "epoch": 17.391797248954504, + "grad_norm": 0.8305006623268127, + "learning_rate": 0.0001, + "loss": 61.3262, + "step": 10800 + }, + { + "epoch": 17.407920592532875, + "grad_norm": 0.897731363773346, + "learning_rate": 0.0001, + "loss": 60.7809, + "step": 10810 + }, + { + "epoch": 17.42404393611125, + "grad_norm": 0.7461619973182678, + "learning_rate": 0.0001, + "loss": 61.3858, + "step": 10820 + }, + { + "epoch": 17.440167279689625, + "grad_norm": 0.6750472187995911, + "learning_rate": 0.0001, + "loss": 60.2692, + "step": 10830 + }, + { + "epoch": 17.456290623268, + "grad_norm": 0.5412933230400085, + "learning_rate": 0.0001, + "loss": 61.7872, + "step": 10840 + }, + { + "epoch": 17.472413966846375, + "grad_norm": 0.6485120058059692, + "learning_rate": 0.0001, + "loss": 59.5407, + "step": 10850 + }, + { + "epoch": 17.48853731042475, + "grad_norm": 0.818708598613739, + "learning_rate": 0.0001, + "loss": 61.8066, + "step": 10860 + }, + { + "epoch": 17.504660654003125, + "grad_norm": 0.7121092081069946, + "learning_rate": 0.0001, + "loss": 61.8086, + "step": 10870 + }, + { + "epoch": 17.5207839975815, + "grad_norm": 0.7031684517860413, + "learning_rate": 0.0001, + "loss": 60.0706, + "step": 10880 + }, + { + "epoch": 17.536907341159875, + "grad_norm": 0.6259323358535767, + "learning_rate": 0.0001, + "loss": 61.9625, + "step": 10890 + }, + { + "epoch": 17.553030684738246, + "grad_norm": 0.8526529669761658, + "learning_rate": 0.0001, + "loss": 61.3299, + "step": 10900 + }, + { + "epoch": 17.56915402831662, + "grad_norm": 0.6860859990119934, + "learning_rate": 0.0001, + "loss": 61.3193, + "step": 10910 + }, + { + "epoch": 17.585277371894996, + "grad_norm": 0.7715994119644165, + "learning_rate": 0.0001, + "loss": 61.9111, + "step": 10920 + }, + { + "epoch": 17.60140071547337, + "grad_norm": 0.7171186804771423, + "learning_rate": 0.0001, + "loss": 60.3638, + "step": 10930 + }, + { + "epoch": 17.617524059051746, + "grad_norm": 0.7796649932861328, + "learning_rate": 0.0001, + "loss": 61.9437, + "step": 10940 + }, + { + "epoch": 17.63364740263012, + "grad_norm": 0.7930101752281189, + "learning_rate": 0.0001, + "loss": 60.1085, + "step": 10950 + }, + { + "epoch": 17.649770746208496, + "grad_norm": 0.6370466351509094, + "learning_rate": 0.0001, + "loss": 61.8433, + "step": 10960 + }, + { + "epoch": 17.66589408978687, + "grad_norm": 0.7830466628074646, + "learning_rate": 0.0001, + "loss": 61.3403, + "step": 10970 + }, + { + "epoch": 17.682017433365242, + "grad_norm": 0.873456597328186, + "learning_rate": 0.0001, + "loss": 62.4567, + "step": 10980 + }, + { + "epoch": 17.698140776943617, + "grad_norm": 0.7560754418373108, + "learning_rate": 0.0001, + "loss": 60.9808, + "step": 10990 + }, + { + "epoch": 17.714264120521992, + "grad_norm": 0.6832230091094971, + "learning_rate": 0.0001, + "loss": 61.5059, + "step": 11000 + }, + { + "epoch": 17.730387464100367, + "grad_norm": 0.7701742649078369, + "learning_rate": 0.0001, + "loss": 62.4963, + "step": 11010 + }, + { + "epoch": 17.746510807678742, + "grad_norm": 0.7088269591331482, + "learning_rate": 0.0001, + "loss": 62.6467, + "step": 11020 + }, + { + "epoch": 17.762634151257117, + "grad_norm": 0.8424612879753113, + "learning_rate": 0.0001, + "loss": 60.1213, + "step": 11030 + }, + { + "epoch": 17.778757494835492, + "grad_norm": 0.9974421858787537, + "learning_rate": 0.0001, + "loss": 61.6051, + "step": 11040 + }, + { + "epoch": 17.794880838413867, + "grad_norm": 0.701970636844635, + "learning_rate": 0.0001, + "loss": 62.4155, + "step": 11050 + }, + { + "epoch": 17.811004181992242, + "grad_norm": 0.6826842427253723, + "learning_rate": 0.0001, + "loss": 60.2127, + "step": 11060 + }, + { + "epoch": 17.827127525570614, + "grad_norm": 0.7019745707511902, + "learning_rate": 0.0001, + "loss": 60.9281, + "step": 11070 + }, + { + "epoch": 17.84325086914899, + "grad_norm": 0.836280345916748, + "learning_rate": 0.0001, + "loss": 61.1217, + "step": 11080 + }, + { + "epoch": 17.859374212727364, + "grad_norm": 0.6405923366546631, + "learning_rate": 0.0001, + "loss": 61.6393, + "step": 11090 + }, + { + "epoch": 17.87549755630574, + "grad_norm": 0.6809528470039368, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 11100 + }, + { + "epoch": 17.891620899884114, + "grad_norm": 0.6983410120010376, + "learning_rate": 0.0001, + "loss": 59.933, + "step": 11110 + }, + { + "epoch": 17.90774424346249, + "grad_norm": 1.0240110158920288, + "learning_rate": 0.0001, + "loss": 61.1964, + "step": 11120 + }, + { + "epoch": 17.923867587040863, + "grad_norm": 0.860657811164856, + "learning_rate": 0.0001, + "loss": 61.1613, + "step": 11130 + }, + { + "epoch": 17.93999093061924, + "grad_norm": 0.6434107422828674, + "learning_rate": 0.0001, + "loss": 60.2435, + "step": 11140 + }, + { + "epoch": 17.956114274197613, + "grad_norm": 0.7400848269462585, + "learning_rate": 0.0001, + "loss": 62.452, + "step": 11150 + }, + { + "epoch": 17.972237617775985, + "grad_norm": 0.6526095867156982, + "learning_rate": 0.0001, + "loss": 62.3344, + "step": 11160 + }, + { + "epoch": 17.98836096135436, + "grad_norm": 0.6971097588539124, + "learning_rate": 0.0001, + "loss": 60.2415, + "step": 11170 + }, + { + "epoch": 18.003224668715674, + "grad_norm": 0.724881649017334, + "learning_rate": 0.0001, + "loss": 56.0513, + "step": 11180 + }, + { + "epoch": 18.01934801229405, + "grad_norm": 0.653710663318634, + "learning_rate": 0.0001, + "loss": 60.8364, + "step": 11190 + }, + { + "epoch": 18.035471355872424, + "grad_norm": 0.7356674075126648, + "learning_rate": 0.0001, + "loss": 61.6681, + "step": 11200 + }, + { + "epoch": 18.0515946994508, + "grad_norm": 0.6726879477500916, + "learning_rate": 0.0001, + "loss": 60.6682, + "step": 11210 + }, + { + "epoch": 18.067718043029174, + "grad_norm": 0.7367722392082214, + "learning_rate": 0.0001, + "loss": 63.0462, + "step": 11220 + }, + { + "epoch": 18.08384138660755, + "grad_norm": 0.6687121987342834, + "learning_rate": 0.0001, + "loss": 60.8707, + "step": 11230 + }, + { + "epoch": 18.09996473018592, + "grad_norm": 1.0027060508728027, + "learning_rate": 0.0001, + "loss": 62.4703, + "step": 11240 + }, + { + "epoch": 18.116088073764296, + "grad_norm": 1.0447720289230347, + "learning_rate": 0.0001, + "loss": 62.1746, + "step": 11250 + }, + { + "epoch": 18.13221141734267, + "grad_norm": 0.7687053084373474, + "learning_rate": 0.0001, + "loss": 60.1413, + "step": 11260 + }, + { + "epoch": 18.148334760921045, + "grad_norm": 0.7250781059265137, + "learning_rate": 0.0001, + "loss": 60.1143, + "step": 11270 + }, + { + "epoch": 18.16445810449942, + "grad_norm": 0.7356208562850952, + "learning_rate": 0.0001, + "loss": 62.0788, + "step": 11280 + }, + { + "epoch": 18.180581448077795, + "grad_norm": 0.7838225364685059, + "learning_rate": 0.0001, + "loss": 61.7046, + "step": 11290 + }, + { + "epoch": 18.19670479165617, + "grad_norm": 0.7600042819976807, + "learning_rate": 0.0001, + "loss": 61.4646, + "step": 11300 + }, + { + "epoch": 18.212828135234545, + "grad_norm": 0.7339571714401245, + "learning_rate": 0.0001, + "loss": 61.0983, + "step": 11310 + }, + { + "epoch": 18.22895147881292, + "grad_norm": 0.6982674598693848, + "learning_rate": 0.0001, + "loss": 60.2853, + "step": 11320 + }, + { + "epoch": 18.24507482239129, + "grad_norm": 0.7572301030158997, + "learning_rate": 0.0001, + "loss": 61.8967, + "step": 11330 + }, + { + "epoch": 18.261198165969667, + "grad_norm": 0.6206921935081482, + "learning_rate": 0.0001, + "loss": 62.1467, + "step": 11340 + }, + { + "epoch": 18.27732150954804, + "grad_norm": 0.658437967300415, + "learning_rate": 0.0001, + "loss": 61.2117, + "step": 11350 + }, + { + "epoch": 18.293444853126417, + "grad_norm": 0.7880764603614807, + "learning_rate": 0.0001, + "loss": 61.6246, + "step": 11360 + }, + { + "epoch": 18.30956819670479, + "grad_norm": 0.6601800918579102, + "learning_rate": 0.0001, + "loss": 61.4818, + "step": 11370 + }, + { + "epoch": 18.325691540283167, + "grad_norm": 0.6816290616989136, + "learning_rate": 0.0001, + "loss": 60.4834, + "step": 11380 + }, + { + "epoch": 18.34181488386154, + "grad_norm": 0.7667511105537415, + "learning_rate": 0.0001, + "loss": 60.9106, + "step": 11390 + }, + { + "epoch": 18.357938227439917, + "grad_norm": 0.6490039825439453, + "learning_rate": 0.0001, + "loss": 59.6968, + "step": 11400 + }, + { + "epoch": 18.37406157101829, + "grad_norm": 0.6865658760070801, + "learning_rate": 0.0001, + "loss": 61.7898, + "step": 11410 + }, + { + "epoch": 18.390184914596663, + "grad_norm": 0.8605737090110779, + "learning_rate": 0.0001, + "loss": 60.924, + "step": 11420 + }, + { + "epoch": 18.406308258175038, + "grad_norm": 1.0474110841751099, + "learning_rate": 0.0001, + "loss": 61.0381, + "step": 11430 + }, + { + "epoch": 18.422431601753413, + "grad_norm": 0.9241340160369873, + "learning_rate": 0.0001, + "loss": 61.2609, + "step": 11440 + }, + { + "epoch": 18.438554945331788, + "grad_norm": 1.1441621780395508, + "learning_rate": 0.0001, + "loss": 60.5495, + "step": 11450 + }, + { + "epoch": 18.454678288910163, + "grad_norm": 0.7277412414550781, + "learning_rate": 0.0001, + "loss": 60.4654, + "step": 11460 + }, + { + "epoch": 18.470801632488538, + "grad_norm": 0.6668717861175537, + "learning_rate": 0.0001, + "loss": 60.057, + "step": 11470 + }, + { + "epoch": 18.486924976066913, + "grad_norm": 0.7335819005966187, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 11480 + }, + { + "epoch": 18.503048319645288, + "grad_norm": 0.7261807918548584, + "learning_rate": 0.0001, + "loss": 61.1508, + "step": 11490 + }, + { + "epoch": 18.519171663223663, + "grad_norm": 0.7715283036231995, + "learning_rate": 0.0001, + "loss": 61.5492, + "step": 11500 + }, + { + "epoch": 18.535295006802034, + "grad_norm": 0.9603212475776672, + "learning_rate": 0.0001, + "loss": 60.4321, + "step": 11510 + }, + { + "epoch": 18.55141835038041, + "grad_norm": 0.6130046844482422, + "learning_rate": 0.0001, + "loss": 61.2942, + "step": 11520 + }, + { + "epoch": 18.567541693958784, + "grad_norm": 0.8147348165512085, + "learning_rate": 0.0001, + "loss": 61.4313, + "step": 11530 + }, + { + "epoch": 18.58366503753716, + "grad_norm": 0.7909948825836182, + "learning_rate": 0.0001, + "loss": 60.3547, + "step": 11540 + }, + { + "epoch": 18.599788381115534, + "grad_norm": 0.6470443606376648, + "learning_rate": 0.0001, + "loss": 61.4144, + "step": 11550 + }, + { + "epoch": 18.61591172469391, + "grad_norm": 0.7132186889648438, + "learning_rate": 0.0001, + "loss": 61.8102, + "step": 11560 + }, + { + "epoch": 18.632035068272284, + "grad_norm": 0.7054323554039001, + "learning_rate": 0.0001, + "loss": 62.1431, + "step": 11570 + }, + { + "epoch": 18.64815841185066, + "grad_norm": 0.8784761428833008, + "learning_rate": 0.0001, + "loss": 61.6653, + "step": 11580 + }, + { + "epoch": 18.66428175542903, + "grad_norm": 0.7143248319625854, + "learning_rate": 0.0001, + "loss": 60.1275, + "step": 11590 + }, + { + "epoch": 18.680405099007405, + "grad_norm": 7.87012243270874, + "learning_rate": 0.0001, + "loss": 61.3484, + "step": 11600 + }, + { + "epoch": 18.69652844258578, + "grad_norm": 0.7263020873069763, + "learning_rate": 0.0001, + "loss": 59.906, + "step": 11610 + }, + { + "epoch": 18.712651786164155, + "grad_norm": 0.6468373537063599, + "learning_rate": 0.0001, + "loss": 61.8781, + "step": 11620 + }, + { + "epoch": 18.72877512974253, + "grad_norm": 3.34307861328125, + "learning_rate": 0.0001, + "loss": 61.0841, + "step": 11630 + }, + { + "epoch": 18.744898473320905, + "grad_norm": 0.7348340153694153, + "learning_rate": 0.0001, + "loss": 61.3201, + "step": 11640 + }, + { + "epoch": 18.76102181689928, + "grad_norm": 0.7646427154541016, + "learning_rate": 0.0001, + "loss": 60.1858, + "step": 11650 + }, + { + "epoch": 18.777145160477655, + "grad_norm": 0.6659495234489441, + "learning_rate": 0.0001, + "loss": 60.8999, + "step": 11660 + }, + { + "epoch": 18.79326850405603, + "grad_norm": 0.9913583993911743, + "learning_rate": 0.0001, + "loss": 59.0732, + "step": 11670 + }, + { + "epoch": 18.8093918476344, + "grad_norm": 0.7480027079582214, + "learning_rate": 0.0001, + "loss": 58.6846, + "step": 11680 + }, + { + "epoch": 18.825515191212777, + "grad_norm": 0.8000630736351013, + "learning_rate": 0.0001, + "loss": 62.8883, + "step": 11690 + }, + { + "epoch": 18.84163853479115, + "grad_norm": 0.8450509309768677, + "learning_rate": 0.0001, + "loss": 63.4962, + "step": 11700 + }, + { + "epoch": 18.857761878369526, + "grad_norm": 0.8365609645843506, + "learning_rate": 0.0001, + "loss": 62.2701, + "step": 11710 + }, + { + "epoch": 18.8738852219479, + "grad_norm": 0.7197144031524658, + "learning_rate": 0.0001, + "loss": 61.3158, + "step": 11720 + }, + { + "epoch": 18.890008565526276, + "grad_norm": 0.6460379958152771, + "learning_rate": 0.0001, + "loss": 62.6645, + "step": 11730 + }, + { + "epoch": 18.90613190910465, + "grad_norm": 0.8203967213630676, + "learning_rate": 0.0001, + "loss": 62.3257, + "step": 11740 + }, + { + "epoch": 18.922255252683026, + "grad_norm": 0.682134211063385, + "learning_rate": 0.0001, + "loss": 59.9144, + "step": 11750 + }, + { + "epoch": 18.9383785962614, + "grad_norm": 0.7080821394920349, + "learning_rate": 0.0001, + "loss": 61.4325, + "step": 11760 + }, + { + "epoch": 18.954501939839773, + "grad_norm": 0.7963712811470032, + "learning_rate": 0.0001, + "loss": 58.6136, + "step": 11770 + }, + { + "epoch": 18.970625283418148, + "grad_norm": 0.7317046523094177, + "learning_rate": 0.0001, + "loss": 61.3507, + "step": 11780 + }, + { + "epoch": 18.986748626996523, + "grad_norm": 0.6586548686027527, + "learning_rate": 0.0001, + "loss": 61.4898, + "step": 11790 + }, + { + "epoch": 19.001612334357837, + "grad_norm": 0.9010424613952637, + "learning_rate": 0.0001, + "loss": 56.8915, + "step": 11800 + }, + { + "epoch": 19.017735677936212, + "grad_norm": 0.7102377414703369, + "learning_rate": 0.0001, + "loss": 62.2097, + "step": 11810 + }, + { + "epoch": 19.033859021514587, + "grad_norm": 0.7959222793579102, + "learning_rate": 0.0001, + "loss": 61.4484, + "step": 11820 + }, + { + "epoch": 19.049982365092962, + "grad_norm": 0.6483829617500305, + "learning_rate": 0.0001, + "loss": 60.3563, + "step": 11830 + }, + { + "epoch": 19.066105708671337, + "grad_norm": 1.1301988363265991, + "learning_rate": 0.0001, + "loss": 61.3698, + "step": 11840 + }, + { + "epoch": 19.082229052249712, + "grad_norm": 0.7304167151451111, + "learning_rate": 0.0001, + "loss": 59.2096, + "step": 11850 + }, + { + "epoch": 19.098352395828083, + "grad_norm": 0.7757825255393982, + "learning_rate": 0.0001, + "loss": 59.914, + "step": 11860 + }, + { + "epoch": 19.11447573940646, + "grad_norm": 0.7009251713752747, + "learning_rate": 0.0001, + "loss": 61.0768, + "step": 11870 + }, + { + "epoch": 19.130599082984833, + "grad_norm": 0.6015574336051941, + "learning_rate": 0.0001, + "loss": 61.2698, + "step": 11880 + }, + { + "epoch": 19.14672242656321, + "grad_norm": 0.8028033971786499, + "learning_rate": 0.0001, + "loss": 61.422, + "step": 11890 + }, + { + "epoch": 19.162845770141583, + "grad_norm": 0.5753505825996399, + "learning_rate": 0.0001, + "loss": 61.5066, + "step": 11900 + }, + { + "epoch": 19.17896911371996, + "grad_norm": 0.7702978849411011, + "learning_rate": 0.0001, + "loss": 60.0737, + "step": 11910 + }, + { + "epoch": 19.195092457298333, + "grad_norm": 0.6469365358352661, + "learning_rate": 0.0001, + "loss": 59.5091, + "step": 11920 + }, + { + "epoch": 19.211215800876708, + "grad_norm": 0.8400216698646545, + "learning_rate": 0.0001, + "loss": 62.3739, + "step": 11930 + }, + { + "epoch": 19.22733914445508, + "grad_norm": 1.9919443130493164, + "learning_rate": 0.0001, + "loss": 59.7271, + "step": 11940 + }, + { + "epoch": 19.243462488033455, + "grad_norm": 1.0247725248336792, + "learning_rate": 0.0001, + "loss": 59.3804, + "step": 11950 + }, + { + "epoch": 19.25958583161183, + "grad_norm": 1.0717105865478516, + "learning_rate": 0.0001, + "loss": 61.3617, + "step": 11960 + }, + { + "epoch": 19.275709175190205, + "grad_norm": 0.6324111223220825, + "learning_rate": 0.0001, + "loss": 59.6094, + "step": 11970 + }, + { + "epoch": 19.29183251876858, + "grad_norm": 0.6665195226669312, + "learning_rate": 0.0001, + "loss": 61.5879, + "step": 11980 + }, + { + "epoch": 19.307955862346954, + "grad_norm": 0.7933892011642456, + "learning_rate": 0.0001, + "loss": 60.4011, + "step": 11990 + }, + { + "epoch": 19.32407920592533, + "grad_norm": 0.8135427236557007, + "learning_rate": 0.0001, + "loss": 61.2936, + "step": 12000 + }, + { + "epoch": 19.340202549503704, + "grad_norm": 0.7154449820518494, + "learning_rate": 0.0001, + "loss": 62.3996, + "step": 12010 + }, + { + "epoch": 19.35632589308208, + "grad_norm": 0.9926077127456665, + "learning_rate": 0.0001, + "loss": 60.5219, + "step": 12020 + }, + { + "epoch": 19.37244923666045, + "grad_norm": 1.013094425201416, + "learning_rate": 0.0001, + "loss": 59.9494, + "step": 12030 + }, + { + "epoch": 19.388572580238826, + "grad_norm": 0.984582245349884, + "learning_rate": 0.0001, + "loss": 59.7877, + "step": 12040 + }, + { + "epoch": 19.4046959238172, + "grad_norm": 1.256519079208374, + "learning_rate": 0.0001, + "loss": 60.2608, + "step": 12050 + }, + { + "epoch": 19.420819267395576, + "grad_norm": 0.7013972997665405, + "learning_rate": 0.0001, + "loss": 63.4573, + "step": 12060 + }, + { + "epoch": 19.43694261097395, + "grad_norm": 0.9027557373046875, + "learning_rate": 0.0001, + "loss": 61.3252, + "step": 12070 + }, + { + "epoch": 19.453065954552326, + "grad_norm": 0.9057058691978455, + "learning_rate": 0.0001, + "loss": 60.4992, + "step": 12080 + }, + { + "epoch": 19.4691892981307, + "grad_norm": 0.6828131079673767, + "learning_rate": 0.0001, + "loss": 61.5369, + "step": 12090 + }, + { + "epoch": 19.485312641709076, + "grad_norm": 0.617858350276947, + "learning_rate": 0.0001, + "loss": 61.4837, + "step": 12100 + }, + { + "epoch": 19.50143598528745, + "grad_norm": 0.7722935676574707, + "learning_rate": 0.0001, + "loss": 61.0886, + "step": 12110 + }, + { + "epoch": 19.517559328865822, + "grad_norm": 0.7039631009101868, + "learning_rate": 0.0001, + "loss": 59.9055, + "step": 12120 + }, + { + "epoch": 19.533682672444197, + "grad_norm": 0.8353239297866821, + "learning_rate": 0.0001, + "loss": 62.0864, + "step": 12130 + }, + { + "epoch": 19.549806016022572, + "grad_norm": 0.764098048210144, + "learning_rate": 0.0001, + "loss": 58.9591, + "step": 12140 + }, + { + "epoch": 19.565929359600947, + "grad_norm": 0.7842984795570374, + "learning_rate": 0.0001, + "loss": 59.0494, + "step": 12150 + }, + { + "epoch": 19.582052703179322, + "grad_norm": 0.7140147686004639, + "learning_rate": 0.0001, + "loss": 60.8067, + "step": 12160 + }, + { + "epoch": 19.598176046757697, + "grad_norm": 0.8182031512260437, + "learning_rate": 0.0001, + "loss": 59.6013, + "step": 12170 + }, + { + "epoch": 19.614299390336072, + "grad_norm": 0.7428567409515381, + "learning_rate": 0.0001, + "loss": 61.8644, + "step": 12180 + }, + { + "epoch": 19.630422733914447, + "grad_norm": 0.6174675822257996, + "learning_rate": 0.0001, + "loss": 60.2086, + "step": 12190 + }, + { + "epoch": 19.646546077492822, + "grad_norm": 0.6416913866996765, + "learning_rate": 0.0001, + "loss": 61.3452, + "step": 12200 + }, + { + "epoch": 19.662669421071193, + "grad_norm": 1.3036755323410034, + "learning_rate": 0.0001, + "loss": 60.5332, + "step": 12210 + }, + { + "epoch": 19.678792764649568, + "grad_norm": 1.230710506439209, + "learning_rate": 0.0001, + "loss": 62.5671, + "step": 12220 + }, + { + "epoch": 19.694916108227943, + "grad_norm": 0.6914290189743042, + "learning_rate": 0.0001, + "loss": 60.7302, + "step": 12230 + }, + { + "epoch": 19.711039451806318, + "grad_norm": 0.6492108106613159, + "learning_rate": 0.0001, + "loss": 60.6465, + "step": 12240 + }, + { + "epoch": 19.727162795384693, + "grad_norm": 0.7104936838150024, + "learning_rate": 0.0001, + "loss": 61.14, + "step": 12250 + }, + { + "epoch": 19.743286138963068, + "grad_norm": 1.0740787982940674, + "learning_rate": 0.0001, + "loss": 60.1977, + "step": 12260 + }, + { + "epoch": 19.759409482541443, + "grad_norm": 0.5764663815498352, + "learning_rate": 0.0001, + "loss": 59.994, + "step": 12270 + }, + { + "epoch": 19.775532826119818, + "grad_norm": 0.6204820871353149, + "learning_rate": 0.0001, + "loss": 60.3091, + "step": 12280 + }, + { + "epoch": 19.791656169698193, + "grad_norm": 0.702944815158844, + "learning_rate": 0.0001, + "loss": 60.8637, + "step": 12290 + }, + { + "epoch": 19.807779513276564, + "grad_norm": 0.7591354250907898, + "learning_rate": 0.0001, + "loss": 59.54, + "step": 12300 + }, + { + "epoch": 19.82390285685494, + "grad_norm": 0.755224883556366, + "learning_rate": 0.0001, + "loss": 60.0285, + "step": 12310 + }, + { + "epoch": 19.840026200433314, + "grad_norm": 0.7563093900680542, + "learning_rate": 0.0001, + "loss": 60.2669, + "step": 12320 + }, + { + "epoch": 19.85614954401169, + "grad_norm": 0.8339024782180786, + "learning_rate": 0.0001, + "loss": 60.606, + "step": 12330 + }, + { + "epoch": 19.872272887590064, + "grad_norm": 0.7017751932144165, + "learning_rate": 0.0001, + "loss": 59.579, + "step": 12340 + }, + { + "epoch": 19.88839623116844, + "grad_norm": 1.3561909198760986, + "learning_rate": 0.0001, + "loss": 59.7727, + "step": 12350 + }, + { + "epoch": 19.904519574746814, + "grad_norm": 0.727020263671875, + "learning_rate": 0.0001, + "loss": 60.6587, + "step": 12360 + }, + { + "epoch": 19.92064291832519, + "grad_norm": 0.6882230043411255, + "learning_rate": 0.0001, + "loss": 60.3916, + "step": 12370 + }, + { + "epoch": 19.93676626190356, + "grad_norm": 0.6410917639732361, + "learning_rate": 0.0001, + "loss": 61.6649, + "step": 12380 + }, + { + "epoch": 19.952889605481936, + "grad_norm": 0.9747995138168335, + "learning_rate": 0.0001, + "loss": 60.4622, + "step": 12390 + }, + { + "epoch": 19.96901294906031, + "grad_norm": 1.260621190071106, + "learning_rate": 0.0001, + "loss": 60.4228, + "step": 12400 + }, + { + "epoch": 19.985136292638686, + "grad_norm": 0.6525571942329407, + "learning_rate": 0.0001, + "loss": 59.0612, + "step": 12410 + }, + { + "epoch": 20.0, + "grad_norm": 0.7864645719528198, + "learning_rate": 0.0001, + "loss": 57.2347, + "step": 12420 + }, + { + "epoch": 20.016123343578375, + "grad_norm": 0.7479318380355835, + "learning_rate": 0.0001, + "loss": 60.9263, + "step": 12430 + }, + { + "epoch": 20.03224668715675, + "grad_norm": 0.6433089375495911, + "learning_rate": 0.0001, + "loss": 59.8846, + "step": 12440 + }, + { + "epoch": 20.048370030735125, + "grad_norm": 0.7594776153564453, + "learning_rate": 0.0001, + "loss": 61.7793, + "step": 12450 + }, + { + "epoch": 20.0644933743135, + "grad_norm": 0.8710535764694214, + "learning_rate": 0.0001, + "loss": 60.5331, + "step": 12460 + }, + { + "epoch": 20.08061671789187, + "grad_norm": 0.7302114963531494, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 12470 + }, + { + "epoch": 20.096740061470246, + "grad_norm": 0.6039112210273743, + "learning_rate": 0.0001, + "loss": 60.8272, + "step": 12480 + }, + { + "epoch": 20.11286340504862, + "grad_norm": 0.7339144945144653, + "learning_rate": 0.0001, + "loss": 61.4149, + "step": 12490 + }, + { + "epoch": 20.128986748626996, + "grad_norm": 0.7278878092765808, + "learning_rate": 0.0001, + "loss": 62.631, + "step": 12500 + }, + { + "epoch": 20.14511009220537, + "grad_norm": 0.7241142988204956, + "learning_rate": 0.0001, + "loss": 59.9328, + "step": 12510 + }, + { + "epoch": 20.161233435783746, + "grad_norm": 0.6967542767524719, + "learning_rate": 0.0001, + "loss": 58.7463, + "step": 12520 + }, + { + "epoch": 20.17735677936212, + "grad_norm": 0.7326555848121643, + "learning_rate": 0.0001, + "loss": 59.7082, + "step": 12530 + }, + { + "epoch": 20.193480122940496, + "grad_norm": 0.6706270575523376, + "learning_rate": 0.0001, + "loss": 60.5978, + "step": 12540 + }, + { + "epoch": 20.20960346651887, + "grad_norm": 0.8541543483734131, + "learning_rate": 0.0001, + "loss": 58.9632, + "step": 12550 + }, + { + "epoch": 20.225726810097242, + "grad_norm": 0.6599560379981995, + "learning_rate": 0.0001, + "loss": 58.4572, + "step": 12560 + }, + { + "epoch": 20.241850153675617, + "grad_norm": 0.6668654680252075, + "learning_rate": 0.0001, + "loss": 61.8741, + "step": 12570 + }, + { + "epoch": 20.257973497253992, + "grad_norm": 0.6453808546066284, + "learning_rate": 0.0001, + "loss": 60.2538, + "step": 12580 + }, + { + "epoch": 20.274096840832367, + "grad_norm": 0.7429870963096619, + "learning_rate": 0.0001, + "loss": 60.5795, + "step": 12590 + }, + { + "epoch": 20.290220184410742, + "grad_norm": 0.7588251829147339, + "learning_rate": 0.0001, + "loss": 60.6261, + "step": 12600 + }, + { + "epoch": 20.306343527989117, + "grad_norm": 0.6430977582931519, + "learning_rate": 0.0001, + "loss": 61.0344, + "step": 12610 + }, + { + "epoch": 20.322466871567492, + "grad_norm": 0.6879712343215942, + "learning_rate": 0.0001, + "loss": 62.2668, + "step": 12620 + }, + { + "epoch": 20.338590215145867, + "grad_norm": 0.7650917172431946, + "learning_rate": 0.0001, + "loss": 60.3852, + "step": 12630 + }, + { + "epoch": 20.35471355872424, + "grad_norm": 0.5326449871063232, + "learning_rate": 0.0001, + "loss": 60.6138, + "step": 12640 + }, + { + "epoch": 20.370836902302614, + "grad_norm": 0.7018648982048035, + "learning_rate": 0.0001, + "loss": 62.1081, + "step": 12650 + }, + { + "epoch": 20.38696024588099, + "grad_norm": 0.7988570332527161, + "learning_rate": 0.0001, + "loss": 61.2661, + "step": 12660 + }, + { + "epoch": 20.403083589459364, + "grad_norm": 0.6800012588500977, + "learning_rate": 0.0001, + "loss": 59.9279, + "step": 12670 + }, + { + "epoch": 20.41920693303774, + "grad_norm": 0.6380598545074463, + "learning_rate": 0.0001, + "loss": 60.4877, + "step": 12680 + }, + { + "epoch": 20.435330276616114, + "grad_norm": 0.6755974888801575, + "learning_rate": 0.0001, + "loss": 60.9729, + "step": 12690 + }, + { + "epoch": 20.45145362019449, + "grad_norm": 0.8057810664176941, + "learning_rate": 0.0001, + "loss": 59.6747, + "step": 12700 + }, + { + "epoch": 20.467576963772864, + "grad_norm": 0.6942827105522156, + "learning_rate": 0.0001, + "loss": 60.7314, + "step": 12710 + }, + { + "epoch": 20.48370030735124, + "grad_norm": 1.107717752456665, + "learning_rate": 0.0001, + "loss": 59.1397, + "step": 12720 + }, + { + "epoch": 20.49982365092961, + "grad_norm": 2.6167991161346436, + "learning_rate": 0.0001, + "loss": 60.5604, + "step": 12730 + }, + { + "epoch": 20.515946994507985, + "grad_norm": 0.6041843891143799, + "learning_rate": 0.0001, + "loss": 60.6188, + "step": 12740 + }, + { + "epoch": 20.53207033808636, + "grad_norm": 0.5643273591995239, + "learning_rate": 0.0001, + "loss": 59.3172, + "step": 12750 + }, + { + "epoch": 20.548193681664735, + "grad_norm": 0.7615086436271667, + "learning_rate": 0.0001, + "loss": 60.9263, + "step": 12760 + }, + { + "epoch": 20.56431702524311, + "grad_norm": 0.692243754863739, + "learning_rate": 0.0001, + "loss": 58.3092, + "step": 12770 + }, + { + "epoch": 20.580440368821485, + "grad_norm": 0.6447681784629822, + "learning_rate": 0.0001, + "loss": 58.0169, + "step": 12780 + }, + { + "epoch": 20.59656371239986, + "grad_norm": 0.7043688297271729, + "learning_rate": 0.0001, + "loss": 58.8346, + "step": 12790 + }, + { + "epoch": 20.612687055978235, + "grad_norm": 0.8017414808273315, + "learning_rate": 0.0001, + "loss": 60.7451, + "step": 12800 + }, + { + "epoch": 20.62881039955661, + "grad_norm": 0.8983294367790222, + "learning_rate": 0.0001, + "loss": 59.6858, + "step": 12810 + }, + { + "epoch": 20.64493374313498, + "grad_norm": 1.0817819833755493, + "learning_rate": 0.0001, + "loss": 60.8232, + "step": 12820 + }, + { + "epoch": 20.661057086713356, + "grad_norm": 0.5739386081695557, + "learning_rate": 0.0001, + "loss": 60.7852, + "step": 12830 + }, + { + "epoch": 20.67718043029173, + "grad_norm": 1.007978916168213, + "learning_rate": 0.0001, + "loss": 61.7002, + "step": 12840 + }, + { + "epoch": 20.693303773870106, + "grad_norm": 0.7019670009613037, + "learning_rate": 0.0001, + "loss": 59.7211, + "step": 12850 + }, + { + "epoch": 20.70942711744848, + "grad_norm": 0.7891802191734314, + "learning_rate": 0.0001, + "loss": 61.4963, + "step": 12860 + }, + { + "epoch": 20.725550461026856, + "grad_norm": 0.6955318450927734, + "learning_rate": 0.0001, + "loss": 60.8528, + "step": 12870 + }, + { + "epoch": 20.74167380460523, + "grad_norm": 1.9988878965377808, + "learning_rate": 0.0001, + "loss": 58.7444, + "step": 12880 + }, + { + "epoch": 20.757797148183606, + "grad_norm": 0.7859933972358704, + "learning_rate": 0.0001, + "loss": 60.0696, + "step": 12890 + }, + { + "epoch": 20.77392049176198, + "grad_norm": 0.8253535628318787, + "learning_rate": 0.0001, + "loss": 61.3408, + "step": 12900 + }, + { + "epoch": 20.790043835340352, + "grad_norm": 0.6639575362205505, + "learning_rate": 0.0001, + "loss": 58.3044, + "step": 12910 + }, + { + "epoch": 20.806167178918727, + "grad_norm": 0.6754361391067505, + "learning_rate": 0.0001, + "loss": 59.8301, + "step": 12920 + }, + { + "epoch": 20.822290522497102, + "grad_norm": 0.7306725382804871, + "learning_rate": 0.0001, + "loss": 60.0021, + "step": 12930 + }, + { + "epoch": 20.838413866075477, + "grad_norm": 6.799670696258545, + "learning_rate": 0.0001, + "loss": 57.7934, + "step": 12940 + }, + { + "epoch": 20.854537209653852, + "grad_norm": 0.6870545744895935, + "learning_rate": 0.0001, + "loss": 59.2313, + "step": 12950 + }, + { + "epoch": 20.870660553232227, + "grad_norm": 0.6980645060539246, + "learning_rate": 0.0001, + "loss": 60.1903, + "step": 12960 + }, + { + "epoch": 20.886783896810602, + "grad_norm": 0.7006897926330566, + "learning_rate": 0.0001, + "loss": 60.9525, + "step": 12970 + }, + { + "epoch": 20.902907240388977, + "grad_norm": 0.7185907363891602, + "learning_rate": 0.0001, + "loss": 61.8833, + "step": 12980 + }, + { + "epoch": 20.91903058396735, + "grad_norm": 0.6029082536697388, + "learning_rate": 0.0001, + "loss": 60.3764, + "step": 12990 + }, + { + "epoch": 20.935153927545723, + "grad_norm": 0.8502999544143677, + "learning_rate": 0.0001, + "loss": 59.0051, + "step": 13000 + }, + { + "epoch": 20.9512772711241, + "grad_norm": 0.908899188041687, + "learning_rate": 0.0001, + "loss": 59.0677, + "step": 13010 + }, + { + "epoch": 20.967400614702473, + "grad_norm": 0.6983920335769653, + "learning_rate": 0.0001, + "loss": 60.8523, + "step": 13020 + }, + { + "epoch": 20.98352395828085, + "grad_norm": 0.9079043865203857, + "learning_rate": 0.0001, + "loss": 58.8014, + "step": 13030 + }, + { + "epoch": 20.999647301859223, + "grad_norm": 0.7139968872070312, + "learning_rate": 0.0001, + "loss": 59.3019, + "step": 13040 + }, + { + "epoch": 21.014511009220538, + "grad_norm": 0.7472447156906128, + "learning_rate": 0.0001, + "loss": 56.1178, + "step": 13050 + }, + { + "epoch": 21.030634352798913, + "grad_norm": 0.5645695328712463, + "learning_rate": 0.0001, + "loss": 60.5944, + "step": 13060 + }, + { + "epoch": 21.046757696377288, + "grad_norm": 0.6419529318809509, + "learning_rate": 0.0001, + "loss": 59.5792, + "step": 13070 + }, + { + "epoch": 21.06288103995566, + "grad_norm": 0.6508122682571411, + "learning_rate": 0.0001, + "loss": 58.995, + "step": 13080 + }, + { + "epoch": 21.079004383534034, + "grad_norm": 0.6605037450790405, + "learning_rate": 0.0001, + "loss": 60.9783, + "step": 13090 + }, + { + "epoch": 21.09512772711241, + "grad_norm": 0.6636376976966858, + "learning_rate": 0.0001, + "loss": 60.3516, + "step": 13100 + }, + { + "epoch": 21.111251070690784, + "grad_norm": 0.7671985030174255, + "learning_rate": 0.0001, + "loss": 61.8918, + "step": 13110 + }, + { + "epoch": 21.12737441426916, + "grad_norm": 0.859834611415863, + "learning_rate": 0.0001, + "loss": 61.0766, + "step": 13120 + }, + { + "epoch": 21.143497757847534, + "grad_norm": 0.7551945447921753, + "learning_rate": 0.0001, + "loss": 60.6674, + "step": 13130 + }, + { + "epoch": 21.15962110142591, + "grad_norm": 0.6225672960281372, + "learning_rate": 0.0001, + "loss": 60.4107, + "step": 13140 + }, + { + "epoch": 21.175744445004284, + "grad_norm": 0.6100388765335083, + "learning_rate": 0.0001, + "loss": 58.8803, + "step": 13150 + }, + { + "epoch": 21.19186778858266, + "grad_norm": 0.669247567653656, + "learning_rate": 0.0001, + "loss": 60.4534, + "step": 13160 + }, + { + "epoch": 21.20799113216103, + "grad_norm": 0.8899680972099304, + "learning_rate": 0.0001, + "loss": 59.3081, + "step": 13170 + }, + { + "epoch": 21.224114475739405, + "grad_norm": 0.6804965138435364, + "learning_rate": 0.0001, + "loss": 58.1406, + "step": 13180 + }, + { + "epoch": 21.24023781931778, + "grad_norm": 0.7442401051521301, + "learning_rate": 0.0001, + "loss": 61.9136, + "step": 13190 + }, + { + "epoch": 21.256361162896155, + "grad_norm": 0.6071797013282776, + "learning_rate": 0.0001, + "loss": 59.7979, + "step": 13200 + }, + { + "epoch": 21.27248450647453, + "grad_norm": 0.6128804683685303, + "learning_rate": 0.0001, + "loss": 58.802, + "step": 13210 + }, + { + "epoch": 21.288607850052905, + "grad_norm": 0.6430484652519226, + "learning_rate": 0.0001, + "loss": 59.9492, + "step": 13220 + }, + { + "epoch": 21.30473119363128, + "grad_norm": 0.6414241194725037, + "learning_rate": 0.0001, + "loss": 60.699, + "step": 13230 + }, + { + "epoch": 21.320854537209655, + "grad_norm": 0.7222250699996948, + "learning_rate": 0.0001, + "loss": 58.5909, + "step": 13240 + }, + { + "epoch": 21.33697788078803, + "grad_norm": 1.0913405418395996, + "learning_rate": 0.0001, + "loss": 59.9657, + "step": 13250 + }, + { + "epoch": 21.3531012243664, + "grad_norm": 0.8245854377746582, + "learning_rate": 0.0001, + "loss": 60.2276, + "step": 13260 + }, + { + "epoch": 21.369224567944777, + "grad_norm": 0.7204747200012207, + "learning_rate": 0.0001, + "loss": 60.1854, + "step": 13270 + }, + { + "epoch": 21.38534791152315, + "grad_norm": 0.5877585411071777, + "learning_rate": 0.0001, + "loss": 60.8509, + "step": 13280 + }, + { + "epoch": 21.401471255101526, + "grad_norm": 0.906605064868927, + "learning_rate": 0.0001, + "loss": 57.9792, + "step": 13290 + }, + { + "epoch": 21.4175945986799, + "grad_norm": 0.910301923751831, + "learning_rate": 0.0001, + "loss": 58.986, + "step": 13300 + }, + { + "epoch": 21.433717942258276, + "grad_norm": 0.7098278403282166, + "learning_rate": 0.0001, + "loss": 60.6727, + "step": 13310 + }, + { + "epoch": 21.44984128583665, + "grad_norm": 0.842569887638092, + "learning_rate": 0.0001, + "loss": 59.3789, + "step": 13320 + }, + { + "epoch": 21.465964629415026, + "grad_norm": 0.7320014834403992, + "learning_rate": 0.0001, + "loss": 60.7055, + "step": 13330 + }, + { + "epoch": 21.482087972993398, + "grad_norm": 0.7192698121070862, + "learning_rate": 0.0001, + "loss": 59.7494, + "step": 13340 + }, + { + "epoch": 21.498211316571773, + "grad_norm": 0.6317388415336609, + "learning_rate": 0.0001, + "loss": 59.2083, + "step": 13350 + }, + { + "epoch": 21.514334660150148, + "grad_norm": 0.7012689709663391, + "learning_rate": 0.0001, + "loss": 58.6733, + "step": 13360 + }, + { + "epoch": 21.530458003728523, + "grad_norm": 0.8148412704467773, + "learning_rate": 0.0001, + "loss": 60.6708, + "step": 13370 + }, + { + "epoch": 21.546581347306898, + "grad_norm": 0.6740885376930237, + "learning_rate": 0.0001, + "loss": 60.2317, + "step": 13380 + }, + { + "epoch": 21.562704690885273, + "grad_norm": 0.6835291981697083, + "learning_rate": 0.0001, + "loss": 60.1807, + "step": 13390 + }, + { + "epoch": 21.578828034463648, + "grad_norm": 0.9604494571685791, + "learning_rate": 0.0001, + "loss": 60.811, + "step": 13400 + }, + { + "epoch": 21.594951378042023, + "grad_norm": 0.765917956829071, + "learning_rate": 0.0001, + "loss": 60.4539, + "step": 13410 + }, + { + "epoch": 21.611074721620398, + "grad_norm": 0.7830843329429626, + "learning_rate": 0.0001, + "loss": 59.2483, + "step": 13420 + }, + { + "epoch": 21.62719806519877, + "grad_norm": 0.6538785099983215, + "learning_rate": 0.0001, + "loss": 60.4062, + "step": 13430 + }, + { + "epoch": 21.643321408777144, + "grad_norm": 0.7341951727867126, + "learning_rate": 0.0001, + "loss": 59.3654, + "step": 13440 + }, + { + "epoch": 21.65944475235552, + "grad_norm": 0.684467077255249, + "learning_rate": 0.0001, + "loss": 58.9634, + "step": 13450 + }, + { + "epoch": 21.675568095933894, + "grad_norm": 0.6942781805992126, + "learning_rate": 0.0001, + "loss": 60.3677, + "step": 13460 + }, + { + "epoch": 21.69169143951227, + "grad_norm": 0.7780037522315979, + "learning_rate": 0.0001, + "loss": 61.6705, + "step": 13470 + }, + { + "epoch": 21.707814783090644, + "grad_norm": 0.8363171219825745, + "learning_rate": 0.0001, + "loss": 59.7802, + "step": 13480 + }, + { + "epoch": 21.72393812666902, + "grad_norm": 0.7392274737358093, + "learning_rate": 0.0001, + "loss": 59.8264, + "step": 13490 + }, + { + "epoch": 21.740061470247394, + "grad_norm": 0.6051466464996338, + "learning_rate": 0.0001, + "loss": 59.6685, + "step": 13500 + }, + { + "epoch": 21.75618481382577, + "grad_norm": 0.7772730588912964, + "learning_rate": 0.0001, + "loss": 58.261, + "step": 13510 + }, + { + "epoch": 21.77230815740414, + "grad_norm": 0.7481459379196167, + "learning_rate": 0.0001, + "loss": 59.3212, + "step": 13520 + }, + { + "epoch": 21.788431500982515, + "grad_norm": 0.8020729422569275, + "learning_rate": 0.0001, + "loss": 59.0113, + "step": 13530 + }, + { + "epoch": 21.80455484456089, + "grad_norm": 0.8504058122634888, + "learning_rate": 0.0001, + "loss": 59.7243, + "step": 13540 + }, + { + "epoch": 21.820678188139265, + "grad_norm": 0.7256322503089905, + "learning_rate": 0.0001, + "loss": 60.7478, + "step": 13550 + }, + { + "epoch": 21.83680153171764, + "grad_norm": 0.6632773876190186, + "learning_rate": 0.0001, + "loss": 59.2331, + "step": 13560 + }, + { + "epoch": 21.852924875296015, + "grad_norm": 0.6225188970565796, + "learning_rate": 0.0001, + "loss": 59.8263, + "step": 13570 + }, + { + "epoch": 21.86904821887439, + "grad_norm": 0.6302409768104553, + "learning_rate": 0.0001, + "loss": 60.8574, + "step": 13580 + }, + { + "epoch": 21.885171562452765, + "grad_norm": 0.7654368877410889, + "learning_rate": 0.0001, + "loss": 59.8633, + "step": 13590 + }, + { + "epoch": 21.90129490603114, + "grad_norm": 1.227220892906189, + "learning_rate": 0.0001, + "loss": 60.7098, + "step": 13600 + }, + { + "epoch": 21.91741824960951, + "grad_norm": 0.6679421067237854, + "learning_rate": 0.0001, + "loss": 59.0663, + "step": 13610 + }, + { + "epoch": 21.933541593187886, + "grad_norm": 0.8533850312232971, + "learning_rate": 0.0001, + "loss": 60.8597, + "step": 13620 + }, + { + "epoch": 21.94966493676626, + "grad_norm": 0.6417121887207031, + "learning_rate": 0.0001, + "loss": 60.1464, + "step": 13630 + }, + { + "epoch": 21.965788280344636, + "grad_norm": 0.679053544998169, + "learning_rate": 0.0001, + "loss": 60.5066, + "step": 13640 + }, + { + "epoch": 21.98191162392301, + "grad_norm": 0.6914607286453247, + "learning_rate": 0.0001, + "loss": 59.6815, + "step": 13650 + }, + { + "epoch": 21.998034967501386, + "grad_norm": 0.5661413669586182, + "learning_rate": 0.0001, + "loss": 61.6076, + "step": 13660 + }, + { + "epoch": 22.0128986748627, + "grad_norm": 1.4387016296386719, + "learning_rate": 0.0001, + "loss": 55.0652, + "step": 13670 + }, + { + "epoch": 22.029022018441076, + "grad_norm": 0.7774295210838318, + "learning_rate": 0.0001, + "loss": 61.4395, + "step": 13680 + }, + { + "epoch": 22.045145362019447, + "grad_norm": 0.7438349723815918, + "learning_rate": 0.0001, + "loss": 60.998, + "step": 13690 + }, + { + "epoch": 22.061268705597822, + "grad_norm": 0.7506164312362671, + "learning_rate": 0.0001, + "loss": 60.6929, + "step": 13700 + }, + { + "epoch": 22.077392049176197, + "grad_norm": 0.6262985467910767, + "learning_rate": 0.0001, + "loss": 59.4921, + "step": 13710 + }, + { + "epoch": 22.093515392754572, + "grad_norm": 1.826015591621399, + "learning_rate": 0.0001, + "loss": 61.3791, + "step": 13720 + }, + { + "epoch": 22.109638736332947, + "grad_norm": 0.6410622596740723, + "learning_rate": 0.0001, + "loss": 62.5485, + "step": 13730 + }, + { + "epoch": 22.125762079911322, + "grad_norm": 0.7042442560195923, + "learning_rate": 0.0001, + "loss": 60.0654, + "step": 13740 + }, + { + "epoch": 22.141885423489697, + "grad_norm": 0.741142988204956, + "learning_rate": 0.0001, + "loss": 59.9831, + "step": 13750 + }, + { + "epoch": 22.158008767068072, + "grad_norm": 0.836445152759552, + "learning_rate": 0.0001, + "loss": 59.6805, + "step": 13760 + }, + { + "epoch": 22.174132110646447, + "grad_norm": 0.8353415727615356, + "learning_rate": 0.0001, + "loss": 60.1912, + "step": 13770 + }, + { + "epoch": 22.19025545422482, + "grad_norm": 0.6416126489639282, + "learning_rate": 0.0001, + "loss": 59.5518, + "step": 13780 + }, + { + "epoch": 22.206378797803193, + "grad_norm": 0.8702544569969177, + "learning_rate": 0.0001, + "loss": 57.7428, + "step": 13790 + }, + { + "epoch": 22.222502141381568, + "grad_norm": 0.6681593656539917, + "learning_rate": 0.0001, + "loss": 59.9499, + "step": 13800 + }, + { + "epoch": 22.238625484959943, + "grad_norm": 0.6185410022735596, + "learning_rate": 0.0001, + "loss": 59.8062, + "step": 13810 + }, + { + "epoch": 22.254748828538318, + "grad_norm": 0.7695328593254089, + "learning_rate": 0.0001, + "loss": 61.3512, + "step": 13820 + }, + { + "epoch": 22.270872172116693, + "grad_norm": 0.6550581455230713, + "learning_rate": 0.0001, + "loss": 59.3379, + "step": 13830 + }, + { + "epoch": 22.286995515695068, + "grad_norm": 0.7764340043067932, + "learning_rate": 0.0001, + "loss": 59.984, + "step": 13840 + }, + { + "epoch": 22.303118859273443, + "grad_norm": 0.6377913355827332, + "learning_rate": 0.0001, + "loss": 59.1862, + "step": 13850 + }, + { + "epoch": 22.319242202851818, + "grad_norm": 0.7837334275245667, + "learning_rate": 0.0001, + "loss": 60.8087, + "step": 13860 + }, + { + "epoch": 22.33536554643019, + "grad_norm": 0.7966547012329102, + "learning_rate": 0.0001, + "loss": 59.9109, + "step": 13870 + }, + { + "epoch": 22.351488890008564, + "grad_norm": 0.6303540468215942, + "learning_rate": 0.0001, + "loss": 61.0486, + "step": 13880 + }, + { + "epoch": 22.36761223358694, + "grad_norm": 0.6628289818763733, + "learning_rate": 0.0001, + "loss": 61.7723, + "step": 13890 + }, + { + "epoch": 22.383735577165314, + "grad_norm": 1.2156575918197632, + "learning_rate": 0.0001, + "loss": 58.0992, + "step": 13900 + }, + { + "epoch": 22.39985892074369, + "grad_norm": 1.0548591613769531, + "learning_rate": 0.0001, + "loss": 58.9139, + "step": 13910 + }, + { + "epoch": 22.415982264322064, + "grad_norm": 0.6389120817184448, + "learning_rate": 0.0001, + "loss": 59.1494, + "step": 13920 + }, + { + "epoch": 22.43210560790044, + "grad_norm": 0.6384537220001221, + "learning_rate": 0.0001, + "loss": 59.7135, + "step": 13930 + }, + { + "epoch": 22.448228951478814, + "grad_norm": 0.7434266209602356, + "learning_rate": 0.0001, + "loss": 60.7941, + "step": 13940 + }, + { + "epoch": 22.46435229505719, + "grad_norm": 0.7207310199737549, + "learning_rate": 0.0001, + "loss": 57.9466, + "step": 13950 + }, + { + "epoch": 22.48047563863556, + "grad_norm": 0.9680810570716858, + "learning_rate": 0.0001, + "loss": 57.5014, + "step": 13960 + }, + { + "epoch": 22.496598982213936, + "grad_norm": 0.5354718565940857, + "learning_rate": 0.0001, + "loss": 59.8771, + "step": 13970 + }, + { + "epoch": 22.51272232579231, + "grad_norm": 1.1624388694763184, + "learning_rate": 0.0001, + "loss": 60.2305, + "step": 13980 + }, + { + "epoch": 22.528845669370686, + "grad_norm": 0.6650974154472351, + "learning_rate": 0.0001, + "loss": 58.1649, + "step": 13990 + }, + { + "epoch": 22.54496901294906, + "grad_norm": 0.6151170134544373, + "learning_rate": 0.0001, + "loss": 60.4392, + "step": 14000 + }, + { + "epoch": 22.561092356527435, + "grad_norm": 0.6685694456100464, + "learning_rate": 0.0001, + "loss": 60.7055, + "step": 14010 + }, + { + "epoch": 22.57721570010581, + "grad_norm": 0.8092939257621765, + "learning_rate": 0.0001, + "loss": 57.8347, + "step": 14020 + }, + { + "epoch": 22.593339043684185, + "grad_norm": 0.7520709037780762, + "learning_rate": 0.0001, + "loss": 60.359, + "step": 14030 + }, + { + "epoch": 22.609462387262557, + "grad_norm": 0.8913780450820923, + "learning_rate": 0.0001, + "loss": 59.0833, + "step": 14040 + }, + { + "epoch": 22.625585730840932, + "grad_norm": 0.5508324503898621, + "learning_rate": 0.0001, + "loss": 60.8168, + "step": 14050 + }, + { + "epoch": 22.641709074419307, + "grad_norm": 0.7124258279800415, + "learning_rate": 0.0001, + "loss": 60.9433, + "step": 14060 + }, + { + "epoch": 22.65783241799768, + "grad_norm": 0.7286089062690735, + "learning_rate": 0.0001, + "loss": 59.8127, + "step": 14070 + }, + { + "epoch": 22.673955761576057, + "grad_norm": 0.8104858994483948, + "learning_rate": 0.0001, + "loss": 59.764, + "step": 14080 + }, + { + "epoch": 22.69007910515443, + "grad_norm": 0.7618247270584106, + "learning_rate": 0.0001, + "loss": 59.5563, + "step": 14090 + }, + { + "epoch": 22.706202448732807, + "grad_norm": 0.8512730598449707, + "learning_rate": 0.0001, + "loss": 59.3122, + "step": 14100 + }, + { + "epoch": 22.72232579231118, + "grad_norm": 0.6115924715995789, + "learning_rate": 0.0001, + "loss": 61.2874, + "step": 14110 + }, + { + "epoch": 22.738449135889557, + "grad_norm": 0.6883788704872131, + "learning_rate": 0.0001, + "loss": 59.427, + "step": 14120 + }, + { + "epoch": 22.754572479467928, + "grad_norm": 0.6240411400794983, + "learning_rate": 0.0001, + "loss": 58.9808, + "step": 14130 + }, + { + "epoch": 22.770695823046303, + "grad_norm": 0.6807436347007751, + "learning_rate": 0.0001, + "loss": 58.9609, + "step": 14140 + }, + { + "epoch": 22.786819166624678, + "grad_norm": 0.6661691665649414, + "learning_rate": 0.0001, + "loss": 59.8573, + "step": 14150 + }, + { + "epoch": 22.802942510203053, + "grad_norm": 0.7038732171058655, + "learning_rate": 0.0001, + "loss": 59.0114, + "step": 14160 + }, + { + "epoch": 22.819065853781428, + "grad_norm": 0.7736789584159851, + "learning_rate": 0.0001, + "loss": 59.6709, + "step": 14170 + }, + { + "epoch": 22.835189197359803, + "grad_norm": 0.6095895767211914, + "learning_rate": 0.0001, + "loss": 60.4468, + "step": 14180 + }, + { + "epoch": 22.851312540938178, + "grad_norm": 0.6531018018722534, + "learning_rate": 0.0001, + "loss": 61.4419, + "step": 14190 + }, + { + "epoch": 22.867435884516553, + "grad_norm": 0.7522090077400208, + "learning_rate": 0.0001, + "loss": 57.6574, + "step": 14200 + }, + { + "epoch": 22.883559228094928, + "grad_norm": 0.8028358817100525, + "learning_rate": 0.0001, + "loss": 59.0353, + "step": 14210 + }, + { + "epoch": 22.8996825716733, + "grad_norm": 0.7619246244430542, + "learning_rate": 0.0001, + "loss": 59.3586, + "step": 14220 + }, + { + "epoch": 22.915805915251674, + "grad_norm": 0.8312122821807861, + "learning_rate": 0.0001, + "loss": 60.8772, + "step": 14230 + }, + { + "epoch": 22.93192925883005, + "grad_norm": 0.6536790132522583, + "learning_rate": 0.0001, + "loss": 60.5941, + "step": 14240 + }, + { + "epoch": 22.948052602408424, + "grad_norm": 0.6973451972007751, + "learning_rate": 0.0001, + "loss": 59.5332, + "step": 14250 + }, + { + "epoch": 22.9641759459868, + "grad_norm": 0.6776084899902344, + "learning_rate": 0.0001, + "loss": 60.2337, + "step": 14260 + }, + { + "epoch": 22.980299289565174, + "grad_norm": 0.5728079080581665, + "learning_rate": 0.0001, + "loss": 61.0065, + "step": 14270 + }, + { + "epoch": 22.99642263314355, + "grad_norm": 0.798130452632904, + "learning_rate": 0.0001, + "loss": 59.7552, + "step": 14280 + }, + { + "epoch": 23.011286340504864, + "grad_norm": 0.6825193166732788, + "learning_rate": 0.0001, + "loss": 54.8593, + "step": 14290 + }, + { + "epoch": 23.02740968408324, + "grad_norm": 0.794450044631958, + "learning_rate": 0.0001, + "loss": 58.1472, + "step": 14300 + }, + { + "epoch": 23.04353302766161, + "grad_norm": 0.8178662657737732, + "learning_rate": 0.0001, + "loss": 58.8868, + "step": 14310 + }, + { + "epoch": 23.059656371239985, + "grad_norm": 0.6562325358390808, + "learning_rate": 0.0001, + "loss": 59.2406, + "step": 14320 + }, + { + "epoch": 23.07577971481836, + "grad_norm": 0.7282053828239441, + "learning_rate": 0.0001, + "loss": 58.1195, + "step": 14330 + }, + { + "epoch": 23.091903058396735, + "grad_norm": 0.6804936528205872, + "learning_rate": 0.0001, + "loss": 60.3755, + "step": 14340 + }, + { + "epoch": 23.10802640197511, + "grad_norm": 0.5851950645446777, + "learning_rate": 0.0001, + "loss": 59.2353, + "step": 14350 + }, + { + "epoch": 23.124149745553485, + "grad_norm": 0.5784706473350525, + "learning_rate": 0.0001, + "loss": 59.351, + "step": 14360 + }, + { + "epoch": 23.14027308913186, + "grad_norm": 0.7353758215904236, + "learning_rate": 0.0001, + "loss": 60.7708, + "step": 14370 + }, + { + "epoch": 23.156396432710235, + "grad_norm": 0.7761120796203613, + "learning_rate": 0.0001, + "loss": 59.6935, + "step": 14380 + }, + { + "epoch": 23.172519776288606, + "grad_norm": 0.6392366290092468, + "learning_rate": 0.0001, + "loss": 61.0263, + "step": 14390 + }, + { + "epoch": 23.18864311986698, + "grad_norm": 0.5722920894622803, + "learning_rate": 0.0001, + "loss": 59.4247, + "step": 14400 + }, + { + "epoch": 23.204766463445356, + "grad_norm": 0.6938275694847107, + "learning_rate": 0.0001, + "loss": 61.0933, + "step": 14410 + }, + { + "epoch": 23.22088980702373, + "grad_norm": 0.9164982438087463, + "learning_rate": 0.0001, + "loss": 58.0045, + "step": 14420 + }, + { + "epoch": 23.237013150602106, + "grad_norm": 0.6641019582748413, + "learning_rate": 0.0001, + "loss": 62.2162, + "step": 14430 + }, + { + "epoch": 23.25313649418048, + "grad_norm": 0.6775721311569214, + "learning_rate": 0.0001, + "loss": 60.2123, + "step": 14440 + }, + { + "epoch": 23.269259837758856, + "grad_norm": 0.6306013464927673, + "learning_rate": 0.0001, + "loss": 60.0638, + "step": 14450 + }, + { + "epoch": 23.28538318133723, + "grad_norm": 0.7770274877548218, + "learning_rate": 0.0001, + "loss": 60.87, + "step": 14460 + }, + { + "epoch": 23.301506524915606, + "grad_norm": 0.6893128752708435, + "learning_rate": 0.0001, + "loss": 59.2247, + "step": 14470 + }, + { + "epoch": 23.317629868493977, + "grad_norm": 0.7028453946113586, + "learning_rate": 0.0001, + "loss": 60.2405, + "step": 14480 + }, + { + "epoch": 23.333753212072352, + "grad_norm": 0.6184173822402954, + "learning_rate": 0.0001, + "loss": 58.5765, + "step": 14490 + }, + { + "epoch": 23.349876555650727, + "grad_norm": 0.793502151966095, + "learning_rate": 0.0001, + "loss": 57.5978, + "step": 14500 + }, + { + "epoch": 23.365999899229102, + "grad_norm": 0.660885751247406, + "learning_rate": 0.0001, + "loss": 60.5535, + "step": 14510 + }, + { + "epoch": 23.382123242807477, + "grad_norm": 0.9039516448974609, + "learning_rate": 0.0001, + "loss": 58.9432, + "step": 14520 + }, + { + "epoch": 23.398246586385852, + "grad_norm": 0.7348603010177612, + "learning_rate": 0.0001, + "loss": 58.3708, + "step": 14530 + }, + { + "epoch": 23.414369929964227, + "grad_norm": 0.5571108460426331, + "learning_rate": 0.0001, + "loss": 60.2059, + "step": 14540 + }, + { + "epoch": 23.430493273542602, + "grad_norm": 0.77266526222229, + "learning_rate": 0.0001, + "loss": 59.9602, + "step": 14550 + }, + { + "epoch": 23.446616617120977, + "grad_norm": 0.7564616203308105, + "learning_rate": 0.0001, + "loss": 59.0114, + "step": 14560 + }, + { + "epoch": 23.46273996069935, + "grad_norm": 1.0693135261535645, + "learning_rate": 0.0001, + "loss": 60.6075, + "step": 14570 + }, + { + "epoch": 23.478863304277724, + "grad_norm": 0.7800086140632629, + "learning_rate": 0.0001, + "loss": 60.6608, + "step": 14580 + }, + { + "epoch": 23.4949866478561, + "grad_norm": 0.8552407026290894, + "learning_rate": 0.0001, + "loss": 60.2378, + "step": 14590 + }, + { + "epoch": 23.511109991434473, + "grad_norm": 0.7270957231521606, + "learning_rate": 0.0001, + "loss": 60.8936, + "step": 14600 + }, + { + "epoch": 23.52723333501285, + "grad_norm": 0.5652869343757629, + "learning_rate": 0.0001, + "loss": 59.9882, + "step": 14610 + }, + { + "epoch": 23.543356678591223, + "grad_norm": 0.5465036034584045, + "learning_rate": 0.0001, + "loss": 57.4725, + "step": 14620 + }, + { + "epoch": 23.5594800221696, + "grad_norm": 0.6404461860656738, + "learning_rate": 0.0001, + "loss": 60.3962, + "step": 14630 + }, + { + "epoch": 23.575603365747973, + "grad_norm": 0.7860730886459351, + "learning_rate": 0.0001, + "loss": 60.3245, + "step": 14640 + }, + { + "epoch": 23.591726709326345, + "grad_norm": 0.6626171469688416, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 14650 + }, + { + "epoch": 23.60785005290472, + "grad_norm": 1.5377638339996338, + "learning_rate": 0.0001, + "loss": 58.9615, + "step": 14660 + }, + { + "epoch": 23.623973396483095, + "grad_norm": 0.6656123995780945, + "learning_rate": 0.0001, + "loss": 60.7947, + "step": 14670 + }, + { + "epoch": 23.64009674006147, + "grad_norm": 0.5155975818634033, + "learning_rate": 0.0001, + "loss": 57.7742, + "step": 14680 + }, + { + "epoch": 23.656220083639845, + "grad_norm": 0.6278973817825317, + "learning_rate": 0.0001, + "loss": 58.6188, + "step": 14690 + }, + { + "epoch": 23.67234342721822, + "grad_norm": 0.7261143326759338, + "learning_rate": 0.0001, + "loss": 56.8495, + "step": 14700 + }, + { + "epoch": 23.688466770796595, + "grad_norm": 0.8105781078338623, + "learning_rate": 0.0001, + "loss": 57.3139, + "step": 14710 + }, + { + "epoch": 23.70459011437497, + "grad_norm": 0.7012073993682861, + "learning_rate": 0.0001, + "loss": 61.1186, + "step": 14720 + }, + { + "epoch": 23.720713457953345, + "grad_norm": 0.6352536082267761, + "learning_rate": 0.0001, + "loss": 59.0136, + "step": 14730 + }, + { + "epoch": 23.736836801531716, + "grad_norm": 0.7279717326164246, + "learning_rate": 0.0001, + "loss": 58.7408, + "step": 14740 + }, + { + "epoch": 23.75296014511009, + "grad_norm": 0.6714015603065491, + "learning_rate": 0.0001, + "loss": 58.6431, + "step": 14750 + }, + { + "epoch": 23.769083488688466, + "grad_norm": 1.3876261711120605, + "learning_rate": 0.0001, + "loss": 59.7586, + "step": 14760 + }, + { + "epoch": 23.78520683226684, + "grad_norm": 0.7192941904067993, + "learning_rate": 0.0001, + "loss": 58.8487, + "step": 14770 + }, + { + "epoch": 23.801330175845216, + "grad_norm": 0.7333327531814575, + "learning_rate": 0.0001, + "loss": 60.5608, + "step": 14780 + }, + { + "epoch": 23.81745351942359, + "grad_norm": 0.8186061382293701, + "learning_rate": 0.0001, + "loss": 59.4232, + "step": 14790 + }, + { + "epoch": 23.833576863001966, + "grad_norm": 1.037039041519165, + "learning_rate": 0.0001, + "loss": 58.2384, + "step": 14800 + }, + { + "epoch": 23.84970020658034, + "grad_norm": 0.6739209890365601, + "learning_rate": 0.0001, + "loss": 57.7256, + "step": 14810 + }, + { + "epoch": 23.865823550158716, + "grad_norm": 0.6515845656394958, + "learning_rate": 0.0001, + "loss": 61.0039, + "step": 14820 + }, + { + "epoch": 23.881946893737087, + "grad_norm": 0.559317946434021, + "learning_rate": 0.0001, + "loss": 60.4458, + "step": 14830 + }, + { + "epoch": 23.898070237315462, + "grad_norm": 0.663934588432312, + "learning_rate": 0.0001, + "loss": 59.1646, + "step": 14840 + }, + { + "epoch": 23.914193580893837, + "grad_norm": 0.6409069299697876, + "learning_rate": 0.0001, + "loss": 58.9508, + "step": 14850 + }, + { + "epoch": 23.930316924472212, + "grad_norm": 0.7624269127845764, + "learning_rate": 0.0001, + "loss": 60.7256, + "step": 14860 + }, + { + "epoch": 23.946440268050587, + "grad_norm": 0.6949140429496765, + "learning_rate": 0.0001, + "loss": 58.8987, + "step": 14870 + }, + { + "epoch": 23.962563611628962, + "grad_norm": 0.714063823223114, + "learning_rate": 0.0001, + "loss": 59.255, + "step": 14880 + }, + { + "epoch": 23.978686955207337, + "grad_norm": 0.6570878028869629, + "learning_rate": 0.0001, + "loss": 59.2637, + "step": 14890 + }, + { + "epoch": 23.994810298785712, + "grad_norm": 0.7898263335227966, + "learning_rate": 0.0001, + "loss": 61.0218, + "step": 14900 + }, + { + "epoch": 24.009674006147026, + "grad_norm": 0.6673877239227295, + "learning_rate": 0.0001, + "loss": 53.8562, + "step": 14910 + }, + { + "epoch": 24.025797349725398, + "grad_norm": 0.5548285245895386, + "learning_rate": 0.0001, + "loss": 59.4445, + "step": 14920 + }, + { + "epoch": 24.041920693303773, + "grad_norm": 0.6273561716079712, + "learning_rate": 0.0001, + "loss": 59.3292, + "step": 14930 + }, + { + "epoch": 24.058044036882148, + "grad_norm": 0.7050267457962036, + "learning_rate": 0.0001, + "loss": 60.4896, + "step": 14940 + }, + { + "epoch": 24.074167380460523, + "grad_norm": 0.7030066251754761, + "learning_rate": 0.0001, + "loss": 59.1622, + "step": 14950 + }, + { + "epoch": 24.090290724038898, + "grad_norm": 0.8014484643936157, + "learning_rate": 0.0001, + "loss": 59.746, + "step": 14960 + }, + { + "epoch": 24.106414067617273, + "grad_norm": 0.869323194026947, + "learning_rate": 0.0001, + "loss": 60.2156, + "step": 14970 + }, + { + "epoch": 24.122537411195648, + "grad_norm": 0.7157219648361206, + "learning_rate": 0.0001, + "loss": 58.7229, + "step": 14980 + }, + { + "epoch": 24.138660754774023, + "grad_norm": 0.6318342685699463, + "learning_rate": 0.0001, + "loss": 61.1832, + "step": 14990 + }, + { + "epoch": 24.154784098352394, + "grad_norm": 0.8689974546432495, + "learning_rate": 0.0001, + "loss": 58.8083, + "step": 15000 + }, + { + "epoch": 24.17090744193077, + "grad_norm": 1.1719586849212646, + "learning_rate": 0.0001, + "loss": 57.745, + "step": 15010 + }, + { + "epoch": 24.187030785509144, + "grad_norm": 0.676295280456543, + "learning_rate": 0.0001, + "loss": 60.6236, + "step": 15020 + }, + { + "epoch": 24.20315412908752, + "grad_norm": 0.6745544672012329, + "learning_rate": 0.0001, + "loss": 59.494, + "step": 15030 + }, + { + "epoch": 24.219277472665894, + "grad_norm": 0.6221221685409546, + "learning_rate": 0.0001, + "loss": 58.894, + "step": 15040 + }, + { + "epoch": 24.23540081624427, + "grad_norm": 0.7013781070709229, + "learning_rate": 0.0001, + "loss": 59.935, + "step": 15050 + }, + { + "epoch": 24.251524159822644, + "grad_norm": 0.6094849705696106, + "learning_rate": 0.0001, + "loss": 59.8276, + "step": 15060 + }, + { + "epoch": 24.26764750340102, + "grad_norm": 0.6969972848892212, + "learning_rate": 0.0001, + "loss": 59.7568, + "step": 15070 + }, + { + "epoch": 24.283770846979394, + "grad_norm": 0.7223725318908691, + "learning_rate": 0.0001, + "loss": 57.9065, + "step": 15080 + }, + { + "epoch": 24.299894190557765, + "grad_norm": 0.7008336186408997, + "learning_rate": 0.0001, + "loss": 59.5206, + "step": 15090 + }, + { + "epoch": 24.31601753413614, + "grad_norm": 0.6490783095359802, + "learning_rate": 0.0001, + "loss": 59.4457, + "step": 15100 + }, + { + "epoch": 24.332140877714515, + "grad_norm": 0.6096290349960327, + "learning_rate": 0.0001, + "loss": 60.8434, + "step": 15110 + }, + { + "epoch": 24.34826422129289, + "grad_norm": 0.771045446395874, + "learning_rate": 0.0001, + "loss": 60.3167, + "step": 15120 + }, + { + "epoch": 24.364387564871265, + "grad_norm": 0.7178729772567749, + "learning_rate": 0.0001, + "loss": 57.6667, + "step": 15130 + }, + { + "epoch": 24.38051090844964, + "grad_norm": 0.6484465003013611, + "learning_rate": 0.0001, + "loss": 60.1199, + "step": 15140 + }, + { + "epoch": 24.396634252028015, + "grad_norm": 1.0030865669250488, + "learning_rate": 0.0001, + "loss": 57.9288, + "step": 15150 + }, + { + "epoch": 24.41275759560639, + "grad_norm": 0.8297843933105469, + "learning_rate": 0.0001, + "loss": 61.0941, + "step": 15160 + }, + { + "epoch": 24.428880939184765, + "grad_norm": 0.9115694165229797, + "learning_rate": 0.0001, + "loss": 58.9574, + "step": 15170 + }, + { + "epoch": 24.445004282763136, + "grad_norm": 0.7758508920669556, + "learning_rate": 0.0001, + "loss": 58.8852, + "step": 15180 + }, + { + "epoch": 24.46112762634151, + "grad_norm": 0.5539108514785767, + "learning_rate": 0.0001, + "loss": 60.9549, + "step": 15190 + }, + { + "epoch": 24.477250969919886, + "grad_norm": 0.7292829751968384, + "learning_rate": 0.0001, + "loss": 58.0818, + "step": 15200 + }, + { + "epoch": 24.49337431349826, + "grad_norm": 0.6253948211669922, + "learning_rate": 0.0001, + "loss": 59.656, + "step": 15210 + }, + { + "epoch": 24.509497657076636, + "grad_norm": 0.8054394125938416, + "learning_rate": 0.0001, + "loss": 60.934, + "step": 15220 + }, + { + "epoch": 24.52562100065501, + "grad_norm": 0.8183079361915588, + "learning_rate": 0.0001, + "loss": 59.5303, + "step": 15230 + }, + { + "epoch": 24.541744344233386, + "grad_norm": 0.6063419580459595, + "learning_rate": 0.0001, + "loss": 59.2852, + "step": 15240 + }, + { + "epoch": 24.55786768781176, + "grad_norm": 0.6372854113578796, + "learning_rate": 0.0001, + "loss": 58.9647, + "step": 15250 + }, + { + "epoch": 24.573991031390136, + "grad_norm": 0.6270446181297302, + "learning_rate": 0.0001, + "loss": 58.5641, + "step": 15260 + }, + { + "epoch": 24.590114374968508, + "grad_norm": 0.6229680776596069, + "learning_rate": 0.0001, + "loss": 58.4142, + "step": 15270 + }, + { + "epoch": 24.606237718546883, + "grad_norm": 0.6567183136940002, + "learning_rate": 0.0001, + "loss": 60.1162, + "step": 15280 + }, + { + "epoch": 24.622361062125258, + "grad_norm": 0.6975446343421936, + "learning_rate": 0.0001, + "loss": 58.7862, + "step": 15290 + }, + { + "epoch": 24.638484405703633, + "grad_norm": 0.6961022615432739, + "learning_rate": 0.0001, + "loss": 58.3247, + "step": 15300 + }, + { + "epoch": 24.654607749282007, + "grad_norm": 5.25744104385376, + "learning_rate": 0.0001, + "loss": 59.3863, + "step": 15310 + }, + { + "epoch": 24.670731092860382, + "grad_norm": 0.8439975380897522, + "learning_rate": 0.0001, + "loss": 59.7157, + "step": 15320 + }, + { + "epoch": 24.686854436438757, + "grad_norm": 0.596845269203186, + "learning_rate": 0.0001, + "loss": 60.0859, + "step": 15330 + }, + { + "epoch": 24.702977780017132, + "grad_norm": 0.6758428812026978, + "learning_rate": 0.0001, + "loss": 59.9305, + "step": 15340 + }, + { + "epoch": 24.719101123595507, + "grad_norm": 0.7418860197067261, + "learning_rate": 0.0001, + "loss": 59.0929, + "step": 15350 + }, + { + "epoch": 24.73522446717388, + "grad_norm": 0.6856047511100769, + "learning_rate": 0.0001, + "loss": 58.7201, + "step": 15360 + }, + { + "epoch": 24.751347810752254, + "grad_norm": 0.5327897071838379, + "learning_rate": 0.0001, + "loss": 59.5899, + "step": 15370 + }, + { + "epoch": 24.76747115433063, + "grad_norm": 0.7127685546875, + "learning_rate": 0.0001, + "loss": 59.7729, + "step": 15380 + }, + { + "epoch": 24.783594497909004, + "grad_norm": 1.8885949850082397, + "learning_rate": 0.0001, + "loss": 58.3813, + "step": 15390 + }, + { + "epoch": 24.79971784148738, + "grad_norm": 0.7997881174087524, + "learning_rate": 0.0001, + "loss": 61.9402, + "step": 15400 + }, + { + "epoch": 24.815841185065754, + "grad_norm": 0.9729613661766052, + "learning_rate": 0.0001, + "loss": 59.7961, + "step": 15410 + }, + { + "epoch": 24.83196452864413, + "grad_norm": 0.8450629115104675, + "learning_rate": 0.0001, + "loss": 59.5969, + "step": 15420 + }, + { + "epoch": 24.848087872222504, + "grad_norm": 0.720668613910675, + "learning_rate": 0.0001, + "loss": 58.187, + "step": 15430 + }, + { + "epoch": 24.864211215800875, + "grad_norm": 0.638506293296814, + "learning_rate": 0.0001, + "loss": 59.4599, + "step": 15440 + }, + { + "epoch": 24.88033455937925, + "grad_norm": 0.6158926486968994, + "learning_rate": 0.0001, + "loss": 61.3858, + "step": 15450 + }, + { + "epoch": 24.896457902957625, + "grad_norm": 0.8130396604537964, + "learning_rate": 0.0001, + "loss": 59.3795, + "step": 15460 + }, + { + "epoch": 24.912581246536, + "grad_norm": 0.7966095805168152, + "learning_rate": 0.0001, + "loss": 60.3319, + "step": 15470 + }, + { + "epoch": 24.928704590114375, + "grad_norm": 0.6240444779396057, + "learning_rate": 0.0001, + "loss": 59.0407, + "step": 15480 + }, + { + "epoch": 24.94482793369275, + "grad_norm": 0.7439318895339966, + "learning_rate": 0.0001, + "loss": 57.7692, + "step": 15490 + }, + { + "epoch": 24.960951277271125, + "grad_norm": 0.6942011713981628, + "learning_rate": 0.0001, + "loss": 56.8418, + "step": 15500 + }, + { + "epoch": 24.9770746208495, + "grad_norm": 0.6470242738723755, + "learning_rate": 0.0001, + "loss": 59.8361, + "step": 15510 + }, + { + "epoch": 24.993197964427875, + "grad_norm": 0.6935222148895264, + "learning_rate": 0.0001, + "loss": 59.5357, + "step": 15520 + }, + { + "epoch": 25.008061671789186, + "grad_norm": 0.8943103551864624, + "learning_rate": 0.0001, + "loss": 54.8185, + "step": 15530 + }, + { + "epoch": 25.02418501536756, + "grad_norm": 0.7637899518013, + "learning_rate": 0.0001, + "loss": 58.4746, + "step": 15540 + }, + { + "epoch": 25.040308358945936, + "grad_norm": 0.8261918425559998, + "learning_rate": 0.0001, + "loss": 57.3993, + "step": 15550 + }, + { + "epoch": 25.05643170252431, + "grad_norm": 0.6395642161369324, + "learning_rate": 0.0001, + "loss": 58.123, + "step": 15560 + }, + { + "epoch": 25.072555046102686, + "grad_norm": 0.7899443507194519, + "learning_rate": 0.0001, + "loss": 57.1487, + "step": 15570 + }, + { + "epoch": 25.08867838968106, + "grad_norm": 0.6202112436294556, + "learning_rate": 0.0001, + "loss": 59.4995, + "step": 15580 + }, + { + "epoch": 25.104801733259436, + "grad_norm": 0.7683939933776855, + "learning_rate": 0.0001, + "loss": 59.1155, + "step": 15590 + }, + { + "epoch": 25.12092507683781, + "grad_norm": 0.6531215906143188, + "learning_rate": 0.0001, + "loss": 61.9998, + "step": 15600 + }, + { + "epoch": 25.137048420416185, + "grad_norm": 0.6354383826255798, + "learning_rate": 0.0001, + "loss": 59.3861, + "step": 15610 + }, + { + "epoch": 25.153171763994557, + "grad_norm": 0.6507435441017151, + "learning_rate": 0.0001, + "loss": 59.939, + "step": 15620 + }, + { + "epoch": 25.169295107572932, + "grad_norm": 0.8014065623283386, + "learning_rate": 0.0001, + "loss": 59.4927, + "step": 15630 + }, + { + "epoch": 25.185418451151307, + "grad_norm": 0.53805011510849, + "learning_rate": 0.0001, + "loss": 58.526, + "step": 15640 + }, + { + "epoch": 25.201541794729682, + "grad_norm": 4.835650444030762, + "learning_rate": 0.0001, + "loss": 58.6711, + "step": 15650 + }, + { + "epoch": 25.217665138308057, + "grad_norm": 0.723210871219635, + "learning_rate": 0.0001, + "loss": 60.3953, + "step": 15660 + }, + { + "epoch": 25.23378848188643, + "grad_norm": 0.7176834940910339, + "learning_rate": 0.0001, + "loss": 58.8576, + "step": 15670 + }, + { + "epoch": 25.249911825464807, + "grad_norm": 0.8280025124549866, + "learning_rate": 0.0001, + "loss": 58.7146, + "step": 15680 + }, + { + "epoch": 25.26603516904318, + "grad_norm": 0.6296973824501038, + "learning_rate": 0.0001, + "loss": 60.7341, + "step": 15690 + }, + { + "epoch": 25.282158512621557, + "grad_norm": 1.3184518814086914, + "learning_rate": 0.0001, + "loss": 60.6906, + "step": 15700 + }, + { + "epoch": 25.298281856199928, + "grad_norm": 1.0408564805984497, + "learning_rate": 0.0001, + "loss": 57.4703, + "step": 15710 + }, + { + "epoch": 25.314405199778303, + "grad_norm": 0.7140268683433533, + "learning_rate": 0.0001, + "loss": 59.8452, + "step": 15720 + }, + { + "epoch": 25.330528543356678, + "grad_norm": 0.7822073698043823, + "learning_rate": 0.0001, + "loss": 60.4394, + "step": 15730 + }, + { + "epoch": 25.346651886935053, + "grad_norm": 0.6579483151435852, + "learning_rate": 0.0001, + "loss": 59.5358, + "step": 15740 + }, + { + "epoch": 25.362775230513428, + "grad_norm": 0.6936199069023132, + "learning_rate": 0.0001, + "loss": 60.2117, + "step": 15750 + }, + { + "epoch": 25.378898574091803, + "grad_norm": 0.7166867852210999, + "learning_rate": 0.0001, + "loss": 58.6835, + "step": 15760 + }, + { + "epoch": 25.395021917670178, + "grad_norm": 0.7240379452705383, + "learning_rate": 0.0001, + "loss": 59.6293, + "step": 15770 + }, + { + "epoch": 25.411145261248553, + "grad_norm": 0.7465791702270508, + "learning_rate": 0.0001, + "loss": 58.2512, + "step": 15780 + }, + { + "epoch": 25.427268604826924, + "grad_norm": 0.6713439226150513, + "learning_rate": 0.0001, + "loss": 58.4149, + "step": 15790 + }, + { + "epoch": 25.4433919484053, + "grad_norm": 0.587433397769928, + "learning_rate": 0.0001, + "loss": 60.7315, + "step": 15800 + }, + { + "epoch": 25.459515291983674, + "grad_norm": 0.8216325044631958, + "learning_rate": 0.0001, + "loss": 58.5339, + "step": 15810 + }, + { + "epoch": 25.47563863556205, + "grad_norm": 0.8947033286094666, + "learning_rate": 0.0001, + "loss": 58.7762, + "step": 15820 + }, + { + "epoch": 25.491761979140424, + "grad_norm": 0.6659840941429138, + "learning_rate": 0.0001, + "loss": 58.9931, + "step": 15830 + }, + { + "epoch": 25.5078853227188, + "grad_norm": 0.7239252328872681, + "learning_rate": 0.0001, + "loss": 59.7113, + "step": 15840 + }, + { + "epoch": 25.524008666297174, + "grad_norm": 0.7233046889305115, + "learning_rate": 0.0001, + "loss": 58.973, + "step": 15850 + }, + { + "epoch": 25.54013200987555, + "grad_norm": 0.6620002388954163, + "learning_rate": 0.0001, + "loss": 57.1255, + "step": 15860 + }, + { + "epoch": 25.556255353453924, + "grad_norm": 0.6544366478919983, + "learning_rate": 0.0001, + "loss": 58.5897, + "step": 15870 + }, + { + "epoch": 25.572378697032295, + "grad_norm": 0.8955110311508179, + "learning_rate": 0.0001, + "loss": 58.3241, + "step": 15880 + }, + { + "epoch": 25.58850204061067, + "grad_norm": 0.6869794726371765, + "learning_rate": 0.0001, + "loss": 58.1951, + "step": 15890 + }, + { + "epoch": 25.604625384189045, + "grad_norm": 0.6819241642951965, + "learning_rate": 0.0001, + "loss": 58.6505, + "step": 15900 + }, + { + "epoch": 25.62074872776742, + "grad_norm": 0.6019999980926514, + "learning_rate": 0.0001, + "loss": 58.1397, + "step": 15910 + }, + { + "epoch": 25.636872071345795, + "grad_norm": 0.6664146184921265, + "learning_rate": 0.0001, + "loss": 58.6735, + "step": 15920 + }, + { + "epoch": 25.65299541492417, + "grad_norm": 0.7511261701583862, + "learning_rate": 0.0001, + "loss": 58.9189, + "step": 15930 + }, + { + "epoch": 25.669118758502545, + "grad_norm": 0.6886443495750427, + "learning_rate": 0.0001, + "loss": 60.1288, + "step": 15940 + }, + { + "epoch": 25.68524210208092, + "grad_norm": 0.7647057771682739, + "learning_rate": 0.0001, + "loss": 58.5042, + "step": 15950 + }, + { + "epoch": 25.701365445659295, + "grad_norm": 0.6636016368865967, + "learning_rate": 0.0001, + "loss": 56.7644, + "step": 15960 + }, + { + "epoch": 25.717488789237667, + "grad_norm": 0.6909965872764587, + "learning_rate": 0.0001, + "loss": 58.9248, + "step": 15970 + }, + { + "epoch": 25.73361213281604, + "grad_norm": 0.7853394150733948, + "learning_rate": 0.0001, + "loss": 58.7267, + "step": 15980 + }, + { + "epoch": 25.749735476394417, + "grad_norm": 0.5992239713668823, + "learning_rate": 0.0001, + "loss": 58.4463, + "step": 15990 + }, + { + "epoch": 25.76585881997279, + "grad_norm": 0.5730507969856262, + "learning_rate": 0.0001, + "loss": 58.2946, + "step": 16000 + }, + { + "epoch": 25.781982163551167, + "grad_norm": 0.8904218077659607, + "learning_rate": 0.0001, + "loss": 58.8344, + "step": 16010 + }, + { + "epoch": 25.79810550712954, + "grad_norm": 0.8680905103683472, + "learning_rate": 0.0001, + "loss": 58.8969, + "step": 16020 + }, + { + "epoch": 25.814228850707917, + "grad_norm": 0.7501125335693359, + "learning_rate": 0.0001, + "loss": 59.0709, + "step": 16030 + }, + { + "epoch": 25.83035219428629, + "grad_norm": 0.7211290597915649, + "learning_rate": 0.0001, + "loss": 58.1067, + "step": 16040 + }, + { + "epoch": 25.846475537864663, + "grad_norm": 0.6130759119987488, + "learning_rate": 0.0001, + "loss": 58.7301, + "step": 16050 + }, + { + "epoch": 25.862598881443038, + "grad_norm": 0.5713672041893005, + "learning_rate": 0.0001, + "loss": 59.7271, + "step": 16060 + }, + { + "epoch": 25.878722225021413, + "grad_norm": 0.5994897484779358, + "learning_rate": 0.0001, + "loss": 59.3829, + "step": 16070 + }, + { + "epoch": 25.894845568599788, + "grad_norm": 0.686657190322876, + "learning_rate": 0.0001, + "loss": 59.2881, + "step": 16080 + }, + { + "epoch": 25.910968912178163, + "grad_norm": 0.6650110483169556, + "learning_rate": 0.0001, + "loss": 56.2951, + "step": 16090 + }, + { + "epoch": 25.927092255756538, + "grad_norm": 0.7304360866546631, + "learning_rate": 0.0001, + "loss": 60.5764, + "step": 16100 + }, + { + "epoch": 25.943215599334913, + "grad_norm": 0.6502479910850525, + "learning_rate": 0.0001, + "loss": 59.3936, + "step": 16110 + }, + { + "epoch": 25.959338942913288, + "grad_norm": 0.7156543731689453, + "learning_rate": 0.0001, + "loss": 58.3354, + "step": 16120 + }, + { + "epoch": 25.975462286491663, + "grad_norm": 0.951116144657135, + "learning_rate": 0.0001, + "loss": 60.3694, + "step": 16130 + }, + { + "epoch": 25.991585630070034, + "grad_norm": 0.6068692207336426, + "learning_rate": 0.0001, + "loss": 58.4947, + "step": 16140 + }, + { + "epoch": 26.00644933743135, + "grad_norm": 0.7921227216720581, + "learning_rate": 0.0001, + "loss": 54.2985, + "step": 16150 + }, + { + "epoch": 26.022572681009724, + "grad_norm": 0.7406607866287231, + "learning_rate": 0.0001, + "loss": 59.1799, + "step": 16160 + }, + { + "epoch": 26.0386960245881, + "grad_norm": 0.6859458088874817, + "learning_rate": 0.0001, + "loss": 58.5136, + "step": 16170 + }, + { + "epoch": 26.054819368166473, + "grad_norm": 0.5251442790031433, + "learning_rate": 0.0001, + "loss": 58.0473, + "step": 16180 + }, + { + "epoch": 26.07094271174485, + "grad_norm": 0.8837170600891113, + "learning_rate": 0.0001, + "loss": 58.7922, + "step": 16190 + }, + { + "epoch": 26.087066055323223, + "grad_norm": 0.6178808808326721, + "learning_rate": 0.0001, + "loss": 58.5929, + "step": 16200 + }, + { + "epoch": 26.1031893989016, + "grad_norm": 0.7404752373695374, + "learning_rate": 0.0001, + "loss": 60.8605, + "step": 16210 + }, + { + "epoch": 26.119312742479973, + "grad_norm": 0.812877357006073, + "learning_rate": 0.0001, + "loss": 60.1625, + "step": 16220 + }, + { + "epoch": 26.135436086058345, + "grad_norm": 0.7631093859672546, + "learning_rate": 0.0001, + "loss": 59.0114, + "step": 16230 + }, + { + "epoch": 26.15155942963672, + "grad_norm": 0.5635222792625427, + "learning_rate": 0.0001, + "loss": 58.3207, + "step": 16240 + }, + { + "epoch": 26.167682773215095, + "grad_norm": 0.6111196875572205, + "learning_rate": 0.0001, + "loss": 60.1611, + "step": 16250 + }, + { + "epoch": 26.18380611679347, + "grad_norm": 0.6021577715873718, + "learning_rate": 0.0001, + "loss": 59.2199, + "step": 16260 + }, + { + "epoch": 26.199929460371845, + "grad_norm": 0.6264477968215942, + "learning_rate": 0.0001, + "loss": 58.9156, + "step": 16270 + }, + { + "epoch": 26.21605280395022, + "grad_norm": 0.5620694160461426, + "learning_rate": 0.0001, + "loss": 58.699, + "step": 16280 + }, + { + "epoch": 26.232176147528595, + "grad_norm": 0.6636842489242554, + "learning_rate": 0.0001, + "loss": 58.1521, + "step": 16290 + }, + { + "epoch": 26.24829949110697, + "grad_norm": 1.903132677078247, + "learning_rate": 0.0001, + "loss": 56.8953, + "step": 16300 + }, + { + "epoch": 26.264422834685345, + "grad_norm": 0.6112220883369446, + "learning_rate": 0.0001, + "loss": 58.6921, + "step": 16310 + }, + { + "epoch": 26.280546178263716, + "grad_norm": 0.6523616909980774, + "learning_rate": 0.0001, + "loss": 59.8097, + "step": 16320 + }, + { + "epoch": 26.29666952184209, + "grad_norm": 0.6033239364624023, + "learning_rate": 0.0001, + "loss": 58.2301, + "step": 16330 + }, + { + "epoch": 26.312792865420466, + "grad_norm": 0.6334905624389648, + "learning_rate": 0.0001, + "loss": 57.4851, + "step": 16340 + }, + { + "epoch": 26.32891620899884, + "grad_norm": 0.8884124159812927, + "learning_rate": 0.0001, + "loss": 58.4404, + "step": 16350 + }, + { + "epoch": 26.345039552577216, + "grad_norm": 0.6134583950042725, + "learning_rate": 0.0001, + "loss": 59.7355, + "step": 16360 + }, + { + "epoch": 26.36116289615559, + "grad_norm": 0.5804709792137146, + "learning_rate": 0.0001, + "loss": 59.3307, + "step": 16370 + }, + { + "epoch": 26.377286239733966, + "grad_norm": 1.3471691608428955, + "learning_rate": 0.0001, + "loss": 60.4356, + "step": 16380 + }, + { + "epoch": 26.39340958331234, + "grad_norm": 0.7593581080436707, + "learning_rate": 0.0001, + "loss": 60.6596, + "step": 16390 + }, + { + "epoch": 26.409532926890712, + "grad_norm": 0.7059800624847412, + "learning_rate": 0.0001, + "loss": 58.8393, + "step": 16400 + }, + { + "epoch": 26.425656270469087, + "grad_norm": 0.836167573928833, + "learning_rate": 0.0001, + "loss": 59.7399, + "step": 16410 + }, + { + "epoch": 26.441779614047462, + "grad_norm": 0.7547316551208496, + "learning_rate": 0.0001, + "loss": 57.8424, + "step": 16420 + }, + { + "epoch": 26.457902957625837, + "grad_norm": 0.6655163168907166, + "learning_rate": 0.0001, + "loss": 58.7625, + "step": 16430 + }, + { + "epoch": 26.474026301204212, + "grad_norm": 0.6393458247184753, + "learning_rate": 0.0001, + "loss": 58.88, + "step": 16440 + }, + { + "epoch": 26.490149644782587, + "grad_norm": 0.7284976840019226, + "learning_rate": 0.0001, + "loss": 59.734, + "step": 16450 + }, + { + "epoch": 26.506272988360962, + "grad_norm": 0.7217265963554382, + "learning_rate": 0.0001, + "loss": 58.9343, + "step": 16460 + }, + { + "epoch": 26.522396331939337, + "grad_norm": 0.6045732498168945, + "learning_rate": 0.0001, + "loss": 58.758, + "step": 16470 + }, + { + "epoch": 26.538519675517712, + "grad_norm": 0.6189675331115723, + "learning_rate": 0.0001, + "loss": 56.4975, + "step": 16480 + }, + { + "epoch": 26.554643019096083, + "grad_norm": 0.5564262270927429, + "learning_rate": 0.0001, + "loss": 58.6003, + "step": 16490 + }, + { + "epoch": 26.57076636267446, + "grad_norm": 0.727336049079895, + "learning_rate": 0.0001, + "loss": 59.097, + "step": 16500 + }, + { + "epoch": 26.586889706252833, + "grad_norm": 0.5768782496452332, + "learning_rate": 0.0001, + "loss": 59.4801, + "step": 16510 + }, + { + "epoch": 26.60301304983121, + "grad_norm": 0.5879726409912109, + "learning_rate": 0.0001, + "loss": 58.8839, + "step": 16520 + }, + { + "epoch": 26.619136393409583, + "grad_norm": 0.6342545747756958, + "learning_rate": 0.0001, + "loss": 58.4209, + "step": 16530 + }, + { + "epoch": 26.63525973698796, + "grad_norm": 0.7538003325462341, + "learning_rate": 0.0001, + "loss": 58.973, + "step": 16540 + }, + { + "epoch": 26.651383080566333, + "grad_norm": 0.7319458723068237, + "learning_rate": 0.0001, + "loss": 59.5449, + "step": 16550 + }, + { + "epoch": 26.667506424144708, + "grad_norm": 0.6014299392700195, + "learning_rate": 0.0001, + "loss": 59.2934, + "step": 16560 + }, + { + "epoch": 26.683629767723083, + "grad_norm": 0.6779450178146362, + "learning_rate": 0.0001, + "loss": 59.5138, + "step": 16570 + }, + { + "epoch": 26.699753111301455, + "grad_norm": 0.6749789118766785, + "learning_rate": 0.0001, + "loss": 58.2765, + "step": 16580 + }, + { + "epoch": 26.71587645487983, + "grad_norm": 0.6327691674232483, + "learning_rate": 0.0001, + "loss": 59.766, + "step": 16590 + }, + { + "epoch": 26.731999798458205, + "grad_norm": 0.5834855437278748, + "learning_rate": 0.0001, + "loss": 57.2493, + "step": 16600 + }, + { + "epoch": 26.74812314203658, + "grad_norm": 0.6618366837501526, + "learning_rate": 0.0001, + "loss": 57.2663, + "step": 16610 + }, + { + "epoch": 26.764246485614954, + "grad_norm": 0.7070173621177673, + "learning_rate": 0.0001, + "loss": 61.1754, + "step": 16620 + }, + { + "epoch": 26.78036982919333, + "grad_norm": 0.651278555393219, + "learning_rate": 0.0001, + "loss": 59.033, + "step": 16630 + }, + { + "epoch": 26.796493172771704, + "grad_norm": 0.634915828704834, + "learning_rate": 0.0001, + "loss": 58.2336, + "step": 16640 + }, + { + "epoch": 26.81261651635008, + "grad_norm": 0.8514180779457092, + "learning_rate": 0.0001, + "loss": 57.1678, + "step": 16650 + }, + { + "epoch": 26.828739859928454, + "grad_norm": 0.6718772053718567, + "learning_rate": 0.0001, + "loss": 58.5139, + "step": 16660 + }, + { + "epoch": 26.844863203506826, + "grad_norm": 0.6963548064231873, + "learning_rate": 0.0001, + "loss": 58.8449, + "step": 16670 + }, + { + "epoch": 26.8609865470852, + "grad_norm": 0.6028134822845459, + "learning_rate": 0.0001, + "loss": 58.4613, + "step": 16680 + }, + { + "epoch": 26.877109890663576, + "grad_norm": 0.7414282560348511, + "learning_rate": 0.0001, + "loss": 58.4078, + "step": 16690 + }, + { + "epoch": 26.89323323424195, + "grad_norm": 0.6019772291183472, + "learning_rate": 0.0001, + "loss": 58.086, + "step": 16700 + }, + { + "epoch": 26.909356577820326, + "grad_norm": 0.6267445683479309, + "learning_rate": 0.0001, + "loss": 58.5921, + "step": 16710 + }, + { + "epoch": 26.9254799213987, + "grad_norm": 0.7474961280822754, + "learning_rate": 0.0001, + "loss": 59.1194, + "step": 16720 + }, + { + "epoch": 26.941603264977076, + "grad_norm": 0.7952353358268738, + "learning_rate": 0.0001, + "loss": 58.4545, + "step": 16730 + }, + { + "epoch": 26.95772660855545, + "grad_norm": 0.6562651991844177, + "learning_rate": 0.0001, + "loss": 58.699, + "step": 16740 + }, + { + "epoch": 26.973849952133826, + "grad_norm": 0.6084649562835693, + "learning_rate": 0.0001, + "loss": 58.5552, + "step": 16750 + }, + { + "epoch": 26.989973295712197, + "grad_norm": 0.5753876566886902, + "learning_rate": 0.0001, + "loss": 58.5317, + "step": 16760 + }, + { + "epoch": 27.00483700307351, + "grad_norm": 0.7396492958068848, + "learning_rate": 0.0001, + "loss": 54.4917, + "step": 16770 + }, + { + "epoch": 27.020960346651886, + "grad_norm": 0.5759479999542236, + "learning_rate": 0.0001, + "loss": 60.5276, + "step": 16780 + }, + { + "epoch": 27.03708369023026, + "grad_norm": 0.6621933579444885, + "learning_rate": 0.0001, + "loss": 57.8109, + "step": 16790 + }, + { + "epoch": 27.053207033808636, + "grad_norm": 0.6219635605812073, + "learning_rate": 0.0001, + "loss": 59.1457, + "step": 16800 + }, + { + "epoch": 27.06933037738701, + "grad_norm": 0.6395589113235474, + "learning_rate": 0.0001, + "loss": 58.2069, + "step": 16810 + }, + { + "epoch": 27.085453720965386, + "grad_norm": 0.9114459753036499, + "learning_rate": 0.0001, + "loss": 58.7224, + "step": 16820 + }, + { + "epoch": 27.10157706454376, + "grad_norm": 0.6759451031684875, + "learning_rate": 0.0001, + "loss": 59.3057, + "step": 16830 + }, + { + "epoch": 27.117700408122133, + "grad_norm": 0.6997210383415222, + "learning_rate": 0.0001, + "loss": 59.7155, + "step": 16840 + }, + { + "epoch": 27.133823751700508, + "grad_norm": 1.4591188430786133, + "learning_rate": 0.0001, + "loss": 59.7251, + "step": 16850 + }, + { + "epoch": 27.149947095278883, + "grad_norm": 0.661625325679779, + "learning_rate": 0.0001, + "loss": 59.065, + "step": 16860 + }, + { + "epoch": 27.166070438857258, + "grad_norm": 0.7471212148666382, + "learning_rate": 0.0001, + "loss": 60.2483, + "step": 16870 + }, + { + "epoch": 27.182193782435633, + "grad_norm": 0.7325915098190308, + "learning_rate": 0.0001, + "loss": 59.5878, + "step": 16880 + }, + { + "epoch": 27.198317126014008, + "grad_norm": 0.6631053686141968, + "learning_rate": 0.0001, + "loss": 60.045, + "step": 16890 + }, + { + "epoch": 27.214440469592382, + "grad_norm": 0.7794095277786255, + "learning_rate": 0.0001, + "loss": 58.6427, + "step": 16900 + }, + { + "epoch": 27.230563813170757, + "grad_norm": 0.683525025844574, + "learning_rate": 0.0001, + "loss": 58.2072, + "step": 16910 + }, + { + "epoch": 27.246687156749132, + "grad_norm": 0.7676731944084167, + "learning_rate": 0.0001, + "loss": 59.5006, + "step": 16920 + }, + { + "epoch": 27.262810500327504, + "grad_norm": 0.6451899409294128, + "learning_rate": 0.0001, + "loss": 59.5833, + "step": 16930 + }, + { + "epoch": 27.27893384390588, + "grad_norm": 0.8003790378570557, + "learning_rate": 0.0001, + "loss": 59.6564, + "step": 16940 + }, + { + "epoch": 27.295057187484254, + "grad_norm": 0.8254024386405945, + "learning_rate": 0.0001, + "loss": 59.8074, + "step": 16950 + }, + { + "epoch": 27.31118053106263, + "grad_norm": 0.8280795812606812, + "learning_rate": 0.0001, + "loss": 57.1944, + "step": 16960 + }, + { + "epoch": 27.327303874641004, + "grad_norm": 0.6558408141136169, + "learning_rate": 0.0001, + "loss": 55.8846, + "step": 16970 + }, + { + "epoch": 27.34342721821938, + "grad_norm": 0.8485943078994751, + "learning_rate": 0.0001, + "loss": 57.6715, + "step": 16980 + }, + { + "epoch": 27.359550561797754, + "grad_norm": 0.7805924415588379, + "learning_rate": 0.0001, + "loss": 61.0427, + "step": 16990 + }, + { + "epoch": 27.37567390537613, + "grad_norm": 0.7988429069519043, + "learning_rate": 0.0001, + "loss": 57.6632, + "step": 17000 + }, + { + "epoch": 27.391797248954504, + "grad_norm": 0.816113293170929, + "learning_rate": 0.0001, + "loss": 59.2601, + "step": 17010 + }, + { + "epoch": 27.407920592532875, + "grad_norm": 0.7985144257545471, + "learning_rate": 0.0001, + "loss": 60.3656, + "step": 17020 + }, + { + "epoch": 27.42404393611125, + "grad_norm": 0.7022702097892761, + "learning_rate": 0.0001, + "loss": 58.657, + "step": 17030 + }, + { + "epoch": 27.440167279689625, + "grad_norm": 0.8588566780090332, + "learning_rate": 0.0001, + "loss": 58.9401, + "step": 17040 + }, + { + "epoch": 27.456290623268, + "grad_norm": 0.6444416046142578, + "learning_rate": 0.0001, + "loss": 57.79, + "step": 17050 + }, + { + "epoch": 27.472413966846375, + "grad_norm": 0.7800275683403015, + "learning_rate": 0.0001, + "loss": 60.0955, + "step": 17060 + }, + { + "epoch": 27.48853731042475, + "grad_norm": 0.5514053702354431, + "learning_rate": 0.0001, + "loss": 59.2458, + "step": 17070 + }, + { + "epoch": 27.504660654003125, + "grad_norm": 0.6667745113372803, + "learning_rate": 0.0001, + "loss": 59.872, + "step": 17080 + }, + { + "epoch": 27.5207839975815, + "grad_norm": 0.6694201231002808, + "learning_rate": 0.0001, + "loss": 59.4699, + "step": 17090 + }, + { + "epoch": 27.536907341159875, + "grad_norm": 0.721091091632843, + "learning_rate": 0.0001, + "loss": 59.8662, + "step": 17100 + }, + { + "epoch": 27.553030684738246, + "grad_norm": 0.8182216882705688, + "learning_rate": 0.0001, + "loss": 57.3634, + "step": 17110 + }, + { + "epoch": 27.56915402831662, + "grad_norm": 1.0078271627426147, + "learning_rate": 0.0001, + "loss": 59.3628, + "step": 17120 + }, + { + "epoch": 27.585277371894996, + "grad_norm": 0.6158314347267151, + "learning_rate": 0.0001, + "loss": 59.123, + "step": 17130 + }, + { + "epoch": 27.60140071547337, + "grad_norm": 0.5983971357345581, + "learning_rate": 0.0001, + "loss": 58.6259, + "step": 17140 + }, + { + "epoch": 27.617524059051746, + "grad_norm": 0.4854629337787628, + "learning_rate": 0.0001, + "loss": 58.781, + "step": 17150 + }, + { + "epoch": 27.63364740263012, + "grad_norm": 0.582637369632721, + "learning_rate": 0.0001, + "loss": 59.6194, + "step": 17160 + }, + { + "epoch": 27.649770746208496, + "grad_norm": 0.7129100561141968, + "learning_rate": 0.0001, + "loss": 58.495, + "step": 17170 + }, + { + "epoch": 27.66589408978687, + "grad_norm": 0.8179757595062256, + "learning_rate": 0.0001, + "loss": 58.7483, + "step": 17180 + }, + { + "epoch": 27.682017433365242, + "grad_norm": 0.7066773772239685, + "learning_rate": 0.0001, + "loss": 58.2444, + "step": 17190 + }, + { + "epoch": 27.698140776943617, + "grad_norm": 0.725228488445282, + "learning_rate": 0.0001, + "loss": 58.0686, + "step": 17200 + }, + { + "epoch": 27.714264120521992, + "grad_norm": 0.630705714225769, + "learning_rate": 0.0001, + "loss": 56.873, + "step": 17210 + }, + { + "epoch": 27.730387464100367, + "grad_norm": 0.7248587608337402, + "learning_rate": 0.0001, + "loss": 59.6893, + "step": 17220 + }, + { + "epoch": 27.746510807678742, + "grad_norm": 0.715254545211792, + "learning_rate": 0.0001, + "loss": 57.7157, + "step": 17230 + }, + { + "epoch": 27.762634151257117, + "grad_norm": 0.6525955200195312, + "learning_rate": 0.0001, + "loss": 58.6077, + "step": 17240 + }, + { + "epoch": 27.778757494835492, + "grad_norm": 0.7938500642776489, + "learning_rate": 0.0001, + "loss": 60.325, + "step": 17250 + }, + { + "epoch": 27.794880838413867, + "grad_norm": 0.6821472644805908, + "learning_rate": 0.0001, + "loss": 56.5372, + "step": 17260 + }, + { + "epoch": 27.811004181992242, + "grad_norm": 0.5398596525192261, + "learning_rate": 0.0001, + "loss": 57.433, + "step": 17270 + }, + { + "epoch": 27.827127525570614, + "grad_norm": 0.6805903911590576, + "learning_rate": 0.0001, + "loss": 58.6501, + "step": 17280 + }, + { + "epoch": 27.84325086914899, + "grad_norm": 0.6547013521194458, + "learning_rate": 0.0001, + "loss": 57.9083, + "step": 17290 + }, + { + "epoch": 27.859374212727364, + "grad_norm": 0.6482307314872742, + "learning_rate": 0.0001, + "loss": 59.2978, + "step": 17300 + }, + { + "epoch": 27.87549755630574, + "grad_norm": 0.867108941078186, + "learning_rate": 0.0001, + "loss": 59.452, + "step": 17310 + }, + { + "epoch": 27.891620899884114, + "grad_norm": 0.5809998512268066, + "learning_rate": 0.0001, + "loss": 59.0863, + "step": 17320 + }, + { + "epoch": 27.90774424346249, + "grad_norm": 0.7422227263450623, + "learning_rate": 0.0001, + "loss": 58.7768, + "step": 17330 + }, + { + "epoch": 27.923867587040863, + "grad_norm": 0.6092364192008972, + "learning_rate": 0.0001, + "loss": 58.1772, + "step": 17340 + }, + { + "epoch": 27.93999093061924, + "grad_norm": 0.688312292098999, + "learning_rate": 0.0001, + "loss": 58.5475, + "step": 17350 + }, + { + "epoch": 27.956114274197613, + "grad_norm": 0.8303641080856323, + "learning_rate": 0.0001, + "loss": 59.8533, + "step": 17360 + }, + { + "epoch": 27.972237617775985, + "grad_norm": 0.903464138507843, + "learning_rate": 0.0001, + "loss": 58.5677, + "step": 17370 + }, + { + "epoch": 27.98836096135436, + "grad_norm": 0.7864969968795776, + "learning_rate": 0.0001, + "loss": 59.1479, + "step": 17380 + }, + { + "epoch": 28.003224668715674, + "grad_norm": 0.7318345904350281, + "learning_rate": 0.0001, + "loss": 56.2894, + "step": 17390 + }, + { + "epoch": 28.01934801229405, + "grad_norm": 0.6740039587020874, + "learning_rate": 0.0001, + "loss": 60.3117, + "step": 17400 + }, + { + "epoch": 28.035471355872424, + "grad_norm": 0.6491691470146179, + "learning_rate": 0.0001, + "loss": 59.2796, + "step": 17410 + }, + { + "epoch": 28.0515946994508, + "grad_norm": 0.8077054619789124, + "learning_rate": 0.0001, + "loss": 60.4482, + "step": 17420 + }, + { + "epoch": 28.067718043029174, + "grad_norm": 0.8239777088165283, + "learning_rate": 0.0001, + "loss": 58.5352, + "step": 17430 + }, + { + "epoch": 28.08384138660755, + "grad_norm": 0.6907490491867065, + "learning_rate": 0.0001, + "loss": 60.3665, + "step": 17440 + }, + { + "epoch": 28.09996473018592, + "grad_norm": 0.7226580381393433, + "learning_rate": 0.0001, + "loss": 58.3535, + "step": 17450 + }, + { + "epoch": 28.116088073764296, + "grad_norm": 0.5712003707885742, + "learning_rate": 0.0001, + "loss": 57.8514, + "step": 17460 + }, + { + "epoch": 28.13221141734267, + "grad_norm": 0.532891571521759, + "learning_rate": 0.0001, + "loss": 57.5121, + "step": 17470 + }, + { + "epoch": 28.148334760921045, + "grad_norm": 0.6007943153381348, + "learning_rate": 0.0001, + "loss": 59.1072, + "step": 17480 + }, + { + "epoch": 28.16445810449942, + "grad_norm": 0.9194851517677307, + "learning_rate": 0.0001, + "loss": 59.8352, + "step": 17490 + }, + { + "epoch": 28.180581448077795, + "grad_norm": 0.5962793827056885, + "learning_rate": 0.0001, + "loss": 58.8552, + "step": 17500 + }, + { + "epoch": 28.19670479165617, + "grad_norm": 0.6472097635269165, + "learning_rate": 0.0001, + "loss": 60.072, + "step": 17510 + }, + { + "epoch": 28.212828135234545, + "grad_norm": 0.7954792976379395, + "learning_rate": 0.0001, + "loss": 57.754, + "step": 17520 + }, + { + "epoch": 28.22895147881292, + "grad_norm": 0.5923368334770203, + "learning_rate": 0.0001, + "loss": 56.9673, + "step": 17530 + }, + { + "epoch": 28.24507482239129, + "grad_norm": 1.0150458812713623, + "learning_rate": 0.0001, + "loss": 57.1309, + "step": 17540 + }, + { + "epoch": 28.261198165969667, + "grad_norm": 0.597544252872467, + "learning_rate": 0.0001, + "loss": 58.6526, + "step": 17550 + }, + { + "epoch": 28.27732150954804, + "grad_norm": 0.8652740716934204, + "learning_rate": 0.0001, + "loss": 58.7396, + "step": 17560 + }, + { + "epoch": 28.293444853126417, + "grad_norm": 0.8436110615730286, + "learning_rate": 0.0001, + "loss": 59.0186, + "step": 17570 + }, + { + "epoch": 28.30956819670479, + "grad_norm": 0.6380386352539062, + "learning_rate": 0.0001, + "loss": 58.3723, + "step": 17580 + }, + { + "epoch": 28.325691540283167, + "grad_norm": 0.6668462157249451, + "learning_rate": 0.0001, + "loss": 58.0738, + "step": 17590 + }, + { + "epoch": 28.34181488386154, + "grad_norm": 0.7173913717269897, + "learning_rate": 0.0001, + "loss": 58.1463, + "step": 17600 + }, + { + "epoch": 28.357938227439917, + "grad_norm": 0.6073333621025085, + "learning_rate": 0.0001, + "loss": 58.0279, + "step": 17610 + }, + { + "epoch": 28.37406157101829, + "grad_norm": 0.6092989444732666, + "learning_rate": 0.0001, + "loss": 56.1147, + "step": 17620 + }, + { + "epoch": 28.390184914596663, + "grad_norm": 0.7214272618293762, + "learning_rate": 0.0001, + "loss": 58.1375, + "step": 17630 + }, + { + "epoch": 28.406308258175038, + "grad_norm": 0.5026358366012573, + "learning_rate": 0.0001, + "loss": 59.2814, + "step": 17640 + }, + { + "epoch": 28.422431601753413, + "grad_norm": 0.6362181901931763, + "learning_rate": 0.0001, + "loss": 60.6815, + "step": 17650 + }, + { + "epoch": 28.438554945331788, + "grad_norm": 0.7994716167449951, + "learning_rate": 0.0001, + "loss": 58.4757, + "step": 17660 + }, + { + "epoch": 28.454678288910163, + "grad_norm": 0.6012625694274902, + "learning_rate": 0.0001, + "loss": 58.1216, + "step": 17670 + }, + { + "epoch": 28.470801632488538, + "grad_norm": 0.5765920877456665, + "learning_rate": 0.0001, + "loss": 60.1921, + "step": 17680 + }, + { + "epoch": 28.486924976066913, + "grad_norm": 0.6323729753494263, + "learning_rate": 0.0001, + "loss": 59.5405, + "step": 17690 + }, + { + "epoch": 28.503048319645288, + "grad_norm": 0.6641665697097778, + "learning_rate": 0.0001, + "loss": 59.3311, + "step": 17700 + }, + { + "epoch": 28.519171663223663, + "grad_norm": 0.7021269202232361, + "learning_rate": 0.0001, + "loss": 58.3317, + "step": 17710 + }, + { + "epoch": 28.535295006802034, + "grad_norm": 0.6319525241851807, + "learning_rate": 0.0001, + "loss": 57.4676, + "step": 17720 + }, + { + "epoch": 28.55141835038041, + "grad_norm": 0.6366249322891235, + "learning_rate": 0.0001, + "loss": 60.761, + "step": 17730 + }, + { + "epoch": 28.567541693958784, + "grad_norm": 1.487281322479248, + "learning_rate": 0.0001, + "loss": 58.2667, + "step": 17740 + }, + { + "epoch": 28.58366503753716, + "grad_norm": 0.6286079287528992, + "learning_rate": 0.0001, + "loss": 56.5199, + "step": 17750 + }, + { + "epoch": 28.599788381115534, + "grad_norm": 0.6458479762077332, + "learning_rate": 0.0001, + "loss": 59.0782, + "step": 17760 + }, + { + "epoch": 28.61591172469391, + "grad_norm": 0.6947740316390991, + "learning_rate": 0.0001, + "loss": 61.1563, + "step": 17770 + }, + { + "epoch": 28.632035068272284, + "grad_norm": 1.3151084184646606, + "learning_rate": 0.0001, + "loss": 58.7798, + "step": 17780 + }, + { + "epoch": 28.64815841185066, + "grad_norm": 0.5333471894264221, + "learning_rate": 0.0001, + "loss": 58.1223, + "step": 17790 + }, + { + "epoch": 28.66428175542903, + "grad_norm": 0.6268185973167419, + "learning_rate": 0.0001, + "loss": 58.9371, + "step": 17800 + }, + { + "epoch": 28.680405099007405, + "grad_norm": 0.5982078313827515, + "learning_rate": 0.0001, + "loss": 57.9883, + "step": 17810 + }, + { + "epoch": 28.69652844258578, + "grad_norm": 0.5744349360466003, + "learning_rate": 0.0001, + "loss": 59.4091, + "step": 17820 + }, + { + "epoch": 28.712651786164155, + "grad_norm": 0.7235150337219238, + "learning_rate": 0.0001, + "loss": 58.1419, + "step": 17830 + }, + { + "epoch": 28.72877512974253, + "grad_norm": 0.5630679726600647, + "learning_rate": 0.0001, + "loss": 59.547, + "step": 17840 + }, + { + "epoch": 28.744898473320905, + "grad_norm": 0.854364812374115, + "learning_rate": 0.0001, + "loss": 58.413, + "step": 17850 + }, + { + "epoch": 28.76102181689928, + "grad_norm": 0.6486361622810364, + "learning_rate": 0.0001, + "loss": 58.0385, + "step": 17860 + }, + { + "epoch": 28.777145160477655, + "grad_norm": 0.5967833995819092, + "learning_rate": 0.0001, + "loss": 57.4478, + "step": 17870 + }, + { + "epoch": 28.79326850405603, + "grad_norm": 0.6199812889099121, + "learning_rate": 0.0001, + "loss": 56.8104, + "step": 17880 + }, + { + "epoch": 28.8093918476344, + "grad_norm": 0.6966105699539185, + "learning_rate": 0.0001, + "loss": 59.4416, + "step": 17890 + }, + { + "epoch": 28.825515191212777, + "grad_norm": 0.7352570295333862, + "learning_rate": 0.0001, + "loss": 58.9475, + "step": 17900 + }, + { + "epoch": 28.84163853479115, + "grad_norm": 0.7693241238594055, + "learning_rate": 0.0001, + "loss": 57.4324, + "step": 17910 + }, + { + "epoch": 28.857761878369526, + "grad_norm": 1.0005452632904053, + "learning_rate": 0.0001, + "loss": 59.0267, + "step": 17920 + }, + { + "epoch": 28.8738852219479, + "grad_norm": 0.6526029109954834, + "learning_rate": 0.0001, + "loss": 57.7836, + "step": 17930 + }, + { + "epoch": 28.890008565526276, + "grad_norm": 0.7833093404769897, + "learning_rate": 0.0001, + "loss": 59.3319, + "step": 17940 + }, + { + "epoch": 28.90613190910465, + "grad_norm": 0.6504502892494202, + "learning_rate": 0.0001, + "loss": 57.5452, + "step": 17950 + }, + { + "epoch": 28.922255252683026, + "grad_norm": 0.5760630369186401, + "learning_rate": 0.0001, + "loss": 56.2115, + "step": 17960 + }, + { + "epoch": 28.9383785962614, + "grad_norm": 0.7640287280082703, + "learning_rate": 0.0001, + "loss": 59.051, + "step": 17970 + }, + { + "epoch": 28.954501939839773, + "grad_norm": 0.6596423983573914, + "learning_rate": 0.0001, + "loss": 59.4821, + "step": 17980 + }, + { + "epoch": 28.970625283418148, + "grad_norm": 2.1421217918395996, + "learning_rate": 0.0001, + "loss": 58.2994, + "step": 17990 + }, + { + "epoch": 28.986748626996523, + "grad_norm": 0.6420243382453918, + "learning_rate": 0.0001, + "loss": 55.9762, + "step": 18000 + }, + { + "epoch": 29.001612334357837, + "grad_norm": 0.6530152559280396, + "learning_rate": 0.0001, + "loss": 52.9146, + "step": 18010 + }, + { + "epoch": 29.017735677936212, + "grad_norm": 0.6559716463088989, + "learning_rate": 0.0001, + "loss": 57.5697, + "step": 18020 + }, + { + "epoch": 29.033859021514587, + "grad_norm": 0.7522405982017517, + "learning_rate": 0.0001, + "loss": 58.5981, + "step": 18030 + }, + { + "epoch": 29.049982365092962, + "grad_norm": 0.7637670636177063, + "learning_rate": 0.0001, + "loss": 58.4835, + "step": 18040 + }, + { + "epoch": 29.066105708671337, + "grad_norm": 0.6451973915100098, + "learning_rate": 0.0001, + "loss": 58.5291, + "step": 18050 + }, + { + "epoch": 29.082229052249712, + "grad_norm": 0.6325218081474304, + "learning_rate": 0.0001, + "loss": 57.3294, + "step": 18060 + }, + { + "epoch": 29.098352395828083, + "grad_norm": 0.7488126754760742, + "learning_rate": 0.0001, + "loss": 59.6257, + "step": 18070 + }, + { + "epoch": 29.11447573940646, + "grad_norm": 0.6828444004058838, + "learning_rate": 0.0001, + "loss": 58.4092, + "step": 18080 + }, + { + "epoch": 29.130599082984833, + "grad_norm": 0.6108933687210083, + "learning_rate": 0.0001, + "loss": 60.2422, + "step": 18090 + }, + { + "epoch": 29.14672242656321, + "grad_norm": 0.640813410282135, + "learning_rate": 0.0001, + "loss": 59.8748, + "step": 18100 + }, + { + "epoch": 29.162845770141583, + "grad_norm": 0.6216452121734619, + "learning_rate": 0.0001, + "loss": 57.5762, + "step": 18110 + }, + { + "epoch": 29.17896911371996, + "grad_norm": 0.5484854578971863, + "learning_rate": 0.0001, + "loss": 57.5171, + "step": 18120 + }, + { + "epoch": 29.195092457298333, + "grad_norm": 0.6442582011222839, + "learning_rate": 0.0001, + "loss": 60.7324, + "step": 18130 + }, + { + "epoch": 29.211215800876708, + "grad_norm": 0.6196489334106445, + "learning_rate": 0.0001, + "loss": 58.6092, + "step": 18140 + }, + { + "epoch": 29.22733914445508, + "grad_norm": 0.6655876636505127, + "learning_rate": 0.0001, + "loss": 56.9686, + "step": 18150 + }, + { + "epoch": 29.243462488033455, + "grad_norm": 5.002511978149414, + "learning_rate": 0.0001, + "loss": 59.1202, + "step": 18160 + }, + { + "epoch": 29.25958583161183, + "grad_norm": 0.5576016902923584, + "learning_rate": 0.0001, + "loss": 57.3112, + "step": 18170 + }, + { + "epoch": 29.275709175190205, + "grad_norm": 0.6420336961746216, + "learning_rate": 0.0001, + "loss": 58.2884, + "step": 18180 + }, + { + "epoch": 29.29183251876858, + "grad_norm": 0.7113134860992432, + "learning_rate": 0.0001, + "loss": 58.9768, + "step": 18190 + }, + { + "epoch": 29.307955862346954, + "grad_norm": 0.7830891609191895, + "learning_rate": 0.0001, + "loss": 58.2999, + "step": 18200 + }, + { + "epoch": 29.32407920592533, + "grad_norm": 0.7687374353408813, + "learning_rate": 0.0001, + "loss": 60.3956, + "step": 18210 + }, + { + "epoch": 29.340202549503704, + "grad_norm": 0.596026599407196, + "learning_rate": 0.0001, + "loss": 56.5389, + "step": 18220 + }, + { + "epoch": 29.35632589308208, + "grad_norm": 0.6291328072547913, + "learning_rate": 0.0001, + "loss": 56.752, + "step": 18230 + }, + { + "epoch": 29.37244923666045, + "grad_norm": 0.8147439360618591, + "learning_rate": 0.0001, + "loss": 57.7872, + "step": 18240 + }, + { + "epoch": 29.388572580238826, + "grad_norm": 0.8024112582206726, + "learning_rate": 0.0001, + "loss": 58.2364, + "step": 18250 + }, + { + "epoch": 29.4046959238172, + "grad_norm": 0.6654099822044373, + "learning_rate": 0.0001, + "loss": 57.1549, + "step": 18260 + }, + { + "epoch": 29.420819267395576, + "grad_norm": 0.7781001329421997, + "learning_rate": 0.0001, + "loss": 57.8705, + "step": 18270 + }, + { + "epoch": 29.43694261097395, + "grad_norm": 0.699664294719696, + "learning_rate": 0.0001, + "loss": 59.2756, + "step": 18280 + }, + { + "epoch": 29.453065954552326, + "grad_norm": 0.6834261417388916, + "learning_rate": 0.0001, + "loss": 57.0605, + "step": 18290 + }, + { + "epoch": 29.4691892981307, + "grad_norm": 2.225274085998535, + "learning_rate": 0.0001, + "loss": 58.5602, + "step": 18300 + }, + { + "epoch": 29.485312641709076, + "grad_norm": 0.8640811443328857, + "learning_rate": 0.0001, + "loss": 57.6847, + "step": 18310 + }, + { + "epoch": 29.50143598528745, + "grad_norm": 0.7287184000015259, + "learning_rate": 0.0001, + "loss": 58.4448, + "step": 18320 + }, + { + "epoch": 29.517559328865822, + "grad_norm": 0.6942048668861389, + "learning_rate": 0.0001, + "loss": 58.3656, + "step": 18330 + }, + { + "epoch": 29.533682672444197, + "grad_norm": 0.8364513516426086, + "learning_rate": 0.0001, + "loss": 57.3624, + "step": 18340 + }, + { + "epoch": 29.549806016022572, + "grad_norm": 0.9943494200706482, + "learning_rate": 0.0001, + "loss": 58.2563, + "step": 18350 + }, + { + "epoch": 29.565929359600947, + "grad_norm": 0.6764407753944397, + "learning_rate": 0.0001, + "loss": 58.0796, + "step": 18360 + }, + { + "epoch": 29.582052703179322, + "grad_norm": 0.6659194827079773, + "learning_rate": 0.0001, + "loss": 60.181, + "step": 18370 + }, + { + "epoch": 29.598176046757697, + "grad_norm": 0.5525355339050293, + "learning_rate": 0.0001, + "loss": 57.7179, + "step": 18380 + }, + { + "epoch": 29.614299390336072, + "grad_norm": 0.6220526099205017, + "learning_rate": 0.0001, + "loss": 57.5135, + "step": 18390 + }, + { + "epoch": 29.630422733914447, + "grad_norm": 0.5199670195579529, + "learning_rate": 0.0001, + "loss": 59.3721, + "step": 18400 + }, + { + "epoch": 29.646546077492822, + "grad_norm": 0.6937117576599121, + "learning_rate": 0.0001, + "loss": 58.4896, + "step": 18410 + }, + { + "epoch": 29.662669421071193, + "grad_norm": 0.6366041302680969, + "learning_rate": 0.0001, + "loss": 59.1487, + "step": 18420 + }, + { + "epoch": 29.678792764649568, + "grad_norm": 0.520003080368042, + "learning_rate": 0.0001, + "loss": 58.8141, + "step": 18430 + }, + { + "epoch": 29.694916108227943, + "grad_norm": 1.3317536115646362, + "learning_rate": 0.0001, + "loss": 56.8534, + "step": 18440 + }, + { + "epoch": 29.711039451806318, + "grad_norm": 0.7395436763763428, + "learning_rate": 0.0001, + "loss": 59.7522, + "step": 18450 + }, + { + "epoch": 29.727162795384693, + "grad_norm": 0.5532745122909546, + "learning_rate": 0.0001, + "loss": 58.2614, + "step": 18460 + }, + { + "epoch": 29.743286138963068, + "grad_norm": 0.7754526138305664, + "learning_rate": 0.0001, + "loss": 59.1621, + "step": 18470 + }, + { + "epoch": 29.759409482541443, + "grad_norm": 0.6477323174476624, + "learning_rate": 0.0001, + "loss": 59.1955, + "step": 18480 + }, + { + "epoch": 29.775532826119818, + "grad_norm": 0.8694291710853577, + "learning_rate": 0.0001, + "loss": 57.7143, + "step": 18490 + }, + { + "epoch": 29.791656169698193, + "grad_norm": 0.6923109292984009, + "learning_rate": 0.0001, + "loss": 57.9599, + "step": 18500 + }, + { + "epoch": 29.807779513276564, + "grad_norm": 0.7835638523101807, + "learning_rate": 0.0001, + "loss": 59.1726, + "step": 18510 + }, + { + "epoch": 29.82390285685494, + "grad_norm": 0.7412428855895996, + "learning_rate": 0.0001, + "loss": 57.7021, + "step": 18520 + }, + { + "epoch": 29.840026200433314, + "grad_norm": 0.7160241007804871, + "learning_rate": 0.0001, + "loss": 57.755, + "step": 18530 + }, + { + "epoch": 29.85614954401169, + "grad_norm": 0.7844975590705872, + "learning_rate": 0.0001, + "loss": 58.6039, + "step": 18540 + }, + { + "epoch": 29.872272887590064, + "grad_norm": 0.8730555772781372, + "learning_rate": 0.0001, + "loss": 57.8274, + "step": 18550 + }, + { + "epoch": 29.88839623116844, + "grad_norm": 0.6070888042449951, + "learning_rate": 0.0001, + "loss": 57.646, + "step": 18560 + }, + { + "epoch": 29.904519574746814, + "grad_norm": 0.7387932538986206, + "learning_rate": 0.0001, + "loss": 58.7126, + "step": 18570 + }, + { + "epoch": 29.92064291832519, + "grad_norm": 0.6337044835090637, + "learning_rate": 0.0001, + "loss": 59.0599, + "step": 18580 + }, + { + "epoch": 29.93676626190356, + "grad_norm": 0.7069441676139832, + "learning_rate": 0.0001, + "loss": 58.3129, + "step": 18590 + }, + { + "epoch": 29.952889605481936, + "grad_norm": 0.6618002653121948, + "learning_rate": 0.0001, + "loss": 58.0288, + "step": 18600 + }, + { + "epoch": 29.96901294906031, + "grad_norm": 0.6438560485839844, + "learning_rate": 0.0001, + "loss": 58.4161, + "step": 18610 + }, + { + "epoch": 29.985136292638686, + "grad_norm": 0.7716667056083679, + "learning_rate": 0.0001, + "loss": 57.9302, + "step": 18620 + }, + { + "epoch": 30.0, + "grad_norm": 0.6670845746994019, + "learning_rate": 0.0001, + "loss": 53.8077, + "step": 18630 + }, + { + "epoch": 30.016123343578375, + "grad_norm": 0.6700365543365479, + "learning_rate": 0.0001, + "loss": 58.1248, + "step": 18640 + }, + { + "epoch": 30.03224668715675, + "grad_norm": 0.5925083756446838, + "learning_rate": 0.0001, + "loss": 58.0006, + "step": 18650 + }, + { + "epoch": 30.048370030735125, + "grad_norm": 0.5727967023849487, + "learning_rate": 0.0001, + "loss": 57.9431, + "step": 18660 + }, + { + "epoch": 30.0644933743135, + "grad_norm": 0.4864082634449005, + "learning_rate": 0.0001, + "loss": 56.5658, + "step": 18670 + }, + { + "epoch": 30.08061671789187, + "grad_norm": 0.8846231698989868, + "learning_rate": 0.0001, + "loss": 57.087, + "step": 18680 + }, + { + "epoch": 30.096740061470246, + "grad_norm": 0.6806173920631409, + "learning_rate": 0.0001, + "loss": 58.3673, + "step": 18690 + }, + { + "epoch": 30.11286340504862, + "grad_norm": 0.6638009548187256, + "learning_rate": 0.0001, + "loss": 59.5407, + "step": 18700 + }, + { + "epoch": 30.128986748626996, + "grad_norm": 0.7844518423080444, + "learning_rate": 0.0001, + "loss": 58.3762, + "step": 18710 + }, + { + "epoch": 30.14511009220537, + "grad_norm": 0.6602208018302917, + "learning_rate": 0.0001, + "loss": 58.3672, + "step": 18720 + }, + { + "epoch": 30.161233435783746, + "grad_norm": 0.5882258415222168, + "learning_rate": 0.0001, + "loss": 58.5334, + "step": 18730 + }, + { + "epoch": 30.17735677936212, + "grad_norm": 0.7399165034294128, + "learning_rate": 0.0001, + "loss": 59.8635, + "step": 18740 + }, + { + "epoch": 30.193480122940496, + "grad_norm": 0.6812124848365784, + "learning_rate": 0.0001, + "loss": 57.8219, + "step": 18750 + }, + { + "epoch": 30.20960346651887, + "grad_norm": 0.5468288660049438, + "learning_rate": 0.0001, + "loss": 57.7236, + "step": 18760 + }, + { + "epoch": 30.225726810097242, + "grad_norm": 0.6843936443328857, + "learning_rate": 0.0001, + "loss": 58.7141, + "step": 18770 + }, + { + "epoch": 30.241850153675617, + "grad_norm": 0.6243616342544556, + "learning_rate": 0.0001, + "loss": 59.791, + "step": 18780 + }, + { + "epoch": 30.257973497253992, + "grad_norm": 0.6359348893165588, + "learning_rate": 0.0001, + "loss": 57.3614, + "step": 18790 + }, + { + "epoch": 30.274096840832367, + "grad_norm": 1.4484305381774902, + "learning_rate": 0.0001, + "loss": 57.3209, + "step": 18800 + }, + { + "epoch": 30.290220184410742, + "grad_norm": 0.594723105430603, + "learning_rate": 0.0001, + "loss": 59.6517, + "step": 18810 + }, + { + "epoch": 30.306343527989117, + "grad_norm": 0.7364279627799988, + "learning_rate": 0.0001, + "loss": 60.1228, + "step": 18820 + }, + { + "epoch": 30.322466871567492, + "grad_norm": 0.7273280024528503, + "learning_rate": 0.0001, + "loss": 57.3952, + "step": 18830 + }, + { + "epoch": 30.338590215145867, + "grad_norm": 0.5930690169334412, + "learning_rate": 0.0001, + "loss": 57.9403, + "step": 18840 + }, + { + "epoch": 30.35471355872424, + "grad_norm": 0.6662129759788513, + "learning_rate": 0.0001, + "loss": 57.2412, + "step": 18850 + }, + { + "epoch": 30.370836902302614, + "grad_norm": 0.7068710923194885, + "learning_rate": 0.0001, + "loss": 56.4046, + "step": 18860 + }, + { + "epoch": 30.38696024588099, + "grad_norm": 0.9942213892936707, + "learning_rate": 0.0001, + "loss": 58.0625, + "step": 18870 + }, + { + "epoch": 30.403083589459364, + "grad_norm": 0.6646976470947266, + "learning_rate": 0.0001, + "loss": 58.0701, + "step": 18880 + }, + { + "epoch": 30.41920693303774, + "grad_norm": 0.7813248038291931, + "learning_rate": 0.0001, + "loss": 56.0009, + "step": 18890 + }, + { + "epoch": 30.435330276616114, + "grad_norm": 0.5491666197776794, + "learning_rate": 0.0001, + "loss": 57.3629, + "step": 18900 + }, + { + "epoch": 30.45145362019449, + "grad_norm": 0.6482571363449097, + "learning_rate": 0.0001, + "loss": 58.9482, + "step": 18910 + }, + { + "epoch": 30.467576963772864, + "grad_norm": 0.8382819890975952, + "learning_rate": 0.0001, + "loss": 57.3887, + "step": 18920 + }, + { + "epoch": 30.48370030735124, + "grad_norm": 0.7013330459594727, + "learning_rate": 0.0001, + "loss": 57.3789, + "step": 18930 + }, + { + "epoch": 30.49982365092961, + "grad_norm": 0.5680707693099976, + "learning_rate": 0.0001, + "loss": 57.2686, + "step": 18940 + }, + { + "epoch": 30.515946994507985, + "grad_norm": 0.5723867416381836, + "learning_rate": 0.0001, + "loss": 56.8546, + "step": 18950 + }, + { + "epoch": 30.53207033808636, + "grad_norm": 0.6836649775505066, + "learning_rate": 0.0001, + "loss": 58.8521, + "step": 18960 + }, + { + "epoch": 30.548193681664735, + "grad_norm": 0.6368657946586609, + "learning_rate": 0.0001, + "loss": 58.5553, + "step": 18970 + }, + { + "epoch": 30.56431702524311, + "grad_norm": 0.8456745743751526, + "learning_rate": 0.0001, + "loss": 58.0349, + "step": 18980 + }, + { + "epoch": 30.580440368821485, + "grad_norm": 0.7769978046417236, + "learning_rate": 0.0001, + "loss": 59.5172, + "step": 18990 + }, + { + "epoch": 30.59656371239986, + "grad_norm": 0.6169134974479675, + "learning_rate": 0.0001, + "loss": 57.8756, + "step": 19000 + }, + { + "epoch": 30.612687055978235, + "grad_norm": 0.6255633234977722, + "learning_rate": 0.0001, + "loss": 58.3724, + "step": 19010 + }, + { + "epoch": 30.62881039955661, + "grad_norm": 0.6262527704238892, + "learning_rate": 0.0001, + "loss": 59.6478, + "step": 19020 + }, + { + "epoch": 30.64493374313498, + "grad_norm": 0.9178198575973511, + "learning_rate": 0.0001, + "loss": 59.4106, + "step": 19030 + }, + { + "epoch": 30.661057086713356, + "grad_norm": 0.5433737635612488, + "learning_rate": 0.0001, + "loss": 57.9312, + "step": 19040 + }, + { + "epoch": 30.67718043029173, + "grad_norm": 0.6306186318397522, + "learning_rate": 0.0001, + "loss": 57.9233, + "step": 19050 + }, + { + "epoch": 30.693303773870106, + "grad_norm": 0.5454888939857483, + "learning_rate": 0.0001, + "loss": 58.1754, + "step": 19060 + }, + { + "epoch": 30.70942711744848, + "grad_norm": 0.53844153881073, + "learning_rate": 0.0001, + "loss": 59.1227, + "step": 19070 + }, + { + "epoch": 30.725550461026856, + "grad_norm": 0.7765403985977173, + "learning_rate": 0.0001, + "loss": 56.9315, + "step": 19080 + }, + { + "epoch": 30.74167380460523, + "grad_norm": 0.657633900642395, + "learning_rate": 0.0001, + "loss": 57.5241, + "step": 19090 + }, + { + "epoch": 30.757797148183606, + "grad_norm": 0.6395857334136963, + "learning_rate": 0.0001, + "loss": 58.8294, + "step": 19100 + }, + { + "epoch": 30.77392049176198, + "grad_norm": 0.679739236831665, + "learning_rate": 0.0001, + "loss": 58.0586, + "step": 19110 + }, + { + "epoch": 30.790043835340352, + "grad_norm": 0.5583140850067139, + "learning_rate": 0.0001, + "loss": 57.604, + "step": 19120 + }, + { + "epoch": 30.806167178918727, + "grad_norm": 1.0134044885635376, + "learning_rate": 0.0001, + "loss": 59.1611, + "step": 19130 + }, + { + "epoch": 30.822290522497102, + "grad_norm": 0.6906328797340393, + "learning_rate": 0.0001, + "loss": 57.9054, + "step": 19140 + }, + { + "epoch": 30.838413866075477, + "grad_norm": 0.6082453727722168, + "learning_rate": 0.0001, + "loss": 57.1707, + "step": 19150 + }, + { + "epoch": 30.854537209653852, + "grad_norm": 0.7322260737419128, + "learning_rate": 0.0001, + "loss": 57.4455, + "step": 19160 + }, + { + "epoch": 30.870660553232227, + "grad_norm": 1.7016369104385376, + "learning_rate": 0.0001, + "loss": 57.2955, + "step": 19170 + }, + { + "epoch": 30.886783896810602, + "grad_norm": 0.6822830438613892, + "learning_rate": 0.0001, + "loss": 57.1203, + "step": 19180 + }, + { + "epoch": 30.902907240388977, + "grad_norm": 0.6205664277076721, + "learning_rate": 0.0001, + "loss": 57.191, + "step": 19190 + }, + { + "epoch": 30.91903058396735, + "grad_norm": 0.6766132712364197, + "learning_rate": 0.0001, + "loss": 58.2869, + "step": 19200 + }, + { + "epoch": 30.935153927545723, + "grad_norm": 0.6686915755271912, + "learning_rate": 0.0001, + "loss": 57.9603, + "step": 19210 + }, + { + "epoch": 30.9512772711241, + "grad_norm": 0.691638171672821, + "learning_rate": 0.0001, + "loss": 58.5623, + "step": 19220 + }, + { + "epoch": 30.967400614702473, + "grad_norm": 0.5663546919822693, + "learning_rate": 0.0001, + "loss": 58.4948, + "step": 19230 + }, + { + "epoch": 30.98352395828085, + "grad_norm": 0.6469123363494873, + "learning_rate": 0.0001, + "loss": 59.2639, + "step": 19240 + }, + { + "epoch": 30.999647301859223, + "grad_norm": 0.8151462078094482, + "learning_rate": 0.0001, + "loss": 58.5958, + "step": 19250 + }, + { + "epoch": 31.014511009220538, + "grad_norm": 0.6884656548500061, + "learning_rate": 0.0001, + "loss": 52.5185, + "step": 19260 + }, + { + "epoch": 31.030634352798913, + "grad_norm": 0.7065051794052124, + "learning_rate": 0.0001, + "loss": 55.8415, + "step": 19270 + }, + { + "epoch": 31.046757696377288, + "grad_norm": 0.5404031872749329, + "learning_rate": 0.0001, + "loss": 58.5464, + "step": 19280 + }, + { + "epoch": 31.06288103995566, + "grad_norm": 0.7152573466300964, + "learning_rate": 0.0001, + "loss": 57.2573, + "step": 19290 + }, + { + "epoch": 31.079004383534034, + "grad_norm": 0.7122138142585754, + "learning_rate": 0.0001, + "loss": 59.6534, + "step": 19300 + }, + { + "epoch": 31.09512772711241, + "grad_norm": 0.633294403553009, + "learning_rate": 0.0001, + "loss": 56.2421, + "step": 19310 + }, + { + "epoch": 31.111251070690784, + "grad_norm": 0.5658999085426331, + "learning_rate": 0.0001, + "loss": 57.4885, + "step": 19320 + }, + { + "epoch": 31.12737441426916, + "grad_norm": 0.5926643013954163, + "learning_rate": 0.0001, + "loss": 58.7341, + "step": 19330 + }, + { + "epoch": 31.143497757847534, + "grad_norm": 0.6227424144744873, + "learning_rate": 0.0001, + "loss": 55.9883, + "step": 19340 + }, + { + "epoch": 31.15962110142591, + "grad_norm": 0.6378360986709595, + "learning_rate": 0.0001, + "loss": 57.8796, + "step": 19350 + }, + { + "epoch": 31.175744445004284, + "grad_norm": 0.7493644952774048, + "learning_rate": 0.0001, + "loss": 58.0501, + "step": 19360 + }, + { + "epoch": 31.19186778858266, + "grad_norm": 0.7071518898010254, + "learning_rate": 0.0001, + "loss": 57.9335, + "step": 19370 + }, + { + "epoch": 31.20799113216103, + "grad_norm": 0.6761748194694519, + "learning_rate": 0.0001, + "loss": 56.7835, + "step": 19380 + }, + { + "epoch": 31.224114475739405, + "grad_norm": 0.49391838908195496, + "learning_rate": 0.0001, + "loss": 57.4453, + "step": 19390 + }, + { + "epoch": 31.24023781931778, + "grad_norm": 0.574937641620636, + "learning_rate": 0.0001, + "loss": 60.3609, + "step": 19400 + }, + { + "epoch": 31.256361162896155, + "grad_norm": 0.6177113652229309, + "learning_rate": 0.0001, + "loss": 56.4432, + "step": 19410 + }, + { + "epoch": 31.27248450647453, + "grad_norm": 0.6551643013954163, + "learning_rate": 0.0001, + "loss": 57.0349, + "step": 19420 + }, + { + "epoch": 31.288607850052905, + "grad_norm": 0.5895516872406006, + "learning_rate": 0.0001, + "loss": 58.0312, + "step": 19430 + }, + { + "epoch": 31.30473119363128, + "grad_norm": 0.629224419593811, + "learning_rate": 0.0001, + "loss": 59.2731, + "step": 19440 + }, + { + "epoch": 31.320854537209655, + "grad_norm": 0.6345664262771606, + "learning_rate": 0.0001, + "loss": 61.5289, + "step": 19450 + }, + { + "epoch": 31.33697788078803, + "grad_norm": 0.6183397173881531, + "learning_rate": 0.0001, + "loss": 58.7228, + "step": 19460 + }, + { + "epoch": 31.3531012243664, + "grad_norm": 0.6631211638450623, + "learning_rate": 0.0001, + "loss": 58.3999, + "step": 19470 + }, + { + "epoch": 31.369224567944777, + "grad_norm": 0.698853611946106, + "learning_rate": 0.0001, + "loss": 59.6244, + "step": 19480 + }, + { + "epoch": 31.38534791152315, + "grad_norm": 0.6930462718009949, + "learning_rate": 0.0001, + "loss": 58.261, + "step": 19490 + }, + { + "epoch": 31.401471255101526, + "grad_norm": 0.6169912219047546, + "learning_rate": 0.0001, + "loss": 56.6868, + "step": 19500 + }, + { + "epoch": 31.4175945986799, + "grad_norm": 0.8206812739372253, + "learning_rate": 0.0001, + "loss": 57.3885, + "step": 19510 + }, + { + "epoch": 31.433717942258276, + "grad_norm": 0.7076541185379028, + "learning_rate": 0.0001, + "loss": 56.7746, + "step": 19520 + }, + { + "epoch": 31.44984128583665, + "grad_norm": 0.548710286617279, + "learning_rate": 0.0001, + "loss": 57.2149, + "step": 19530 + }, + { + "epoch": 31.465964629415026, + "grad_norm": 0.6735765337944031, + "learning_rate": 0.0001, + "loss": 58.8017, + "step": 19540 + }, + { + "epoch": 31.482087972993398, + "grad_norm": 0.6010057330131531, + "learning_rate": 0.0001, + "loss": 59.166, + "step": 19550 + }, + { + "epoch": 31.498211316571773, + "grad_norm": 0.5643196105957031, + "learning_rate": 0.0001, + "loss": 56.2109, + "step": 19560 + }, + { + "epoch": 31.514334660150148, + "grad_norm": 0.7801907062530518, + "learning_rate": 0.0001, + "loss": 58.0577, + "step": 19570 + }, + { + "epoch": 31.530458003728523, + "grad_norm": 0.630387544631958, + "learning_rate": 0.0001, + "loss": 56.1545, + "step": 19580 + }, + { + "epoch": 31.546581347306898, + "grad_norm": 0.6961885094642639, + "learning_rate": 0.0001, + "loss": 57.8717, + "step": 19590 + }, + { + "epoch": 31.562704690885273, + "grad_norm": 0.5340575575828552, + "learning_rate": 0.0001, + "loss": 58.2695, + "step": 19600 + }, + { + "epoch": 31.578828034463648, + "grad_norm": 0.657589316368103, + "learning_rate": 0.0001, + "loss": 59.3382, + "step": 19610 + }, + { + "epoch": 31.594951378042023, + "grad_norm": 0.6341890096664429, + "learning_rate": 0.0001, + "loss": 58.159, + "step": 19620 + }, + { + "epoch": 31.611074721620398, + "grad_norm": 0.7497854828834534, + "learning_rate": 0.0001, + "loss": 55.7069, + "step": 19630 + }, + { + "epoch": 31.62719806519877, + "grad_norm": 0.7069994211196899, + "learning_rate": 0.0001, + "loss": 57.2228, + "step": 19640 + }, + { + "epoch": 31.643321408777144, + "grad_norm": 0.7523748278617859, + "learning_rate": 0.0001, + "loss": 59.5134, + "step": 19650 + }, + { + "epoch": 31.65944475235552, + "grad_norm": 0.6233903169631958, + "learning_rate": 0.0001, + "loss": 58.5135, + "step": 19660 + }, + { + "epoch": 31.675568095933894, + "grad_norm": 0.6637253761291504, + "learning_rate": 0.0001, + "loss": 58.1763, + "step": 19670 + }, + { + "epoch": 31.69169143951227, + "grad_norm": 0.8216729760169983, + "learning_rate": 0.0001, + "loss": 60.0401, + "step": 19680 + }, + { + "epoch": 31.707814783090644, + "grad_norm": 0.534037709236145, + "learning_rate": 0.0001, + "loss": 57.4307, + "step": 19690 + }, + { + "epoch": 31.72393812666902, + "grad_norm": 0.5953571200370789, + "learning_rate": 0.0001, + "loss": 59.8216, + "step": 19700 + }, + { + "epoch": 31.740061470247394, + "grad_norm": 0.5853090286254883, + "learning_rate": 0.0001, + "loss": 58.2106, + "step": 19710 + }, + { + "epoch": 31.75618481382577, + "grad_norm": 0.7370662689208984, + "learning_rate": 0.0001, + "loss": 56.6751, + "step": 19720 + }, + { + "epoch": 31.77230815740414, + "grad_norm": 0.6396768689155579, + "learning_rate": 0.0001, + "loss": 55.3496, + "step": 19730 + }, + { + "epoch": 31.788431500982515, + "grad_norm": 0.7257981896400452, + "learning_rate": 0.0001, + "loss": 56.8753, + "step": 19740 + }, + { + "epoch": 31.80455484456089, + "grad_norm": 0.623684287071228, + "learning_rate": 0.0001, + "loss": 58.8956, + "step": 19750 + }, + { + "epoch": 31.820678188139265, + "grad_norm": 0.680181622505188, + "learning_rate": 0.0001, + "loss": 58.0901, + "step": 19760 + }, + { + "epoch": 31.83680153171764, + "grad_norm": 0.6862589120864868, + "learning_rate": 0.0001, + "loss": 55.7762, + "step": 19770 + }, + { + "epoch": 31.852924875296015, + "grad_norm": 0.5723304152488708, + "learning_rate": 0.0001, + "loss": 57.1686, + "step": 19780 + }, + { + "epoch": 31.86904821887439, + "grad_norm": 0.7448561787605286, + "learning_rate": 0.0001, + "loss": 57.7282, + "step": 19790 + }, + { + "epoch": 31.885171562452765, + "grad_norm": 0.6579545736312866, + "learning_rate": 0.0001, + "loss": 59.0573, + "step": 19800 + }, + { + "epoch": 31.90129490603114, + "grad_norm": 0.7359026670455933, + "learning_rate": 0.0001, + "loss": 56.4827, + "step": 19810 + }, + { + "epoch": 31.91741824960951, + "grad_norm": 0.5813704133033752, + "learning_rate": 0.0001, + "loss": 57.2898, + "step": 19820 + }, + { + "epoch": 31.933541593187886, + "grad_norm": 0.8108206987380981, + "learning_rate": 0.0001, + "loss": 59.701, + "step": 19830 + }, + { + "epoch": 31.94966493676626, + "grad_norm": 0.7966176271438599, + "learning_rate": 0.0001, + "loss": 58.4976, + "step": 19840 + }, + { + "epoch": 31.965788280344636, + "grad_norm": 0.7988247871398926, + "learning_rate": 0.0001, + "loss": 58.5399, + "step": 19850 + }, + { + "epoch": 31.98191162392301, + "grad_norm": 0.7580775022506714, + "learning_rate": 0.0001, + "loss": 59.3756, + "step": 19860 + }, + { + "epoch": 31.998034967501386, + "grad_norm": 0.6985690593719482, + "learning_rate": 0.0001, + "loss": 58.5244, + "step": 19870 + }, + { + "epoch": 32.0128986748627, + "grad_norm": 0.5899099111557007, + "learning_rate": 0.0001, + "loss": 51.9032, + "step": 19880 + }, + { + "epoch": 32.029022018441076, + "grad_norm": 0.5154966115951538, + "learning_rate": 0.0001, + "loss": 57.8494, + "step": 19890 + }, + { + "epoch": 32.04514536201945, + "grad_norm": 0.7414538860321045, + "learning_rate": 0.0001, + "loss": 60.1024, + "step": 19900 + }, + { + "epoch": 32.061268705597826, + "grad_norm": 0.6928898692131042, + "learning_rate": 0.0001, + "loss": 58.2986, + "step": 19910 + }, + { + "epoch": 32.0773920491762, + "grad_norm": 0.6019315719604492, + "learning_rate": 0.0001, + "loss": 58.7716, + "step": 19920 + }, + { + "epoch": 32.093515392754576, + "grad_norm": 0.5992019176483154, + "learning_rate": 0.0001, + "loss": 56.7196, + "step": 19930 + }, + { + "epoch": 32.10963873633295, + "grad_norm": 0.8731210231781006, + "learning_rate": 0.0001, + "loss": 58.7054, + "step": 19940 + }, + { + "epoch": 32.12576207991132, + "grad_norm": 0.6548051238059998, + "learning_rate": 0.0001, + "loss": 57.9742, + "step": 19950 + }, + { + "epoch": 32.1418854234897, + "grad_norm": 0.6520991325378418, + "learning_rate": 0.0001, + "loss": 58.1439, + "step": 19960 + }, + { + "epoch": 32.15800876706807, + "grad_norm": 0.7181893587112427, + "learning_rate": 0.0001, + "loss": 58.985, + "step": 19970 + }, + { + "epoch": 32.17413211064645, + "grad_norm": 0.7861577272415161, + "learning_rate": 0.0001, + "loss": 59.0184, + "step": 19980 + }, + { + "epoch": 32.19025545422482, + "grad_norm": 2.1008453369140625, + "learning_rate": 0.0001, + "loss": 57.6168, + "step": 19990 + }, + { + "epoch": 32.2063787978032, + "grad_norm": 0.7860429883003235, + "learning_rate": 0.0001, + "loss": 57.1095, + "step": 20000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 33, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.445941126059912e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ade09d5233289ba4bc5076af39b8652078582dee --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016a4e9b5d71a1947317d3801b00fe5d0604edbef1f981d7b6c92f219131717c +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1dabf96cdf7ae32cdb3a32fa950975e1f63abb17 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c276b5be6b095fe12c1c6367a32ea6415e10a0dd1e3398252aee140b52fb4871 +size 1853729968 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7567f9c671ff227d30cc0cc4d084cf846b18387b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8890cfa4791e73c97e3331b471729c8d9160cde50887cd359575d817db57a155 +size 1853730480 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e50747da27625cd8ce2daeaa0ef3515eeae8e2cf --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cf44e981f55c1c0744d0e20551bf8392cda2889f10743ba9250d23a273477a0 +size 1853731312 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a16871fd9d8d34eff550fab2de69f82f1ee0aeae --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad2d8e07fffcd28dabd485970d737dce0a57be584f7b0b34e09b9208f2aaffe1 +size 1853731120 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa323651e12fc6fc24a9530da90d4b30c680d32d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb730f945d3c0816181a54ce5890aa1b2ace8e177867a10cab278a9e04d81a2c +size 1853731312 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..816e555ad1f6187b1f8f58096b8a1375f1f33852 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4696208c96ee184de083b701a31c3a6706c19cb958c910e783932b8a85111f4 +size 1853731120 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5fbc5fca36ae88f489b3d80ae381742976e68efd --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d652509a2c67293014afe7d088fa9ed03c1dcf48b5b6a07d89784d083bbbeef +size 1853731440 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..400a7ee66e9a68610169364901226604b731abf7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f7e6d3eaf477d970becd031d40eb51ae5c1e40cde9710f6d03fd7b2c3ff0963 +size 1853731056 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/mp_rank_00_model_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..592ef39ae9c253bba2ddd604dab4eb2ddb317c9f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/global_step20001/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df8b50c8f12772883c192a335e805825f4d81c8507fa59edb9d96a13ceafdac9 +size 2471676664 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/latest b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/latest new file mode 100644 index 0000000000000000000000000000000000000000..9a2c8ed9dbbf768e0e21e8565887c4b349489762 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/latest @@ -0,0 +1 @@ +global_step20001 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..99b5928dc4b23a286b8a39a81b7ba51cb3fc3a73 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc143181971049502f06632936eea9f3c8ccfa87a6b5982687ef9ab02141409f +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..8c85d32684362c9d148f0b0aeda09b1b61a21c4d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f94cc3c0d80b9d939666536dfb588630fc70fa24af4bc0faeeb413cf6c2352d5 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..9c21436c97a3ee439c4fab63b01056bbd0ad5853 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2af647b5cfb6310a6c4466f49c84b866fe317ca586a6310f363a4af4b1837d91 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..9696b40eaff70995751a45b283080e8ecbe34b9d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c473fc911e2cf728557fcd96c679389a80a8f9e2d9d98c653d8c91eb42e08b8 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..88f1c5e68da31f201edce158e939339c56cecac5 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f65ff4c5942f1c714965284ab16fca0702fc7df16a25c1d74ccb0d8860cb65ff +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..ad4d32e5a887ec50fc7fe114fba6e7608870d5c8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bf80c4681651f27778a7ce478549903436a0bc282e7622e50b7ee0fb8238ad1 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..92480fd211d91e724180246508a32ac8e8409256 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8272bca8eb0e2b8723da74bcc3660db931e8763ea14b9303fd187ebf1cc297b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..39d7f7be3e345ed20f2800513ff7a01fbf356658 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:469d231dacecbb16cf6c02e195620a7232c8a7bb2b9a3885fe89f404c0451c45 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..1b9c7fdfe5d17dba084d722504e90585084602b7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2ed9b862bb0d5c137dff86a434ca0ef0a713fa31e1a844ff508830d623fb96a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca6940163989dabfb68d5dac21ac685099f34267 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb77887ee043e3bfe60b681bbe38efab0f93c4a41e079d5bb0436a0d146fc2a4 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..389032259b0b3f3cfcabec47be423fe8425737f9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/trainer_state.json @@ -0,0 +1,14034 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 32.207991132161034, + "eval_steps": 2000, + "global_step": 20001, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.016123343578374565, + "grad_norm": 48.11856460571289, + "learning_rate": 0.0001, + "loss": 299.1088, + "step": 10 + }, + { + "epoch": 0.03224668715674913, + "grad_norm": 9.07711410522461, + "learning_rate": 0.0001, + "loss": 234.0888, + "step": 20 + }, + { + "epoch": 0.0483700307351237, + "grad_norm": 3.5935330390930176, + "learning_rate": 0.0001, + "loss": 231.4619, + "step": 30 + }, + { + "epoch": 0.06449337431349826, + "grad_norm": 3.5784752368927, + "learning_rate": 0.0001, + "loss": 229.7082, + "step": 40 + }, + { + "epoch": 0.08061671789187283, + "grad_norm": 3.5757956504821777, + "learning_rate": 0.0001, + "loss": 230.1062, + "step": 50 + }, + { + "epoch": 0.0967400614702474, + "grad_norm": 3.8763978481292725, + "learning_rate": 0.0001, + "loss": 229.1914, + "step": 60 + }, + { + "epoch": 0.11286340504862195, + "grad_norm": 3.006988525390625, + "learning_rate": 0.0001, + "loss": 229.6482, + "step": 70 + }, + { + "epoch": 0.12898674862699652, + "grad_norm": 2.3527095317840576, + "learning_rate": 0.0001, + "loss": 228.1141, + "step": 80 + }, + { + "epoch": 0.14511009220537108, + "grad_norm": 5.353086471557617, + "learning_rate": 0.0001, + "loss": 228.1448, + "step": 90 + }, + { + "epoch": 0.16123343578374566, + "grad_norm": 2.251854419708252, + "learning_rate": 0.0001, + "loss": 228.3897, + "step": 100 + }, + { + "epoch": 0.17735677936212021, + "grad_norm": 1.5789915323257446, + "learning_rate": 0.0001, + "loss": 227.7662, + "step": 110 + }, + { + "epoch": 0.1934801229404948, + "grad_norm": 3.7646398544311523, + "learning_rate": 0.0001, + "loss": 223.6411, + "step": 120 + }, + { + "epoch": 0.20960346651886935, + "grad_norm": 3.444000720977783, + "learning_rate": 0.0001, + "loss": 221.2073, + "step": 130 + }, + { + "epoch": 0.2257268100972439, + "grad_norm": 2.1871185302734375, + "learning_rate": 0.0001, + "loss": 215.9944, + "step": 140 + }, + { + "epoch": 0.2418501536756185, + "grad_norm": 2.266244411468506, + "learning_rate": 0.0001, + "loss": 212.3066, + "step": 150 + }, + { + "epoch": 0.25797349725399304, + "grad_norm": 2.043832778930664, + "learning_rate": 0.0001, + "loss": 208.1343, + "step": 160 + }, + { + "epoch": 0.2740968408323676, + "grad_norm": 2.2641494274139404, + "learning_rate": 0.0001, + "loss": 203.1485, + "step": 170 + }, + { + "epoch": 0.29022018441074215, + "grad_norm": 2.9727790355682373, + "learning_rate": 0.0001, + "loss": 198.181, + "step": 180 + }, + { + "epoch": 0.30634352798911674, + "grad_norm": 2.751523494720459, + "learning_rate": 0.0001, + "loss": 194.8422, + "step": 190 + }, + { + "epoch": 0.3224668715674913, + "grad_norm": 1.9561562538146973, + "learning_rate": 0.0001, + "loss": 189.9401, + "step": 200 + }, + { + "epoch": 0.33859021514586585, + "grad_norm": 2.3264107704162598, + "learning_rate": 0.0001, + "loss": 186.1309, + "step": 210 + }, + { + "epoch": 0.35471355872424043, + "grad_norm": 5.212426662445068, + "learning_rate": 0.0001, + "loss": 182.2255, + "step": 220 + }, + { + "epoch": 0.370836902302615, + "grad_norm": 2.488032102584839, + "learning_rate": 0.0001, + "loss": 181.0103, + "step": 230 + }, + { + "epoch": 0.3869602458809896, + "grad_norm": 2.6173202991485596, + "learning_rate": 0.0001, + "loss": 177.6162, + "step": 240 + }, + { + "epoch": 0.4030835894593641, + "grad_norm": 2.6245291233062744, + "learning_rate": 0.0001, + "loss": 173.0431, + "step": 250 + }, + { + "epoch": 0.4192069330377387, + "grad_norm": 5.356006145477295, + "learning_rate": 0.0001, + "loss": 172.2371, + "step": 260 + }, + { + "epoch": 0.4353302766161133, + "grad_norm": 2.0453829765319824, + "learning_rate": 0.0001, + "loss": 169.7781, + "step": 270 + }, + { + "epoch": 0.4514536201944878, + "grad_norm": 1.7002843618392944, + "learning_rate": 0.0001, + "loss": 166.9564, + "step": 280 + }, + { + "epoch": 0.4675769637728624, + "grad_norm": 2.181410312652588, + "learning_rate": 0.0001, + "loss": 166.0094, + "step": 290 + }, + { + "epoch": 0.483700307351237, + "grad_norm": 2.2948265075683594, + "learning_rate": 0.0001, + "loss": 162.4388, + "step": 300 + }, + { + "epoch": 0.4998236509296115, + "grad_norm": 3.0272789001464844, + "learning_rate": 0.0001, + "loss": 160.3446, + "step": 310 + }, + { + "epoch": 0.5159469945079861, + "grad_norm": 1.948184847831726, + "learning_rate": 0.0001, + "loss": 158.488, + "step": 320 + }, + { + "epoch": 0.5320703380863606, + "grad_norm": 1.7981538772583008, + "learning_rate": 0.0001, + "loss": 156.2107, + "step": 330 + }, + { + "epoch": 0.5481936816647353, + "grad_norm": 2.049323797225952, + "learning_rate": 0.0001, + "loss": 155.5434, + "step": 340 + }, + { + "epoch": 0.5643170252431098, + "grad_norm": 1.9415442943572998, + "learning_rate": 0.0001, + "loss": 153.8682, + "step": 350 + }, + { + "epoch": 0.5804403688214843, + "grad_norm": 2.2168045043945312, + "learning_rate": 0.0001, + "loss": 150.4087, + "step": 360 + }, + { + "epoch": 0.596563712399859, + "grad_norm": 1.510750651359558, + "learning_rate": 0.0001, + "loss": 151.7655, + "step": 370 + }, + { + "epoch": 0.6126870559782335, + "grad_norm": 2.7469751834869385, + "learning_rate": 0.0001, + "loss": 147.4665, + "step": 380 + }, + { + "epoch": 0.628810399556608, + "grad_norm": 2.0436975955963135, + "learning_rate": 0.0001, + "loss": 147.4405, + "step": 390 + }, + { + "epoch": 0.6449337431349826, + "grad_norm": 1.8219181299209595, + "learning_rate": 0.0001, + "loss": 146.6332, + "step": 400 + }, + { + "epoch": 0.6610570867133572, + "grad_norm": 1.948431134223938, + "learning_rate": 0.0001, + "loss": 143.3298, + "step": 410 + }, + { + "epoch": 0.6771804302917317, + "grad_norm": 1.8887908458709717, + "learning_rate": 0.0001, + "loss": 140.5058, + "step": 420 + }, + { + "epoch": 0.6933037738701063, + "grad_norm": 1.8739320039749146, + "learning_rate": 0.0001, + "loss": 137.6334, + "step": 430 + }, + { + "epoch": 0.7094271174484809, + "grad_norm": 2.057689666748047, + "learning_rate": 0.0001, + "loss": 135.0343, + "step": 440 + }, + { + "epoch": 0.7255504610268555, + "grad_norm": 1.6926478147506714, + "learning_rate": 0.0001, + "loss": 132.8766, + "step": 450 + }, + { + "epoch": 0.74167380460523, + "grad_norm": 1.5039501190185547, + "learning_rate": 0.0001, + "loss": 131.6774, + "step": 460 + }, + { + "epoch": 0.7577971481836046, + "grad_norm": 1.6903547048568726, + "learning_rate": 0.0001, + "loss": 129.3116, + "step": 470 + }, + { + "epoch": 0.7739204917619792, + "grad_norm": 3.7062244415283203, + "learning_rate": 0.0001, + "loss": 127.9314, + "step": 480 + }, + { + "epoch": 0.7900438353403537, + "grad_norm": 2.0586934089660645, + "learning_rate": 0.0001, + "loss": 126.6398, + "step": 490 + }, + { + "epoch": 0.8061671789187282, + "grad_norm": 1.7598215341567993, + "learning_rate": 0.0001, + "loss": 125.1501, + "step": 500 + }, + { + "epoch": 0.8222905224971029, + "grad_norm": 1.5992189645767212, + "learning_rate": 0.0001, + "loss": 124.7039, + "step": 510 + }, + { + "epoch": 0.8384138660754774, + "grad_norm": 2.8735458850860596, + "learning_rate": 0.0001, + "loss": 122.1619, + "step": 520 + }, + { + "epoch": 0.8545372096538519, + "grad_norm": 1.867098093032837, + "learning_rate": 0.0001, + "loss": 122.4506, + "step": 530 + }, + { + "epoch": 0.8706605532322266, + "grad_norm": 2.2088418006896973, + "learning_rate": 0.0001, + "loss": 122.1047, + "step": 540 + }, + { + "epoch": 0.8867838968106011, + "grad_norm": 1.642657995223999, + "learning_rate": 0.0001, + "loss": 120.1141, + "step": 550 + }, + { + "epoch": 0.9029072403889756, + "grad_norm": 1.5567114353179932, + "learning_rate": 0.0001, + "loss": 118.532, + "step": 560 + }, + { + "epoch": 0.9190305839673503, + "grad_norm": 1.667843222618103, + "learning_rate": 0.0001, + "loss": 118.0734, + "step": 570 + }, + { + "epoch": 0.9351539275457248, + "grad_norm": 1.5304442644119263, + "learning_rate": 0.0001, + "loss": 114.4925, + "step": 580 + }, + { + "epoch": 0.9512772711240993, + "grad_norm": 1.6920255422592163, + "learning_rate": 0.0001, + "loss": 114.0729, + "step": 590 + }, + { + "epoch": 0.967400614702474, + "grad_norm": 1.6313318014144897, + "learning_rate": 0.0001, + "loss": 114.2879, + "step": 600 + }, + { + "epoch": 0.9835239582808485, + "grad_norm": 1.9033657312393188, + "learning_rate": 0.0001, + "loss": 110.5299, + "step": 610 + }, + { + "epoch": 0.999647301859223, + "grad_norm": 1.9010754823684692, + "learning_rate": 0.0001, + "loss": 109.1145, + "step": 620 + }, + { + "epoch": 1.0145110092205372, + "grad_norm": 3.5390989780426025, + "learning_rate": 0.0001, + "loss": 101.0258, + "step": 630 + }, + { + "epoch": 1.0306343527989117, + "grad_norm": 2.2925171852111816, + "learning_rate": 0.0001, + "loss": 107.3693, + "step": 640 + }, + { + "epoch": 1.0467576963772862, + "grad_norm": 2.329108238220215, + "learning_rate": 0.0001, + "loss": 104.3778, + "step": 650 + }, + { + "epoch": 1.0628810399556607, + "grad_norm": 1.7325661182403564, + "learning_rate": 0.0001, + "loss": 102.8409, + "step": 660 + }, + { + "epoch": 1.0790043835340353, + "grad_norm": 1.989586353302002, + "learning_rate": 0.0001, + "loss": 104.2577, + "step": 670 + }, + { + "epoch": 1.09512772711241, + "grad_norm": 1.900484323501587, + "learning_rate": 0.0001, + "loss": 102.3051, + "step": 680 + }, + { + "epoch": 1.1112510706907845, + "grad_norm": 2.015859842300415, + "learning_rate": 0.0001, + "loss": 100.893, + "step": 690 + }, + { + "epoch": 1.127374414269159, + "grad_norm": 1.7695508003234863, + "learning_rate": 0.0001, + "loss": 99.215, + "step": 700 + }, + { + "epoch": 1.1434977578475336, + "grad_norm": 2.025923490524292, + "learning_rate": 0.0001, + "loss": 98.8768, + "step": 710 + }, + { + "epoch": 1.1596211014259081, + "grad_norm": 2.946708917617798, + "learning_rate": 0.0001, + "loss": 96.5588, + "step": 720 + }, + { + "epoch": 1.1757444450042827, + "grad_norm": 2.4511258602142334, + "learning_rate": 0.0001, + "loss": 95.0545, + "step": 730 + }, + { + "epoch": 1.1918677885826574, + "grad_norm": 2.3094682693481445, + "learning_rate": 0.0001, + "loss": 97.004, + "step": 740 + }, + { + "epoch": 1.207991132161032, + "grad_norm": 2.1313347816467285, + "learning_rate": 0.0001, + "loss": 94.8819, + "step": 750 + }, + { + "epoch": 1.2241144757394065, + "grad_norm": 1.614182710647583, + "learning_rate": 0.0001, + "loss": 95.5339, + "step": 760 + }, + { + "epoch": 1.240237819317781, + "grad_norm": 2.029953956604004, + "learning_rate": 0.0001, + "loss": 93.7303, + "step": 770 + }, + { + "epoch": 1.2563611628961555, + "grad_norm": 1.7871663570404053, + "learning_rate": 0.0001, + "loss": 93.8354, + "step": 780 + }, + { + "epoch": 1.2724845064745303, + "grad_norm": 1.72455632686615, + "learning_rate": 0.0001, + "loss": 94.7348, + "step": 790 + }, + { + "epoch": 1.2886078500529048, + "grad_norm": 1.9776639938354492, + "learning_rate": 0.0001, + "loss": 93.5957, + "step": 800 + }, + { + "epoch": 1.3047311936312793, + "grad_norm": 1.7187533378601074, + "learning_rate": 0.0001, + "loss": 92.9583, + "step": 810 + }, + { + "epoch": 1.3208545372096538, + "grad_norm": 1.5949162244796753, + "learning_rate": 0.0001, + "loss": 93.3088, + "step": 820 + }, + { + "epoch": 1.3369778807880284, + "grad_norm": 1.4616206884384155, + "learning_rate": 0.0001, + "loss": 93.2125, + "step": 830 + }, + { + "epoch": 1.353101224366403, + "grad_norm": 2.832163095474243, + "learning_rate": 0.0001, + "loss": 90.3301, + "step": 840 + }, + { + "epoch": 1.3692245679447774, + "grad_norm": 2.174501419067383, + "learning_rate": 0.0001, + "loss": 91.6988, + "step": 850 + }, + { + "epoch": 1.3853479115231522, + "grad_norm": 1.2156339883804321, + "learning_rate": 0.0001, + "loss": 92.7419, + "step": 860 + }, + { + "epoch": 1.4014712551015267, + "grad_norm": 1.608261227607727, + "learning_rate": 0.0001, + "loss": 89.8255, + "step": 870 + }, + { + "epoch": 1.4175945986799012, + "grad_norm": 1.6671923398971558, + "learning_rate": 0.0001, + "loss": 90.9683, + "step": 880 + }, + { + "epoch": 1.4337179422582758, + "grad_norm": 1.2346433401107788, + "learning_rate": 0.0001, + "loss": 90.6092, + "step": 890 + }, + { + "epoch": 1.4498412858366505, + "grad_norm": 1.561085820198059, + "learning_rate": 0.0001, + "loss": 88.0842, + "step": 900 + }, + { + "epoch": 1.465964629415025, + "grad_norm": 1.5475201606750488, + "learning_rate": 0.0001, + "loss": 89.8786, + "step": 910 + }, + { + "epoch": 1.4820879729933996, + "grad_norm": 1.6044800281524658, + "learning_rate": 0.0001, + "loss": 88.0364, + "step": 920 + }, + { + "epoch": 1.498211316571774, + "grad_norm": 1.4081908464431763, + "learning_rate": 0.0001, + "loss": 88.8799, + "step": 930 + }, + { + "epoch": 1.5143346601501486, + "grad_norm": 2.341053009033203, + "learning_rate": 0.0001, + "loss": 88.2023, + "step": 940 + }, + { + "epoch": 1.5304580037285231, + "grad_norm": 1.3976799249649048, + "learning_rate": 0.0001, + "loss": 86.6493, + "step": 950 + }, + { + "epoch": 1.5465813473068977, + "grad_norm": 1.2485483884811401, + "learning_rate": 0.0001, + "loss": 88.9224, + "step": 960 + }, + { + "epoch": 1.5627046908852722, + "grad_norm": 1.6824235916137695, + "learning_rate": 0.0001, + "loss": 89.6484, + "step": 970 + }, + { + "epoch": 1.578828034463647, + "grad_norm": 1.64790940284729, + "learning_rate": 0.0001, + "loss": 87.2479, + "step": 980 + }, + { + "epoch": 1.5949513780420215, + "grad_norm": 1.2093443870544434, + "learning_rate": 0.0001, + "loss": 86.8241, + "step": 990 + }, + { + "epoch": 1.611074721620396, + "grad_norm": 1.5306812524795532, + "learning_rate": 0.0001, + "loss": 88.0006, + "step": 1000 + }, + { + "epoch": 1.6271980651987707, + "grad_norm": 1.2465763092041016, + "learning_rate": 0.0001, + "loss": 86.8143, + "step": 1010 + }, + { + "epoch": 1.6433214087771453, + "grad_norm": 1.5224069356918335, + "learning_rate": 0.0001, + "loss": 87.0102, + "step": 1020 + }, + { + "epoch": 1.6594447523555198, + "grad_norm": 1.312179684638977, + "learning_rate": 0.0001, + "loss": 88.7395, + "step": 1030 + }, + { + "epoch": 1.6755680959338943, + "grad_norm": 1.2343158721923828, + "learning_rate": 0.0001, + "loss": 87.2868, + "step": 1040 + }, + { + "epoch": 1.6916914395122689, + "grad_norm": 1.2778265476226807, + "learning_rate": 0.0001, + "loss": 87.043, + "step": 1050 + }, + { + "epoch": 1.7078147830906434, + "grad_norm": 1.2799627780914307, + "learning_rate": 0.0001, + "loss": 87.7843, + "step": 1060 + }, + { + "epoch": 1.723938126669018, + "grad_norm": 1.3688952922821045, + "learning_rate": 0.0001, + "loss": 85.8985, + "step": 1070 + }, + { + "epoch": 1.7400614702473924, + "grad_norm": 1.4317796230316162, + "learning_rate": 0.0001, + "loss": 85.5408, + "step": 1080 + }, + { + "epoch": 1.756184813825767, + "grad_norm": 1.164801001548767, + "learning_rate": 0.0001, + "loss": 85.8484, + "step": 1090 + }, + { + "epoch": 1.7723081574041417, + "grad_norm": 1.3929173946380615, + "learning_rate": 0.0001, + "loss": 86.9031, + "step": 1100 + }, + { + "epoch": 1.7884315009825162, + "grad_norm": 2.116119861602783, + "learning_rate": 0.0001, + "loss": 86.5343, + "step": 1110 + }, + { + "epoch": 1.8045548445608908, + "grad_norm": 1.4117276668548584, + "learning_rate": 0.0001, + "loss": 85.2678, + "step": 1120 + }, + { + "epoch": 1.8206781881392655, + "grad_norm": 1.3213504552841187, + "learning_rate": 0.0001, + "loss": 85.8935, + "step": 1130 + }, + { + "epoch": 1.83680153171764, + "grad_norm": 1.3589770793914795, + "learning_rate": 0.0001, + "loss": 85.032, + "step": 1140 + }, + { + "epoch": 1.8529248752960146, + "grad_norm": 1.3336007595062256, + "learning_rate": 0.0001, + "loss": 84.0993, + "step": 1150 + }, + { + "epoch": 1.869048218874389, + "grad_norm": 1.1081451177597046, + "learning_rate": 0.0001, + "loss": 84.2225, + "step": 1160 + }, + { + "epoch": 1.8851715624527636, + "grad_norm": 1.482038974761963, + "learning_rate": 0.0001, + "loss": 82.1439, + "step": 1170 + }, + { + "epoch": 1.9012949060311382, + "grad_norm": 1.1571478843688965, + "learning_rate": 0.0001, + "loss": 82.5045, + "step": 1180 + }, + { + "epoch": 1.9174182496095127, + "grad_norm": 1.2665468454360962, + "learning_rate": 0.0001, + "loss": 82.8761, + "step": 1190 + }, + { + "epoch": 1.9335415931878872, + "grad_norm": 1.0771509408950806, + "learning_rate": 0.0001, + "loss": 81.5162, + "step": 1200 + }, + { + "epoch": 1.949664936766262, + "grad_norm": 1.2992299795150757, + "learning_rate": 0.0001, + "loss": 82.4038, + "step": 1210 + }, + { + "epoch": 1.9657882803446365, + "grad_norm": 1.2946668863296509, + "learning_rate": 0.0001, + "loss": 83.4837, + "step": 1220 + }, + { + "epoch": 1.981911623923011, + "grad_norm": 1.3505686521530151, + "learning_rate": 0.0001, + "loss": 81.8993, + "step": 1230 + }, + { + "epoch": 1.9980349675013858, + "grad_norm": 1.1247715950012207, + "learning_rate": 0.0001, + "loss": 82.9006, + "step": 1240 + }, + { + "epoch": 2.0128986748627, + "grad_norm": 1.5652127265930176, + "learning_rate": 0.0001, + "loss": 75.8347, + "step": 1250 + }, + { + "epoch": 2.0290220184410743, + "grad_norm": 7.311964988708496, + "learning_rate": 0.0001, + "loss": 80.5435, + "step": 1260 + }, + { + "epoch": 2.045145362019449, + "grad_norm": 1.2229893207550049, + "learning_rate": 0.0001, + "loss": 83.3277, + "step": 1270 + }, + { + "epoch": 2.0612687055978234, + "grad_norm": 1.1387693881988525, + "learning_rate": 0.0001, + "loss": 81.3252, + "step": 1280 + }, + { + "epoch": 2.077392049176198, + "grad_norm": 1.307201623916626, + "learning_rate": 0.0001, + "loss": 82.3311, + "step": 1290 + }, + { + "epoch": 2.0935153927545724, + "grad_norm": 1.1502896547317505, + "learning_rate": 0.0001, + "loss": 81.1682, + "step": 1300 + }, + { + "epoch": 2.109638736332947, + "grad_norm": 0.9291583299636841, + "learning_rate": 0.0001, + "loss": 81.7017, + "step": 1310 + }, + { + "epoch": 2.1257620799113215, + "grad_norm": 1.0334666967391968, + "learning_rate": 0.0001, + "loss": 81.718, + "step": 1320 + }, + { + "epoch": 2.141885423489696, + "grad_norm": 1.1099056005477905, + "learning_rate": 0.0001, + "loss": 81.2232, + "step": 1330 + }, + { + "epoch": 2.1580087670680705, + "grad_norm": 1.289478063583374, + "learning_rate": 0.0001, + "loss": 81.8736, + "step": 1340 + }, + { + "epoch": 2.174132110646445, + "grad_norm": 1.1634794473648071, + "learning_rate": 0.0001, + "loss": 79.9434, + "step": 1350 + }, + { + "epoch": 2.19025545422482, + "grad_norm": 0.9563440680503845, + "learning_rate": 0.0001, + "loss": 80.628, + "step": 1360 + }, + { + "epoch": 2.2063787978031946, + "grad_norm": 1.1268844604492188, + "learning_rate": 0.0001, + "loss": 82.1389, + "step": 1370 + }, + { + "epoch": 2.222502141381569, + "grad_norm": 1.1050443649291992, + "learning_rate": 0.0001, + "loss": 78.9023, + "step": 1380 + }, + { + "epoch": 2.2386254849599436, + "grad_norm": 1.1936142444610596, + "learning_rate": 0.0001, + "loss": 81.2118, + "step": 1390 + }, + { + "epoch": 2.254748828538318, + "grad_norm": 1.7825465202331543, + "learning_rate": 0.0001, + "loss": 80.8804, + "step": 1400 + }, + { + "epoch": 2.2708721721166927, + "grad_norm": 2.437251329421997, + "learning_rate": 0.0001, + "loss": 79.3807, + "step": 1410 + }, + { + "epoch": 2.286995515695067, + "grad_norm": 1.0604854822158813, + "learning_rate": 0.0001, + "loss": 79.5881, + "step": 1420 + }, + { + "epoch": 2.3031188592734417, + "grad_norm": 1.2285529375076294, + "learning_rate": 0.0001, + "loss": 79.5517, + "step": 1430 + }, + { + "epoch": 2.3192422028518163, + "grad_norm": 1.1916660070419312, + "learning_rate": 0.0001, + "loss": 80.5887, + "step": 1440 + }, + { + "epoch": 2.335365546430191, + "grad_norm": 1.224304437637329, + "learning_rate": 0.0001, + "loss": 79.8109, + "step": 1450 + }, + { + "epoch": 2.3514888900085653, + "grad_norm": 0.9228448867797852, + "learning_rate": 0.0001, + "loss": 79.2787, + "step": 1460 + }, + { + "epoch": 2.3676122335869403, + "grad_norm": 1.2085226774215698, + "learning_rate": 0.0001, + "loss": 78.4983, + "step": 1470 + }, + { + "epoch": 2.383735577165315, + "grad_norm": 2.424083948135376, + "learning_rate": 0.0001, + "loss": 79.8657, + "step": 1480 + }, + { + "epoch": 2.3998589207436893, + "grad_norm": 1.0277849435806274, + "learning_rate": 0.0001, + "loss": 80.6915, + "step": 1490 + }, + { + "epoch": 2.415982264322064, + "grad_norm": 1.0226075649261475, + "learning_rate": 0.0001, + "loss": 78.9751, + "step": 1500 + }, + { + "epoch": 2.4321056079004384, + "grad_norm": 0.8860388398170471, + "learning_rate": 0.0001, + "loss": 78.3915, + "step": 1510 + }, + { + "epoch": 2.448228951478813, + "grad_norm": 1.265637755393982, + "learning_rate": 0.0001, + "loss": 78.8737, + "step": 1520 + }, + { + "epoch": 2.4643522950571874, + "grad_norm": 1.2518186569213867, + "learning_rate": 0.0001, + "loss": 76.5169, + "step": 1530 + }, + { + "epoch": 2.480475638635562, + "grad_norm": 1.074277400970459, + "learning_rate": 0.0001, + "loss": 77.9156, + "step": 1540 + }, + { + "epoch": 2.4965989822139365, + "grad_norm": 1.0666711330413818, + "learning_rate": 0.0001, + "loss": 78.4103, + "step": 1550 + }, + { + "epoch": 2.512722325792311, + "grad_norm": 0.9492248892784119, + "learning_rate": 0.0001, + "loss": 76.7393, + "step": 1560 + }, + { + "epoch": 2.5288456693706856, + "grad_norm": 1.127238154411316, + "learning_rate": 0.0001, + "loss": 79.729, + "step": 1570 + }, + { + "epoch": 2.5449690129490605, + "grad_norm": 1.1520992517471313, + "learning_rate": 0.0001, + "loss": 78.5618, + "step": 1580 + }, + { + "epoch": 2.5610923565274346, + "grad_norm": 0.8632035255432129, + "learning_rate": 0.0001, + "loss": 79.032, + "step": 1590 + }, + { + "epoch": 2.5772157001058096, + "grad_norm": 1.0506329536437988, + "learning_rate": 0.0001, + "loss": 77.6204, + "step": 1600 + }, + { + "epoch": 2.593339043684184, + "grad_norm": 1.019895315170288, + "learning_rate": 0.0001, + "loss": 79.2634, + "step": 1610 + }, + { + "epoch": 2.6094623872625586, + "grad_norm": 1.1685571670532227, + "learning_rate": 0.0001, + "loss": 78.8371, + "step": 1620 + }, + { + "epoch": 2.625585730840933, + "grad_norm": 1.147546648979187, + "learning_rate": 0.0001, + "loss": 78.2113, + "step": 1630 + }, + { + "epoch": 2.6417090744193077, + "grad_norm": 1.7336841821670532, + "learning_rate": 0.0001, + "loss": 78.7866, + "step": 1640 + }, + { + "epoch": 2.657832417997682, + "grad_norm": 1.2150614261627197, + "learning_rate": 0.0001, + "loss": 78.5621, + "step": 1650 + }, + { + "epoch": 2.6739557615760567, + "grad_norm": 1.2257503271102905, + "learning_rate": 0.0001, + "loss": 77.5835, + "step": 1660 + }, + { + "epoch": 2.6900791051544313, + "grad_norm": 1.0375311374664307, + "learning_rate": 0.0001, + "loss": 77.7509, + "step": 1670 + }, + { + "epoch": 2.706202448732806, + "grad_norm": 1.2551004886627197, + "learning_rate": 0.0001, + "loss": 77.8615, + "step": 1680 + }, + { + "epoch": 2.7223257923111808, + "grad_norm": 1.0464223623275757, + "learning_rate": 0.0001, + "loss": 77.2837, + "step": 1690 + }, + { + "epoch": 2.738449135889555, + "grad_norm": 1.0254806280136108, + "learning_rate": 0.0001, + "loss": 76.9616, + "step": 1700 + }, + { + "epoch": 2.75457247946793, + "grad_norm": 1.1665682792663574, + "learning_rate": 0.0001, + "loss": 77.9108, + "step": 1710 + }, + { + "epoch": 2.7706958230463044, + "grad_norm": 1.1349453926086426, + "learning_rate": 0.0001, + "loss": 78.7636, + "step": 1720 + }, + { + "epoch": 2.786819166624679, + "grad_norm": 1.4961057901382446, + "learning_rate": 0.0001, + "loss": 76.3862, + "step": 1730 + }, + { + "epoch": 2.8029425102030534, + "grad_norm": 1.10721755027771, + "learning_rate": 0.0001, + "loss": 77.4105, + "step": 1740 + }, + { + "epoch": 2.819065853781428, + "grad_norm": 0.9878576397895813, + "learning_rate": 0.0001, + "loss": 78.4855, + "step": 1750 + }, + { + "epoch": 2.8351891973598025, + "grad_norm": 1.1722687482833862, + "learning_rate": 0.0001, + "loss": 75.1589, + "step": 1760 + }, + { + "epoch": 2.851312540938177, + "grad_norm": 1.6624362468719482, + "learning_rate": 0.0001, + "loss": 77.1647, + "step": 1770 + }, + { + "epoch": 2.8674358845165515, + "grad_norm": 0.9542295932769775, + "learning_rate": 0.0001, + "loss": 77.1656, + "step": 1780 + }, + { + "epoch": 2.883559228094926, + "grad_norm": 2.328542947769165, + "learning_rate": 0.0001, + "loss": 76.4686, + "step": 1790 + }, + { + "epoch": 2.899682571673301, + "grad_norm": 1.2532029151916504, + "learning_rate": 0.0001, + "loss": 76.1727, + "step": 1800 + }, + { + "epoch": 2.915805915251675, + "grad_norm": 0.9604716897010803, + "learning_rate": 0.0001, + "loss": 74.3132, + "step": 1810 + }, + { + "epoch": 2.93192925883005, + "grad_norm": 0.9182595610618591, + "learning_rate": 0.0001, + "loss": 77.8689, + "step": 1820 + }, + { + "epoch": 2.9480526024084246, + "grad_norm": 0.9623042941093445, + "learning_rate": 0.0001, + "loss": 77.7531, + "step": 1830 + }, + { + "epoch": 2.964175945986799, + "grad_norm": 1.0500119924545288, + "learning_rate": 0.0001, + "loss": 77.6007, + "step": 1840 + }, + { + "epoch": 2.9802992895651736, + "grad_norm": 1.0874103307724, + "learning_rate": 0.0001, + "loss": 77.042, + "step": 1850 + }, + { + "epoch": 2.996422633143548, + "grad_norm": 0.9955475330352783, + "learning_rate": 0.0001, + "loss": 75.468, + "step": 1860 + }, + { + "epoch": 3.011286340504862, + "grad_norm": 1.4281760454177856, + "learning_rate": 0.0001, + "loss": 69.5972, + "step": 1870 + }, + { + "epoch": 3.0274096840832367, + "grad_norm": 1.0292123556137085, + "learning_rate": 0.0001, + "loss": 76.0186, + "step": 1880 + }, + { + "epoch": 3.0435330276616113, + "grad_norm": 0.9535816311836243, + "learning_rate": 0.0001, + "loss": 76.0979, + "step": 1890 + }, + { + "epoch": 3.059656371239986, + "grad_norm": 0.9099674820899963, + "learning_rate": 0.0001, + "loss": 75.9641, + "step": 1900 + }, + { + "epoch": 3.0757797148183603, + "grad_norm": 0.9568380117416382, + "learning_rate": 0.0001, + "loss": 75.0656, + "step": 1910 + }, + { + "epoch": 3.091903058396735, + "grad_norm": 0.9701060652732849, + "learning_rate": 0.0001, + "loss": 75.1422, + "step": 1920 + }, + { + "epoch": 3.1080264019751094, + "grad_norm": 1.3879098892211914, + "learning_rate": 0.0001, + "loss": 76.0731, + "step": 1930 + }, + { + "epoch": 3.1241497455534843, + "grad_norm": 0.9782084226608276, + "learning_rate": 0.0001, + "loss": 75.9666, + "step": 1940 + }, + { + "epoch": 3.140273089131859, + "grad_norm": 1.0856964588165283, + "learning_rate": 0.0001, + "loss": 75.0217, + "step": 1950 + }, + { + "epoch": 3.1563964327102334, + "grad_norm": 0.8506444692611694, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 1960 + }, + { + "epoch": 3.172519776288608, + "grad_norm": 1.044980525970459, + "learning_rate": 0.0001, + "loss": 75.3471, + "step": 1970 + }, + { + "epoch": 3.1886431198669825, + "grad_norm": 0.8432291746139526, + "learning_rate": 0.0001, + "loss": 75.3563, + "step": 1980 + }, + { + "epoch": 3.204766463445357, + "grad_norm": 0.9651834964752197, + "learning_rate": 0.0001, + "loss": 76.3991, + "step": 1990 + }, + { + "epoch": 3.2208898070237315, + "grad_norm": 0.957943320274353, + "learning_rate": 0.0001, + "loss": 77.2613, + "step": 2000 + }, + { + "epoch": 3.237013150602106, + "grad_norm": 0.9495745301246643, + "learning_rate": 0.0001, + "loss": 73.3801, + "step": 2010 + }, + { + "epoch": 3.2531364941804806, + "grad_norm": 1.182983160018921, + "learning_rate": 0.0001, + "loss": 74.5695, + "step": 2020 + }, + { + "epoch": 3.269259837758855, + "grad_norm": 0.828452467918396, + "learning_rate": 0.0001, + "loss": 75.4313, + "step": 2030 + }, + { + "epoch": 3.2853831813372296, + "grad_norm": 0.8340104818344116, + "learning_rate": 0.0001, + "loss": 75.1086, + "step": 2040 + }, + { + "epoch": 3.301506524915604, + "grad_norm": 1.1700762510299683, + "learning_rate": 0.0001, + "loss": 75.7496, + "step": 2050 + }, + { + "epoch": 3.317629868493979, + "grad_norm": 0.8709110617637634, + "learning_rate": 0.0001, + "loss": 73.8403, + "step": 2060 + }, + { + "epoch": 3.3337532120723536, + "grad_norm": 0.8928954601287842, + "learning_rate": 0.0001, + "loss": 75.6134, + "step": 2070 + }, + { + "epoch": 3.349876555650728, + "grad_norm": 0.9787353277206421, + "learning_rate": 0.0001, + "loss": 75.8116, + "step": 2080 + }, + { + "epoch": 3.3659998992291027, + "grad_norm": 0.9515123963356018, + "learning_rate": 0.0001, + "loss": 75.5024, + "step": 2090 + }, + { + "epoch": 3.3821232428074772, + "grad_norm": 0.9529262185096741, + "learning_rate": 0.0001, + "loss": 74.1553, + "step": 2100 + }, + { + "epoch": 3.3982465863858518, + "grad_norm": 0.8934710621833801, + "learning_rate": 0.0001, + "loss": 74.1501, + "step": 2110 + }, + { + "epoch": 3.4143699299642263, + "grad_norm": 1.089447021484375, + "learning_rate": 0.0001, + "loss": 74.9123, + "step": 2120 + }, + { + "epoch": 3.430493273542601, + "grad_norm": 1.2569622993469238, + "learning_rate": 0.0001, + "loss": 74.3962, + "step": 2130 + }, + { + "epoch": 3.4466166171209753, + "grad_norm": 1.020094394683838, + "learning_rate": 0.0001, + "loss": 74.6617, + "step": 2140 + }, + { + "epoch": 3.46273996069935, + "grad_norm": 1.1579667329788208, + "learning_rate": 0.0001, + "loss": 74.2745, + "step": 2150 + }, + { + "epoch": 3.4788633042777244, + "grad_norm": 1.448935627937317, + "learning_rate": 0.0001, + "loss": 71.5285, + "step": 2160 + }, + { + "epoch": 3.4949866478560994, + "grad_norm": 1.0030494928359985, + "learning_rate": 0.0001, + "loss": 74.6666, + "step": 2170 + }, + { + "epoch": 3.511109991434474, + "grad_norm": 0.9010888934135437, + "learning_rate": 0.0001, + "loss": 74.9355, + "step": 2180 + }, + { + "epoch": 3.5272333350128484, + "grad_norm": 1.1595124006271362, + "learning_rate": 0.0001, + "loss": 74.5049, + "step": 2190 + }, + { + "epoch": 3.543356678591223, + "grad_norm": 1.066405177116394, + "learning_rate": 0.0001, + "loss": 74.6336, + "step": 2200 + }, + { + "epoch": 3.5594800221695975, + "grad_norm": 1.2301596403121948, + "learning_rate": 0.0001, + "loss": 74.6664, + "step": 2210 + }, + { + "epoch": 3.575603365747972, + "grad_norm": 1.0073511600494385, + "learning_rate": 0.0001, + "loss": 73.1405, + "step": 2220 + }, + { + "epoch": 3.5917267093263465, + "grad_norm": 0.9913995862007141, + "learning_rate": 0.0001, + "loss": 75.7443, + "step": 2230 + }, + { + "epoch": 3.607850052904721, + "grad_norm": 0.9555648565292358, + "learning_rate": 0.0001, + "loss": 74.8036, + "step": 2240 + }, + { + "epoch": 3.6239733964830956, + "grad_norm": 0.9350649118423462, + "learning_rate": 0.0001, + "loss": 73.5783, + "step": 2250 + }, + { + "epoch": 3.64009674006147, + "grad_norm": 0.935840368270874, + "learning_rate": 0.0001, + "loss": 75.1694, + "step": 2260 + }, + { + "epoch": 3.6562200836398446, + "grad_norm": 0.9303803443908691, + "learning_rate": 0.0001, + "loss": 72.8282, + "step": 2270 + }, + { + "epoch": 3.6723434272182196, + "grad_norm": 0.937164843082428, + "learning_rate": 0.0001, + "loss": 70.3305, + "step": 2280 + }, + { + "epoch": 3.6884667707965937, + "grad_norm": 1.027329444885254, + "learning_rate": 0.0001, + "loss": 73.748, + "step": 2290 + }, + { + "epoch": 3.7045901143749687, + "grad_norm": 1.0915215015411377, + "learning_rate": 0.0001, + "loss": 72.2265, + "step": 2300 + }, + { + "epoch": 3.720713457953343, + "grad_norm": 1.1031261682510376, + "learning_rate": 0.0001, + "loss": 71.2795, + "step": 2310 + }, + { + "epoch": 3.7368368015317177, + "grad_norm": 1.1466140747070312, + "learning_rate": 0.0001, + "loss": 72.0116, + "step": 2320 + }, + { + "epoch": 3.7529601451100922, + "grad_norm": 1.0441662073135376, + "learning_rate": 0.0001, + "loss": 73.3735, + "step": 2330 + }, + { + "epoch": 3.7690834886884668, + "grad_norm": 0.8610432744026184, + "learning_rate": 0.0001, + "loss": 73.3433, + "step": 2340 + }, + { + "epoch": 3.7852068322668413, + "grad_norm": 0.885720431804657, + "learning_rate": 0.0001, + "loss": 73.575, + "step": 2350 + }, + { + "epoch": 3.801330175845216, + "grad_norm": 1.0023950338363647, + "learning_rate": 0.0001, + "loss": 73.4166, + "step": 2360 + }, + { + "epoch": 3.8174535194235903, + "grad_norm": 1.3820304870605469, + "learning_rate": 0.0001, + "loss": 72.9599, + "step": 2370 + }, + { + "epoch": 3.833576863001965, + "grad_norm": 0.9645710587501526, + "learning_rate": 0.0001, + "loss": 72.3922, + "step": 2380 + }, + { + "epoch": 3.84970020658034, + "grad_norm": 1.273369312286377, + "learning_rate": 0.0001, + "loss": 72.0158, + "step": 2390 + }, + { + "epoch": 3.865823550158714, + "grad_norm": 0.859863817691803, + "learning_rate": 0.0001, + "loss": 74.6642, + "step": 2400 + }, + { + "epoch": 3.881946893737089, + "grad_norm": 0.9396682381629944, + "learning_rate": 0.0001, + "loss": 71.9003, + "step": 2410 + }, + { + "epoch": 3.8980702373154634, + "grad_norm": 0.9976151585578918, + "learning_rate": 0.0001, + "loss": 73.0716, + "step": 2420 + }, + { + "epoch": 3.914193580893838, + "grad_norm": 1.0730767250061035, + "learning_rate": 0.0001, + "loss": 72.7847, + "step": 2430 + }, + { + "epoch": 3.9303169244722125, + "grad_norm": 1.002018690109253, + "learning_rate": 0.0001, + "loss": 71.7309, + "step": 2440 + }, + { + "epoch": 3.946440268050587, + "grad_norm": 0.9102753400802612, + "learning_rate": 0.0001, + "loss": 71.616, + "step": 2450 + }, + { + "epoch": 3.9625636116289615, + "grad_norm": 0.9827467799186707, + "learning_rate": 0.0001, + "loss": 72.22, + "step": 2460 + }, + { + "epoch": 3.978686955207336, + "grad_norm": 1.2781893014907837, + "learning_rate": 0.0001, + "loss": 73.2048, + "step": 2470 + }, + { + "epoch": 3.9948102987857106, + "grad_norm": 0.9042410850524902, + "learning_rate": 0.0001, + "loss": 72.9843, + "step": 2480 + }, + { + "epoch": 4.009674006147025, + "grad_norm": 0.8662117123603821, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 2490 + }, + { + "epoch": 4.0257973497254, + "grad_norm": 0.9448756575584412, + "learning_rate": 0.0001, + "loss": 73.36, + "step": 2500 + }, + { + "epoch": 4.041920693303774, + "grad_norm": 0.9054566621780396, + "learning_rate": 0.0001, + "loss": 73.94, + "step": 2510 + }, + { + "epoch": 4.058044036882149, + "grad_norm": 0.957398533821106, + "learning_rate": 0.0001, + "loss": 73.2019, + "step": 2520 + }, + { + "epoch": 4.074167380460523, + "grad_norm": 0.9444408416748047, + "learning_rate": 0.0001, + "loss": 70.6553, + "step": 2530 + }, + { + "epoch": 4.090290724038898, + "grad_norm": 0.8716063499450684, + "learning_rate": 0.0001, + "loss": 70.1048, + "step": 2540 + }, + { + "epoch": 4.106414067617272, + "grad_norm": 0.949249267578125, + "learning_rate": 0.0001, + "loss": 72.4453, + "step": 2550 + }, + { + "epoch": 4.122537411195647, + "grad_norm": 0.9843282699584961, + "learning_rate": 0.0001, + "loss": 72.1725, + "step": 2560 + }, + { + "epoch": 4.138660754774021, + "grad_norm": 0.8977480530738831, + "learning_rate": 0.0001, + "loss": 72.5272, + "step": 2570 + }, + { + "epoch": 4.154784098352396, + "grad_norm": 0.9957827925682068, + "learning_rate": 0.0001, + "loss": 72.289, + "step": 2580 + }, + { + "epoch": 4.170907441930771, + "grad_norm": 1.1358791589736938, + "learning_rate": 0.0001, + "loss": 72.1949, + "step": 2590 + }, + { + "epoch": 4.187030785509145, + "grad_norm": 0.9953081607818604, + "learning_rate": 0.0001, + "loss": 72.6004, + "step": 2600 + }, + { + "epoch": 4.20315412908752, + "grad_norm": 1.019061803817749, + "learning_rate": 0.0001, + "loss": 72.3772, + "step": 2610 + }, + { + "epoch": 4.219277472665894, + "grad_norm": 0.9062562584877014, + "learning_rate": 0.0001, + "loss": 72.0566, + "step": 2620 + }, + { + "epoch": 4.235400816244269, + "grad_norm": 1.0877611637115479, + "learning_rate": 0.0001, + "loss": 72.854, + "step": 2630 + }, + { + "epoch": 4.251524159822643, + "grad_norm": 1.2419648170471191, + "learning_rate": 0.0001, + "loss": 72.6498, + "step": 2640 + }, + { + "epoch": 4.267647503401018, + "grad_norm": 0.9507225751876831, + "learning_rate": 0.0001, + "loss": 68.9583, + "step": 2650 + }, + { + "epoch": 4.283770846979392, + "grad_norm": 1.121191382408142, + "learning_rate": 0.0001, + "loss": 72.8464, + "step": 2660 + }, + { + "epoch": 4.299894190557767, + "grad_norm": 1.0543361902236938, + "learning_rate": 0.0001, + "loss": 71.9882, + "step": 2670 + }, + { + "epoch": 4.316017534136141, + "grad_norm": 0.8622955083847046, + "learning_rate": 0.0001, + "loss": 71.7373, + "step": 2680 + }, + { + "epoch": 4.332140877714516, + "grad_norm": 0.911253809928894, + "learning_rate": 0.0001, + "loss": 71.1968, + "step": 2690 + }, + { + "epoch": 4.34826422129289, + "grad_norm": 4.405472755432129, + "learning_rate": 0.0001, + "loss": 70.4128, + "step": 2700 + }, + { + "epoch": 4.364387564871265, + "grad_norm": 0.9142985939979553, + "learning_rate": 0.0001, + "loss": 69.7548, + "step": 2710 + }, + { + "epoch": 4.38051090844964, + "grad_norm": 1.1022522449493408, + "learning_rate": 0.0001, + "loss": 71.8282, + "step": 2720 + }, + { + "epoch": 4.396634252028014, + "grad_norm": 1.018386721611023, + "learning_rate": 0.0001, + "loss": 71.0853, + "step": 2730 + }, + { + "epoch": 4.412757595606389, + "grad_norm": 1.0483211278915405, + "learning_rate": 0.0001, + "loss": 72.1478, + "step": 2740 + }, + { + "epoch": 4.428880939184763, + "grad_norm": 1.026861548423767, + "learning_rate": 0.0001, + "loss": 71.8947, + "step": 2750 + }, + { + "epoch": 4.445004282763138, + "grad_norm": 1.1858552694320679, + "learning_rate": 0.0001, + "loss": 71.7016, + "step": 2760 + }, + { + "epoch": 4.461127626341512, + "grad_norm": 1.1212221384048462, + "learning_rate": 0.0001, + "loss": 71.4454, + "step": 2770 + }, + { + "epoch": 4.477250969919887, + "grad_norm": 0.8062102794647217, + "learning_rate": 0.0001, + "loss": 73.2672, + "step": 2780 + }, + { + "epoch": 4.493374313498261, + "grad_norm": 1.104483962059021, + "learning_rate": 0.0001, + "loss": 70.548, + "step": 2790 + }, + { + "epoch": 4.509497657076636, + "grad_norm": 0.942057728767395, + "learning_rate": 0.0001, + "loss": 72.3584, + "step": 2800 + }, + { + "epoch": 4.525621000655011, + "grad_norm": 0.9028333425521851, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 2810 + }, + { + "epoch": 4.541744344233385, + "grad_norm": 0.9096795916557312, + "learning_rate": 0.0001, + "loss": 70.6596, + "step": 2820 + }, + { + "epoch": 4.55786768781176, + "grad_norm": 0.9038829207420349, + "learning_rate": 0.0001, + "loss": 72.6148, + "step": 2830 + }, + { + "epoch": 4.573991031390134, + "grad_norm": 1.8511613607406616, + "learning_rate": 0.0001, + "loss": 71.7205, + "step": 2840 + }, + { + "epoch": 4.590114374968509, + "grad_norm": 1.3492343425750732, + "learning_rate": 0.0001, + "loss": 72.1492, + "step": 2850 + }, + { + "epoch": 4.6062377185468835, + "grad_norm": 1.1469528675079346, + "learning_rate": 0.0001, + "loss": 70.5892, + "step": 2860 + }, + { + "epoch": 4.622361062125258, + "grad_norm": 1.1374379396438599, + "learning_rate": 0.0001, + "loss": 70.061, + "step": 2870 + }, + { + "epoch": 4.6384844057036325, + "grad_norm": 0.9597565531730652, + "learning_rate": 0.0001, + "loss": 71.0869, + "step": 2880 + }, + { + "epoch": 4.6546077492820075, + "grad_norm": 0.8356521129608154, + "learning_rate": 0.0001, + "loss": 72.1891, + "step": 2890 + }, + { + "epoch": 4.670731092860382, + "grad_norm": 0.8882647752761841, + "learning_rate": 0.0001, + "loss": 71.3845, + "step": 2900 + }, + { + "epoch": 4.6868544364387565, + "grad_norm": 1.1904003620147705, + "learning_rate": 0.0001, + "loss": 72.3086, + "step": 2910 + }, + { + "epoch": 4.702977780017131, + "grad_norm": 0.8010085821151733, + "learning_rate": 0.0001, + "loss": 71.3051, + "step": 2920 + }, + { + "epoch": 4.719101123595506, + "grad_norm": 0.7572135329246521, + "learning_rate": 0.0001, + "loss": 72.5605, + "step": 2930 + }, + { + "epoch": 4.735224467173881, + "grad_norm": 1.0866942405700684, + "learning_rate": 0.0001, + "loss": 70.0935, + "step": 2940 + }, + { + "epoch": 4.751347810752255, + "grad_norm": 0.9358721971511841, + "learning_rate": 0.0001, + "loss": 71.0391, + "step": 2950 + }, + { + "epoch": 4.76747115433063, + "grad_norm": 1.0276161432266235, + "learning_rate": 0.0001, + "loss": 69.8462, + "step": 2960 + }, + { + "epoch": 4.783594497909004, + "grad_norm": 0.9573063850402832, + "learning_rate": 0.0001, + "loss": 69.7089, + "step": 2970 + }, + { + "epoch": 4.799717841487379, + "grad_norm": 1.0638892650604248, + "learning_rate": 0.0001, + "loss": 71.8376, + "step": 2980 + }, + { + "epoch": 4.815841185065753, + "grad_norm": 0.9894681572914124, + "learning_rate": 0.0001, + "loss": 70.1764, + "step": 2990 + }, + { + "epoch": 4.831964528644128, + "grad_norm": 1.1265769004821777, + "learning_rate": 0.0001, + "loss": 71.7918, + "step": 3000 + }, + { + "epoch": 4.848087872222502, + "grad_norm": 2.2792513370513916, + "learning_rate": 0.0001, + "loss": 70.4341, + "step": 3010 + }, + { + "epoch": 4.864211215800877, + "grad_norm": 0.8469329476356506, + "learning_rate": 0.0001, + "loss": 72.1137, + "step": 3020 + }, + { + "epoch": 4.880334559379252, + "grad_norm": 0.9333013296127319, + "learning_rate": 0.0001, + "loss": 71.0789, + "step": 3030 + }, + { + "epoch": 4.896457902957626, + "grad_norm": 0.7888994216918945, + "learning_rate": 0.0001, + "loss": 70.8197, + "step": 3040 + }, + { + "epoch": 4.912581246536, + "grad_norm": 0.9194874167442322, + "learning_rate": 0.0001, + "loss": 72.071, + "step": 3050 + }, + { + "epoch": 4.928704590114375, + "grad_norm": 1.1252421140670776, + "learning_rate": 0.0001, + "loss": 71.6943, + "step": 3060 + }, + { + "epoch": 4.94482793369275, + "grad_norm": 1.1801071166992188, + "learning_rate": 0.0001, + "loss": 70.8937, + "step": 3070 + }, + { + "epoch": 4.960951277271124, + "grad_norm": 0.9162415266036987, + "learning_rate": 0.0001, + "loss": 70.024, + "step": 3080 + }, + { + "epoch": 4.977074620849499, + "grad_norm": 0.8833363652229309, + "learning_rate": 0.0001, + "loss": 71.3835, + "step": 3090 + }, + { + "epoch": 4.993197964427873, + "grad_norm": 0.9985883831977844, + "learning_rate": 0.0001, + "loss": 71.635, + "step": 3100 + }, + { + "epoch": 5.0080616717891875, + "grad_norm": 1.097298264503479, + "learning_rate": 0.0001, + "loss": 65.3542, + "step": 3110 + }, + { + "epoch": 5.024185015367562, + "grad_norm": 0.9703903198242188, + "learning_rate": 0.0001, + "loss": 69.8587, + "step": 3120 + }, + { + "epoch": 5.0403083589459365, + "grad_norm": 0.8894070386886597, + "learning_rate": 0.0001, + "loss": 71.2868, + "step": 3130 + }, + { + "epoch": 5.056431702524311, + "grad_norm": 0.6984127163887024, + "learning_rate": 0.0001, + "loss": 69.0425, + "step": 3140 + }, + { + "epoch": 5.072555046102686, + "grad_norm": 0.8607762455940247, + "learning_rate": 0.0001, + "loss": 69.5354, + "step": 3150 + }, + { + "epoch": 5.08867838968106, + "grad_norm": 1.1215914487838745, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 3160 + }, + { + "epoch": 5.104801733259435, + "grad_norm": 0.9114409685134888, + "learning_rate": 0.0001, + "loss": 68.4186, + "step": 3170 + }, + { + "epoch": 5.12092507683781, + "grad_norm": 1.010400652885437, + "learning_rate": 0.0001, + "loss": 68.3965, + "step": 3180 + }, + { + "epoch": 5.137048420416184, + "grad_norm": 0.8741958141326904, + "learning_rate": 0.0001, + "loss": 70.6728, + "step": 3190 + }, + { + "epoch": 5.153171763994559, + "grad_norm": 0.8760207891464233, + "learning_rate": 0.0001, + "loss": 72.0704, + "step": 3200 + }, + { + "epoch": 5.169295107572933, + "grad_norm": 1.328247308731079, + "learning_rate": 0.0001, + "loss": 69.9082, + "step": 3210 + }, + { + "epoch": 5.185418451151308, + "grad_norm": 0.8263765573501587, + "learning_rate": 0.0001, + "loss": 70.3539, + "step": 3220 + }, + { + "epoch": 5.201541794729682, + "grad_norm": 0.9358834028244019, + "learning_rate": 0.0001, + "loss": 70.4221, + "step": 3230 + }, + { + "epoch": 5.217665138308057, + "grad_norm": 1.0769531726837158, + "learning_rate": 0.0001, + "loss": 68.8167, + "step": 3240 + }, + { + "epoch": 5.233788481886431, + "grad_norm": 1.1438790559768677, + "learning_rate": 0.0001, + "loss": 68.9288, + "step": 3250 + }, + { + "epoch": 5.249911825464806, + "grad_norm": 0.9534218311309814, + "learning_rate": 0.0001, + "loss": 70.3277, + "step": 3260 + }, + { + "epoch": 5.26603516904318, + "grad_norm": 0.88532954454422, + "learning_rate": 0.0001, + "loss": 67.5781, + "step": 3270 + }, + { + "epoch": 5.282158512621555, + "grad_norm": 0.7986133694648743, + "learning_rate": 0.0001, + "loss": 69.759, + "step": 3280 + }, + { + "epoch": 5.29828185619993, + "grad_norm": 0.9100566506385803, + "learning_rate": 0.0001, + "loss": 68.6368, + "step": 3290 + }, + { + "epoch": 5.314405199778304, + "grad_norm": 0.8861745595932007, + "learning_rate": 0.0001, + "loss": 70.5756, + "step": 3300 + }, + { + "epoch": 5.330528543356679, + "grad_norm": 1.0729680061340332, + "learning_rate": 0.0001, + "loss": 70.3178, + "step": 3310 + }, + { + "epoch": 5.346651886935053, + "grad_norm": 1.0288280248641968, + "learning_rate": 0.0001, + "loss": 69.422, + "step": 3320 + }, + { + "epoch": 5.362775230513428, + "grad_norm": 0.8335140347480774, + "learning_rate": 0.0001, + "loss": 69.8281, + "step": 3330 + }, + { + "epoch": 5.378898574091802, + "grad_norm": 0.9789717197418213, + "learning_rate": 0.0001, + "loss": 71.1532, + "step": 3340 + }, + { + "epoch": 5.395021917670177, + "grad_norm": 0.9884677529335022, + "learning_rate": 0.0001, + "loss": 69.5673, + "step": 3350 + }, + { + "epoch": 5.411145261248551, + "grad_norm": 1.050154447555542, + "learning_rate": 0.0001, + "loss": 70.8015, + "step": 3360 + }, + { + "epoch": 5.427268604826926, + "grad_norm": 0.9645342230796814, + "learning_rate": 0.0001, + "loss": 69.1615, + "step": 3370 + }, + { + "epoch": 5.4433919484053, + "grad_norm": 0.8383159041404724, + "learning_rate": 0.0001, + "loss": 70.6299, + "step": 3380 + }, + { + "epoch": 5.459515291983675, + "grad_norm": 0.9250556826591492, + "learning_rate": 0.0001, + "loss": 69.231, + "step": 3390 + }, + { + "epoch": 5.475638635562049, + "grad_norm": 0.8108687400817871, + "learning_rate": 0.0001, + "loss": 70.5858, + "step": 3400 + }, + { + "epoch": 5.491761979140424, + "grad_norm": 0.8748818039894104, + "learning_rate": 0.0001, + "loss": 69.9001, + "step": 3410 + }, + { + "epoch": 5.507885322718799, + "grad_norm": 0.7411381006240845, + "learning_rate": 0.0001, + "loss": 68.3248, + "step": 3420 + }, + { + "epoch": 5.524008666297173, + "grad_norm": 1.0753326416015625, + "learning_rate": 0.0001, + "loss": 69.5831, + "step": 3430 + }, + { + "epoch": 5.540132009875548, + "grad_norm": 0.875254213809967, + "learning_rate": 0.0001, + "loss": 69.7516, + "step": 3440 + }, + { + "epoch": 5.556255353453922, + "grad_norm": 0.7637733817100525, + "learning_rate": 0.0001, + "loss": 68.0078, + "step": 3450 + }, + { + "epoch": 5.572378697032297, + "grad_norm": 0.968203604221344, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 3460 + }, + { + "epoch": 5.588502040610671, + "grad_norm": 0.8903141617774963, + "learning_rate": 0.0001, + "loss": 68.5835, + "step": 3470 + }, + { + "epoch": 5.604625384189046, + "grad_norm": 0.9536195397377014, + "learning_rate": 0.0001, + "loss": 71.0259, + "step": 3480 + }, + { + "epoch": 5.62074872776742, + "grad_norm": 0.8951714038848877, + "learning_rate": 0.0001, + "loss": 69.4422, + "step": 3490 + }, + { + "epoch": 5.636872071345795, + "grad_norm": 0.9639522433280945, + "learning_rate": 0.0001, + "loss": 66.5173, + "step": 3500 + }, + { + "epoch": 5.65299541492417, + "grad_norm": 1.0022730827331543, + "learning_rate": 0.0001, + "loss": 68.3443, + "step": 3510 + }, + { + "epoch": 5.669118758502544, + "grad_norm": 1.0654487609863281, + "learning_rate": 0.0001, + "loss": 69.0835, + "step": 3520 + }, + { + "epoch": 5.685242102080919, + "grad_norm": 0.9934629797935486, + "learning_rate": 0.0001, + "loss": 69.2278, + "step": 3530 + }, + { + "epoch": 5.7013654456592935, + "grad_norm": 0.8773566484451294, + "learning_rate": 0.0001, + "loss": 68.9339, + "step": 3540 + }, + { + "epoch": 5.7174887892376685, + "grad_norm": 0.7756998538970947, + "learning_rate": 0.0001, + "loss": 69.2683, + "step": 3550 + }, + { + "epoch": 5.7336121328160425, + "grad_norm": 0.8819359540939331, + "learning_rate": 0.0001, + "loss": 68.5071, + "step": 3560 + }, + { + "epoch": 5.7497354763944175, + "grad_norm": 1.1069825887680054, + "learning_rate": 0.0001, + "loss": 68.5236, + "step": 3570 + }, + { + "epoch": 5.765858819972792, + "grad_norm": 0.8006312847137451, + "learning_rate": 0.0001, + "loss": 68.7192, + "step": 3580 + }, + { + "epoch": 5.781982163551167, + "grad_norm": 1.0278719663619995, + "learning_rate": 0.0001, + "loss": 69.2545, + "step": 3590 + }, + { + "epoch": 5.798105507129541, + "grad_norm": 0.9205041527748108, + "learning_rate": 0.0001, + "loss": 69.7698, + "step": 3600 + }, + { + "epoch": 5.814228850707916, + "grad_norm": 1.0769038200378418, + "learning_rate": 0.0001, + "loss": 67.6568, + "step": 3610 + }, + { + "epoch": 5.83035219428629, + "grad_norm": 1.0565663576126099, + "learning_rate": 0.0001, + "loss": 68.9166, + "step": 3620 + }, + { + "epoch": 5.846475537864665, + "grad_norm": 0.9008663296699524, + "learning_rate": 0.0001, + "loss": 69.5837, + "step": 3630 + }, + { + "epoch": 5.86259888144304, + "grad_norm": 0.8369411826133728, + "learning_rate": 0.0001, + "loss": 68.5661, + "step": 3640 + }, + { + "epoch": 5.878722225021414, + "grad_norm": 1.0472379922866821, + "learning_rate": 0.0001, + "loss": 68.3122, + "step": 3650 + }, + { + "epoch": 5.894845568599789, + "grad_norm": 0.965277373790741, + "learning_rate": 0.0001, + "loss": 69.8062, + "step": 3660 + }, + { + "epoch": 5.910968912178163, + "grad_norm": 0.8936495780944824, + "learning_rate": 0.0001, + "loss": 67.1016, + "step": 3670 + }, + { + "epoch": 5.927092255756538, + "grad_norm": 0.8559162020683289, + "learning_rate": 0.0001, + "loss": 69.9986, + "step": 3680 + }, + { + "epoch": 5.943215599334912, + "grad_norm": 1.4815773963928223, + "learning_rate": 0.0001, + "loss": 66.7648, + "step": 3690 + }, + { + "epoch": 5.959338942913287, + "grad_norm": 0.9034469723701477, + "learning_rate": 0.0001, + "loss": 68.0767, + "step": 3700 + }, + { + "epoch": 5.975462286491661, + "grad_norm": 1.838799238204956, + "learning_rate": 0.0001, + "loss": 66.5848, + "step": 3710 + }, + { + "epoch": 5.991585630070036, + "grad_norm": 0.9274787902832031, + "learning_rate": 0.0001, + "loss": 70.0396, + "step": 3720 + }, + { + "epoch": 6.0064493374313495, + "grad_norm": 1.016974687576294, + "learning_rate": 0.0001, + "loss": 62.2268, + "step": 3730 + }, + { + "epoch": 6.022572681009724, + "grad_norm": 1.0522642135620117, + "learning_rate": 0.0001, + "loss": 69.3381, + "step": 3740 + }, + { + "epoch": 6.0386960245880985, + "grad_norm": 1.1157186031341553, + "learning_rate": 0.0001, + "loss": 68.8969, + "step": 3750 + }, + { + "epoch": 6.0548193681664735, + "grad_norm": 0.9913288950920105, + "learning_rate": 0.0001, + "loss": 67.3634, + "step": 3760 + }, + { + "epoch": 6.0709427117448485, + "grad_norm": 0.8823490142822266, + "learning_rate": 0.0001, + "loss": 70.978, + "step": 3770 + }, + { + "epoch": 6.0870660553232225, + "grad_norm": 0.853837788105011, + "learning_rate": 0.0001, + "loss": 68.9924, + "step": 3780 + }, + { + "epoch": 6.1031893989015975, + "grad_norm": 0.7937766909599304, + "learning_rate": 0.0001, + "loss": 68.1994, + "step": 3790 + }, + { + "epoch": 6.119312742479972, + "grad_norm": 1.7638057470321655, + "learning_rate": 0.0001, + "loss": 68.5078, + "step": 3800 + }, + { + "epoch": 6.135436086058347, + "grad_norm": 1.0542887449264526, + "learning_rate": 0.0001, + "loss": 68.9362, + "step": 3810 + }, + { + "epoch": 6.151559429636721, + "grad_norm": 1.06019926071167, + "learning_rate": 0.0001, + "loss": 67.2493, + "step": 3820 + }, + { + "epoch": 6.167682773215096, + "grad_norm": 0.9765111804008484, + "learning_rate": 0.0001, + "loss": 70.4797, + "step": 3830 + }, + { + "epoch": 6.18380611679347, + "grad_norm": 1.006372332572937, + "learning_rate": 0.0001, + "loss": 69.4233, + "step": 3840 + }, + { + "epoch": 6.199929460371845, + "grad_norm": 0.8274241089820862, + "learning_rate": 0.0001, + "loss": 67.9907, + "step": 3850 + }, + { + "epoch": 6.216052803950219, + "grad_norm": 0.9905765056610107, + "learning_rate": 0.0001, + "loss": 69.6988, + "step": 3860 + }, + { + "epoch": 6.232176147528594, + "grad_norm": 1.0065921545028687, + "learning_rate": 0.0001, + "loss": 68.7678, + "step": 3870 + }, + { + "epoch": 6.248299491106969, + "grad_norm": 0.8845255970954895, + "learning_rate": 0.0001, + "loss": 68.3039, + "step": 3880 + }, + { + "epoch": 6.264422834685343, + "grad_norm": 0.9030968546867371, + "learning_rate": 0.0001, + "loss": 68.671, + "step": 3890 + }, + { + "epoch": 6.280546178263718, + "grad_norm": 0.8650258183479309, + "learning_rate": 0.0001, + "loss": 68.5309, + "step": 3900 + }, + { + "epoch": 6.296669521842092, + "grad_norm": 0.8961272835731506, + "learning_rate": 0.0001, + "loss": 67.5611, + "step": 3910 + }, + { + "epoch": 6.312792865420467, + "grad_norm": 1.1702407598495483, + "learning_rate": 0.0001, + "loss": 68.2768, + "step": 3920 + }, + { + "epoch": 6.328916208998841, + "grad_norm": 0.8598962426185608, + "learning_rate": 0.0001, + "loss": 69.1961, + "step": 3930 + }, + { + "epoch": 6.345039552577216, + "grad_norm": 0.8714122176170349, + "learning_rate": 0.0001, + "loss": 66.8628, + "step": 3940 + }, + { + "epoch": 6.36116289615559, + "grad_norm": 0.7757895588874817, + "learning_rate": 0.0001, + "loss": 68.6908, + "step": 3950 + }, + { + "epoch": 6.377286239733965, + "grad_norm": 0.9189490675926208, + "learning_rate": 0.0001, + "loss": 67.8685, + "step": 3960 + }, + { + "epoch": 6.393409583312339, + "grad_norm": 0.8161299228668213, + "learning_rate": 0.0001, + "loss": 66.816, + "step": 3970 + }, + { + "epoch": 6.409532926890714, + "grad_norm": 0.8559791445732117, + "learning_rate": 0.0001, + "loss": 68.4302, + "step": 3980 + }, + { + "epoch": 6.425656270469089, + "grad_norm": 0.8087984919548035, + "learning_rate": 0.0001, + "loss": 66.2646, + "step": 3990 + }, + { + "epoch": 6.441779614047463, + "grad_norm": 0.7733110785484314, + "learning_rate": 0.0001, + "loss": 69.2769, + "step": 4000 + }, + { + "epoch": 6.457902957625838, + "grad_norm": 0.9335984587669373, + "learning_rate": 0.0001, + "loss": 67.1199, + "step": 4010 + }, + { + "epoch": 6.474026301204212, + "grad_norm": 0.9546418786048889, + "learning_rate": 0.0001, + "loss": 68.8706, + "step": 4020 + }, + { + "epoch": 6.490149644782587, + "grad_norm": 0.9670593738555908, + "learning_rate": 0.0001, + "loss": 68.2329, + "step": 4030 + }, + { + "epoch": 6.506272988360961, + "grad_norm": 0.8428588509559631, + "learning_rate": 0.0001, + "loss": 68.0105, + "step": 4040 + }, + { + "epoch": 6.522396331939336, + "grad_norm": 0.9755622744560242, + "learning_rate": 0.0001, + "loss": 68.656, + "step": 4050 + }, + { + "epoch": 6.53851967551771, + "grad_norm": 0.8972346186637878, + "learning_rate": 0.0001, + "loss": 69.3665, + "step": 4060 + }, + { + "epoch": 6.554643019096085, + "grad_norm": 0.9716668128967285, + "learning_rate": 0.0001, + "loss": 68.1053, + "step": 4070 + }, + { + "epoch": 6.570766362674459, + "grad_norm": 0.8147910833358765, + "learning_rate": 0.0001, + "loss": 67.9778, + "step": 4080 + }, + { + "epoch": 6.586889706252834, + "grad_norm": 0.8304234147071838, + "learning_rate": 0.0001, + "loss": 67.7069, + "step": 4090 + }, + { + "epoch": 6.603013049831208, + "grad_norm": 0.9356856346130371, + "learning_rate": 0.0001, + "loss": 68.1942, + "step": 4100 + }, + { + "epoch": 6.619136393409583, + "grad_norm": 1.017134189605713, + "learning_rate": 0.0001, + "loss": 67.4211, + "step": 4110 + }, + { + "epoch": 6.635259736987958, + "grad_norm": 1.142558217048645, + "learning_rate": 0.0001, + "loss": 68.4413, + "step": 4120 + }, + { + "epoch": 6.651383080566332, + "grad_norm": 0.9071153998374939, + "learning_rate": 0.0001, + "loss": 69.3666, + "step": 4130 + }, + { + "epoch": 6.667506424144707, + "grad_norm": 0.9620397686958313, + "learning_rate": 0.0001, + "loss": 68.208, + "step": 4140 + }, + { + "epoch": 6.683629767723081, + "grad_norm": 0.8282128572463989, + "learning_rate": 0.0001, + "loss": 67.1702, + "step": 4150 + }, + { + "epoch": 6.699753111301456, + "grad_norm": 0.8838199973106384, + "learning_rate": 0.0001, + "loss": 69.0074, + "step": 4160 + }, + { + "epoch": 6.71587645487983, + "grad_norm": 0.8864356279373169, + "learning_rate": 0.0001, + "loss": 67.8052, + "step": 4170 + }, + { + "epoch": 6.731999798458205, + "grad_norm": 0.9373385906219482, + "learning_rate": 0.0001, + "loss": 67.446, + "step": 4180 + }, + { + "epoch": 6.7481231420365795, + "grad_norm": 0.8308956027030945, + "learning_rate": 0.0001, + "loss": 68.716, + "step": 4190 + }, + { + "epoch": 6.7642464856149545, + "grad_norm": 0.8031372427940369, + "learning_rate": 0.0001, + "loss": 68.0482, + "step": 4200 + }, + { + "epoch": 6.780369829193329, + "grad_norm": 0.9258586764335632, + "learning_rate": 0.0001, + "loss": 68.5607, + "step": 4210 + }, + { + "epoch": 6.7964931727717035, + "grad_norm": 0.8147327899932861, + "learning_rate": 0.0001, + "loss": 67.7265, + "step": 4220 + }, + { + "epoch": 6.8126165163500785, + "grad_norm": 0.8717535734176636, + "learning_rate": 0.0001, + "loss": 67.8097, + "step": 4230 + }, + { + "epoch": 6.828739859928453, + "grad_norm": 0.9876209497451782, + "learning_rate": 0.0001, + "loss": 67.9068, + "step": 4240 + }, + { + "epoch": 6.8448632035068275, + "grad_norm": 0.718376874923706, + "learning_rate": 0.0001, + "loss": 68.1762, + "step": 4250 + }, + { + "epoch": 6.860986547085202, + "grad_norm": 0.8264320492744446, + "learning_rate": 0.0001, + "loss": 67.8391, + "step": 4260 + }, + { + "epoch": 6.877109890663577, + "grad_norm": 0.7478179335594177, + "learning_rate": 0.0001, + "loss": 67.6635, + "step": 4270 + }, + { + "epoch": 6.893233234241951, + "grad_norm": 0.8394957184791565, + "learning_rate": 0.0001, + "loss": 68.6073, + "step": 4280 + }, + { + "epoch": 6.909356577820326, + "grad_norm": 0.8534071445465088, + "learning_rate": 0.0001, + "loss": 69.6206, + "step": 4290 + }, + { + "epoch": 6.9254799213987, + "grad_norm": 0.9137582778930664, + "learning_rate": 0.0001, + "loss": 67.4362, + "step": 4300 + }, + { + "epoch": 6.941603264977075, + "grad_norm": 0.8869971036911011, + "learning_rate": 0.0001, + "loss": 67.6811, + "step": 4310 + }, + { + "epoch": 6.957726608555449, + "grad_norm": 0.889025866985321, + "learning_rate": 0.0001, + "loss": 65.3354, + "step": 4320 + }, + { + "epoch": 6.973849952133824, + "grad_norm": 0.7968549728393555, + "learning_rate": 0.0001, + "loss": 66.192, + "step": 4330 + }, + { + "epoch": 6.989973295712199, + "grad_norm": 1.0077433586120605, + "learning_rate": 0.0001, + "loss": 67.5216, + "step": 4340 + }, + { + "epoch": 7.004837003073512, + "grad_norm": 0.7900649309158325, + "learning_rate": 0.0001, + "loss": 62.2421, + "step": 4350 + }, + { + "epoch": 7.020960346651887, + "grad_norm": 1.0160410404205322, + "learning_rate": 0.0001, + "loss": 68.0978, + "step": 4360 + }, + { + "epoch": 7.037083690230261, + "grad_norm": 0.841387927532196, + "learning_rate": 0.0001, + "loss": 68.1008, + "step": 4370 + }, + { + "epoch": 7.053207033808636, + "grad_norm": 0.87884122133255, + "learning_rate": 0.0001, + "loss": 66.4667, + "step": 4380 + }, + { + "epoch": 7.06933037738701, + "grad_norm": 0.8580564856529236, + "learning_rate": 0.0001, + "loss": 67.2002, + "step": 4390 + }, + { + "epoch": 7.085453720965385, + "grad_norm": 0.7560063600540161, + "learning_rate": 0.0001, + "loss": 67.6743, + "step": 4400 + }, + { + "epoch": 7.1015770645437595, + "grad_norm": 0.855478048324585, + "learning_rate": 0.0001, + "loss": 67.822, + "step": 4410 + }, + { + "epoch": 7.1177004081221344, + "grad_norm": 0.7994911670684814, + "learning_rate": 0.0001, + "loss": 66.0094, + "step": 4420 + }, + { + "epoch": 7.1338237517005085, + "grad_norm": 0.8728616833686829, + "learning_rate": 0.0001, + "loss": 66.405, + "step": 4430 + }, + { + "epoch": 7.1499470952788835, + "grad_norm": 0.8434081673622131, + "learning_rate": 0.0001, + "loss": 67.0535, + "step": 4440 + }, + { + "epoch": 7.166070438857258, + "grad_norm": 0.8543094396591187, + "learning_rate": 0.0001, + "loss": 66.2417, + "step": 4450 + }, + { + "epoch": 7.182193782435633, + "grad_norm": 0.7209319472312927, + "learning_rate": 0.0001, + "loss": 67.3958, + "step": 4460 + }, + { + "epoch": 7.1983171260140075, + "grad_norm": 0.6735154390335083, + "learning_rate": 0.0001, + "loss": 67.4086, + "step": 4470 + }, + { + "epoch": 7.214440469592382, + "grad_norm": 0.9126104116439819, + "learning_rate": 0.0001, + "loss": 68.4191, + "step": 4480 + }, + { + "epoch": 7.230563813170757, + "grad_norm": 0.9334300756454468, + "learning_rate": 0.0001, + "loss": 68.3247, + "step": 4490 + }, + { + "epoch": 7.246687156749131, + "grad_norm": 1.5956038236618042, + "learning_rate": 0.0001, + "loss": 68.2779, + "step": 4500 + }, + { + "epoch": 7.262810500327506, + "grad_norm": 1.5012775659561157, + "learning_rate": 0.0001, + "loss": 68.3533, + "step": 4510 + }, + { + "epoch": 7.27893384390588, + "grad_norm": 1.1376632452011108, + "learning_rate": 0.0001, + "loss": 65.3627, + "step": 4520 + }, + { + "epoch": 7.295057187484255, + "grad_norm": 0.8597291111946106, + "learning_rate": 0.0001, + "loss": 67.8653, + "step": 4530 + }, + { + "epoch": 7.311180531062629, + "grad_norm": 0.7645048499107361, + "learning_rate": 0.0001, + "loss": 65.4592, + "step": 4540 + }, + { + "epoch": 7.327303874641004, + "grad_norm": 0.7717318534851074, + "learning_rate": 0.0001, + "loss": 66.902, + "step": 4550 + }, + { + "epoch": 7.343427218219378, + "grad_norm": 0.8884950876235962, + "learning_rate": 0.0001, + "loss": 67.1858, + "step": 4560 + }, + { + "epoch": 7.359550561797753, + "grad_norm": 0.7826898694038391, + "learning_rate": 0.0001, + "loss": 68.4673, + "step": 4570 + }, + { + "epoch": 7.375673905376128, + "grad_norm": 0.7484182119369507, + "learning_rate": 0.0001, + "loss": 67.4859, + "step": 4580 + }, + { + "epoch": 7.391797248954502, + "grad_norm": 0.918194591999054, + "learning_rate": 0.0001, + "loss": 67.7393, + "step": 4590 + }, + { + "epoch": 7.407920592532877, + "grad_norm": 1.0696896314620972, + "learning_rate": 0.0001, + "loss": 67.0101, + "step": 4600 + }, + { + "epoch": 7.424043936111251, + "grad_norm": 0.8190692067146301, + "learning_rate": 0.0001, + "loss": 68.5358, + "step": 4610 + }, + { + "epoch": 7.440167279689626, + "grad_norm": 0.9063783884048462, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 4620 + }, + { + "epoch": 7.456290623268, + "grad_norm": 0.7588100433349609, + "learning_rate": 0.0001, + "loss": 68.0066, + "step": 4630 + }, + { + "epoch": 7.472413966846375, + "grad_norm": 0.897281289100647, + "learning_rate": 0.0001, + "loss": 67.2499, + "step": 4640 + }, + { + "epoch": 7.488537310424749, + "grad_norm": 0.9505569338798523, + "learning_rate": 0.0001, + "loss": 67.5452, + "step": 4650 + }, + { + "epoch": 7.504660654003124, + "grad_norm": 0.8409159183502197, + "learning_rate": 0.0001, + "loss": 66.6293, + "step": 4660 + }, + { + "epoch": 7.520783997581498, + "grad_norm": 0.9878574013710022, + "learning_rate": 0.0001, + "loss": 65.3379, + "step": 4670 + }, + { + "epoch": 7.536907341159873, + "grad_norm": 1.0410685539245605, + "learning_rate": 0.0001, + "loss": 65.8028, + "step": 4680 + }, + { + "epoch": 7.553030684738248, + "grad_norm": 0.8417725563049316, + "learning_rate": 0.0001, + "loss": 65.4283, + "step": 4690 + }, + { + "epoch": 7.569154028316622, + "grad_norm": 0.7660863399505615, + "learning_rate": 0.0001, + "loss": 66.6765, + "step": 4700 + }, + { + "epoch": 7.585277371894997, + "grad_norm": 0.8074233531951904, + "learning_rate": 0.0001, + "loss": 67.1082, + "step": 4710 + }, + { + "epoch": 7.601400715473371, + "grad_norm": 0.7653326988220215, + "learning_rate": 0.0001, + "loss": 66.028, + "step": 4720 + }, + { + "epoch": 7.617524059051746, + "grad_norm": 1.1764041185379028, + "learning_rate": 0.0001, + "loss": 65.2587, + "step": 4730 + }, + { + "epoch": 7.63364740263012, + "grad_norm": 1.7604950666427612, + "learning_rate": 0.0001, + "loss": 66.0543, + "step": 4740 + }, + { + "epoch": 7.649770746208495, + "grad_norm": 0.941438615322113, + "learning_rate": 0.0001, + "loss": 65.6954, + "step": 4750 + }, + { + "epoch": 7.665894089786869, + "grad_norm": 0.740069568157196, + "learning_rate": 0.0001, + "loss": 67.9701, + "step": 4760 + }, + { + "epoch": 7.682017433365244, + "grad_norm": 0.9867332577705383, + "learning_rate": 0.0001, + "loss": 66.1445, + "step": 4770 + }, + { + "epoch": 7.698140776943618, + "grad_norm": 0.8156166672706604, + "learning_rate": 0.0001, + "loss": 67.2647, + "step": 4780 + }, + { + "epoch": 7.714264120521993, + "grad_norm": 0.7689579129219055, + "learning_rate": 0.0001, + "loss": 67.0961, + "step": 4790 + }, + { + "epoch": 7.730387464100367, + "grad_norm": 0.8159700036048889, + "learning_rate": 0.0001, + "loss": 66.4974, + "step": 4800 + }, + { + "epoch": 7.746510807678742, + "grad_norm": 0.8554222583770752, + "learning_rate": 0.0001, + "loss": 64.5057, + "step": 4810 + }, + { + "epoch": 7.762634151257117, + "grad_norm": 0.856629490852356, + "learning_rate": 0.0001, + "loss": 67.7819, + "step": 4820 + }, + { + "epoch": 7.778757494835491, + "grad_norm": 1.157179832458496, + "learning_rate": 0.0001, + "loss": 66.0122, + "step": 4830 + }, + { + "epoch": 7.794880838413866, + "grad_norm": 0.9597711563110352, + "learning_rate": 0.0001, + "loss": 66.8248, + "step": 4840 + }, + { + "epoch": 7.8110041819922404, + "grad_norm": 0.9175744652748108, + "learning_rate": 0.0001, + "loss": 65.7998, + "step": 4850 + }, + { + "epoch": 7.827127525570615, + "grad_norm": 0.8467298150062561, + "learning_rate": 0.0001, + "loss": 67.5363, + "step": 4860 + }, + { + "epoch": 7.8432508691489895, + "grad_norm": 0.863922655582428, + "learning_rate": 0.0001, + "loss": 67.7219, + "step": 4870 + }, + { + "epoch": 7.8593742127273645, + "grad_norm": 0.9896290302276611, + "learning_rate": 0.0001, + "loss": 66.7098, + "step": 4880 + }, + { + "epoch": 7.8754975563057386, + "grad_norm": 0.8828520774841309, + "learning_rate": 0.0001, + "loss": 66.0192, + "step": 4890 + }, + { + "epoch": 7.8916208998841135, + "grad_norm": 0.9093656539916992, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 4900 + }, + { + "epoch": 7.9077442434624885, + "grad_norm": 0.8665091395378113, + "learning_rate": 0.0001, + "loss": 66.5851, + "step": 4910 + }, + { + "epoch": 7.923867587040863, + "grad_norm": 0.8027398586273193, + "learning_rate": 0.0001, + "loss": 64.8607, + "step": 4920 + }, + { + "epoch": 7.9399909306192376, + "grad_norm": 0.7808436751365662, + "learning_rate": 0.0001, + "loss": 65.052, + "step": 4930 + }, + { + "epoch": 7.956114274197612, + "grad_norm": 0.7246070504188538, + "learning_rate": 0.0001, + "loss": 64.789, + "step": 4940 + }, + { + "epoch": 7.972237617775987, + "grad_norm": 1.0405027866363525, + "learning_rate": 0.0001, + "loss": 66.7735, + "step": 4950 + }, + { + "epoch": 7.988360961354361, + "grad_norm": 0.812921404838562, + "learning_rate": 0.0001, + "loss": 64.1286, + "step": 4960 + }, + { + "epoch": 8.003224668715674, + "grad_norm": 0.884215772151947, + "learning_rate": 0.0001, + "loss": 61.0812, + "step": 4970 + }, + { + "epoch": 8.01934801229405, + "grad_norm": 0.8519641757011414, + "learning_rate": 0.0001, + "loss": 66.2784, + "step": 4980 + }, + { + "epoch": 8.035471355872424, + "grad_norm": 0.8331199884414673, + "learning_rate": 0.0001, + "loss": 67.1405, + "step": 4990 + }, + { + "epoch": 8.0515946994508, + "grad_norm": 0.8568746447563171, + "learning_rate": 0.0001, + "loss": 66.7835, + "step": 5000 + }, + { + "epoch": 8.067718043029172, + "grad_norm": 1.170482873916626, + "learning_rate": 0.0001, + "loss": 64.2269, + "step": 5010 + }, + { + "epoch": 8.083841386607547, + "grad_norm": 0.9171113967895508, + "learning_rate": 0.0001, + "loss": 66.5567, + "step": 5020 + }, + { + "epoch": 8.099964730185922, + "grad_norm": 0.8988714814186096, + "learning_rate": 0.0001, + "loss": 66.0471, + "step": 5030 + }, + { + "epoch": 8.116088073764297, + "grad_norm": 0.7651671171188354, + "learning_rate": 0.0001, + "loss": 65.7185, + "step": 5040 + }, + { + "epoch": 8.132211417342672, + "grad_norm": 0.6890843510627747, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 5050 + }, + { + "epoch": 8.148334760921045, + "grad_norm": 0.7683818340301514, + "learning_rate": 0.0001, + "loss": 65.895, + "step": 5060 + }, + { + "epoch": 8.16445810449942, + "grad_norm": 0.9398573637008667, + "learning_rate": 0.0001, + "loss": 64.674, + "step": 5070 + }, + { + "epoch": 8.180581448077795, + "grad_norm": 0.7708531022071838, + "learning_rate": 0.0001, + "loss": 65.0637, + "step": 5080 + }, + { + "epoch": 8.19670479165617, + "grad_norm": 0.8840665221214294, + "learning_rate": 0.0001, + "loss": 65.6449, + "step": 5090 + }, + { + "epoch": 8.212828135234544, + "grad_norm": 0.8390524983406067, + "learning_rate": 0.0001, + "loss": 65.4738, + "step": 5100 + }, + { + "epoch": 8.228951478812919, + "grad_norm": 1.7559128999710083, + "learning_rate": 0.0001, + "loss": 68.911, + "step": 5110 + }, + { + "epoch": 8.245074822391294, + "grad_norm": 0.8293817043304443, + "learning_rate": 0.0001, + "loss": 65.2601, + "step": 5120 + }, + { + "epoch": 8.261198165969668, + "grad_norm": 0.8947123289108276, + "learning_rate": 0.0001, + "loss": 65.8611, + "step": 5130 + }, + { + "epoch": 8.277321509548042, + "grad_norm": 0.8526618480682373, + "learning_rate": 0.0001, + "loss": 65.559, + "step": 5140 + }, + { + "epoch": 8.293444853126417, + "grad_norm": 1.7423408031463623, + "learning_rate": 0.0001, + "loss": 64.5007, + "step": 5150 + }, + { + "epoch": 8.309568196704792, + "grad_norm": 1.0512502193450928, + "learning_rate": 0.0001, + "loss": 66.2593, + "step": 5160 + }, + { + "epoch": 8.325691540283167, + "grad_norm": 0.8129869103431702, + "learning_rate": 0.0001, + "loss": 64.7989, + "step": 5170 + }, + { + "epoch": 8.341814883861542, + "grad_norm": 0.852199912071228, + "learning_rate": 0.0001, + "loss": 65.9627, + "step": 5180 + }, + { + "epoch": 8.357938227439915, + "grad_norm": 0.8932474851608276, + "learning_rate": 0.0001, + "loss": 64.1035, + "step": 5190 + }, + { + "epoch": 8.37406157101829, + "grad_norm": 1.2631597518920898, + "learning_rate": 0.0001, + "loss": 65.3406, + "step": 5200 + }, + { + "epoch": 8.390184914596665, + "grad_norm": 0.9062619209289551, + "learning_rate": 0.0001, + "loss": 65.1841, + "step": 5210 + }, + { + "epoch": 8.40630825817504, + "grad_norm": 0.9153940081596375, + "learning_rate": 0.0001, + "loss": 66.3852, + "step": 5220 + }, + { + "epoch": 8.422431601753413, + "grad_norm": 0.8056164979934692, + "learning_rate": 0.0001, + "loss": 66.511, + "step": 5230 + }, + { + "epoch": 8.438554945331788, + "grad_norm": 0.7506036162376404, + "learning_rate": 0.0001, + "loss": 65.275, + "step": 5240 + }, + { + "epoch": 8.454678288910163, + "grad_norm": 0.7994081377983093, + "learning_rate": 0.0001, + "loss": 64.9946, + "step": 5250 + }, + { + "epoch": 8.470801632488538, + "grad_norm": 0.9887873530387878, + "learning_rate": 0.0001, + "loss": 67.2057, + "step": 5260 + }, + { + "epoch": 8.486924976066913, + "grad_norm": 0.8495044112205505, + "learning_rate": 0.0001, + "loss": 66.2481, + "step": 5270 + }, + { + "epoch": 8.503048319645286, + "grad_norm": 0.7353820204734802, + "learning_rate": 0.0001, + "loss": 65.6163, + "step": 5280 + }, + { + "epoch": 8.519171663223661, + "grad_norm": 0.795216977596283, + "learning_rate": 0.0001, + "loss": 64.808, + "step": 5290 + }, + { + "epoch": 8.535295006802036, + "grad_norm": 0.8121851682662964, + "learning_rate": 0.0001, + "loss": 65.9755, + "step": 5300 + }, + { + "epoch": 8.55141835038041, + "grad_norm": 0.8164243698120117, + "learning_rate": 0.0001, + "loss": 66.0583, + "step": 5310 + }, + { + "epoch": 8.567541693958784, + "grad_norm": 0.9893137216567993, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 5320 + }, + { + "epoch": 8.583665037537159, + "grad_norm": 1.3616434335708618, + "learning_rate": 0.0001, + "loss": 64.1444, + "step": 5330 + }, + { + "epoch": 8.599788381115534, + "grad_norm": 0.7002740502357483, + "learning_rate": 0.0001, + "loss": 65.0906, + "step": 5340 + }, + { + "epoch": 8.615911724693909, + "grad_norm": 0.838424563407898, + "learning_rate": 0.0001, + "loss": 65.4578, + "step": 5350 + }, + { + "epoch": 8.632035068272282, + "grad_norm": 0.988902747631073, + "learning_rate": 0.0001, + "loss": 64.3405, + "step": 5360 + }, + { + "epoch": 8.648158411850657, + "grad_norm": 0.7941252589225769, + "learning_rate": 0.0001, + "loss": 67.0146, + "step": 5370 + }, + { + "epoch": 8.664281755429032, + "grad_norm": 0.7727184295654297, + "learning_rate": 0.0001, + "loss": 64.8011, + "step": 5380 + }, + { + "epoch": 8.680405099007407, + "grad_norm": 0.9514367580413818, + "learning_rate": 0.0001, + "loss": 65.8165, + "step": 5390 + }, + { + "epoch": 8.69652844258578, + "grad_norm": 0.870544970035553, + "learning_rate": 0.0001, + "loss": 65.803, + "step": 5400 + }, + { + "epoch": 8.712651786164155, + "grad_norm": 0.7348082065582275, + "learning_rate": 0.0001, + "loss": 65.8409, + "step": 5410 + }, + { + "epoch": 8.72877512974253, + "grad_norm": 0.7543814778327942, + "learning_rate": 0.0001, + "loss": 65.5958, + "step": 5420 + }, + { + "epoch": 8.744898473320905, + "grad_norm": 0.7520368099212646, + "learning_rate": 0.0001, + "loss": 65.3323, + "step": 5430 + }, + { + "epoch": 8.76102181689928, + "grad_norm": 0.7610031366348267, + "learning_rate": 0.0001, + "loss": 65.4386, + "step": 5440 + }, + { + "epoch": 8.777145160477653, + "grad_norm": 0.8854930400848389, + "learning_rate": 0.0001, + "loss": 66.181, + "step": 5450 + }, + { + "epoch": 8.793268504056028, + "grad_norm": 0.7659342885017395, + "learning_rate": 0.0001, + "loss": 65.2586, + "step": 5460 + }, + { + "epoch": 8.809391847634403, + "grad_norm": 0.8592572212219238, + "learning_rate": 0.0001, + "loss": 66.3306, + "step": 5470 + }, + { + "epoch": 8.825515191212778, + "grad_norm": 0.8169172406196594, + "learning_rate": 0.0001, + "loss": 65.8283, + "step": 5480 + }, + { + "epoch": 8.841638534791151, + "grad_norm": 0.7380614876747131, + "learning_rate": 0.0001, + "loss": 65.8711, + "step": 5490 + }, + { + "epoch": 8.857761878369526, + "grad_norm": 0.9993331432342529, + "learning_rate": 0.0001, + "loss": 64.9378, + "step": 5500 + }, + { + "epoch": 8.873885221947901, + "grad_norm": 0.6252862811088562, + "learning_rate": 0.0001, + "loss": 64.1655, + "step": 5510 + }, + { + "epoch": 8.890008565526276, + "grad_norm": 0.8441852331161499, + "learning_rate": 0.0001, + "loss": 64.9795, + "step": 5520 + }, + { + "epoch": 8.906131909104651, + "grad_norm": 0.7075499892234802, + "learning_rate": 0.0001, + "loss": 65.5843, + "step": 5530 + }, + { + "epoch": 8.922255252683025, + "grad_norm": 0.7374144196510315, + "learning_rate": 0.0001, + "loss": 67.5064, + "step": 5540 + }, + { + "epoch": 8.9383785962614, + "grad_norm": 0.8073632121086121, + "learning_rate": 0.0001, + "loss": 67.0182, + "step": 5550 + }, + { + "epoch": 8.954501939839774, + "grad_norm": 0.7576195001602173, + "learning_rate": 0.0001, + "loss": 65.9139, + "step": 5560 + }, + { + "epoch": 8.97062528341815, + "grad_norm": 1.6479626893997192, + "learning_rate": 0.0001, + "loss": 66.3517, + "step": 5570 + }, + { + "epoch": 8.986748626996523, + "grad_norm": 0.846513569355011, + "learning_rate": 0.0001, + "loss": 64.2908, + "step": 5580 + }, + { + "epoch": 9.001612334357837, + "grad_norm": 0.9551037549972534, + "learning_rate": 0.0001, + "loss": 61.7312, + "step": 5590 + }, + { + "epoch": 9.017735677936212, + "grad_norm": 0.8250304460525513, + "learning_rate": 0.0001, + "loss": 66.3351, + "step": 5600 + }, + { + "epoch": 9.033859021514587, + "grad_norm": 0.7932960391044617, + "learning_rate": 0.0001, + "loss": 66.7753, + "step": 5610 + }, + { + "epoch": 9.04998236509296, + "grad_norm": 0.9839881658554077, + "learning_rate": 0.0001, + "loss": 64.5146, + "step": 5620 + }, + { + "epoch": 9.066105708671335, + "grad_norm": 0.7337850332260132, + "learning_rate": 0.0001, + "loss": 65.0544, + "step": 5630 + }, + { + "epoch": 9.08222905224971, + "grad_norm": 0.7496634721755981, + "learning_rate": 0.0001, + "loss": 64.1451, + "step": 5640 + }, + { + "epoch": 9.098352395828085, + "grad_norm": 0.9457297921180725, + "learning_rate": 0.0001, + "loss": 65.3036, + "step": 5650 + }, + { + "epoch": 9.11447573940646, + "grad_norm": 0.8315348625183105, + "learning_rate": 0.0001, + "loss": 66.0585, + "step": 5660 + }, + { + "epoch": 9.130599082984833, + "grad_norm": 0.9666195511817932, + "learning_rate": 0.0001, + "loss": 64.6802, + "step": 5670 + }, + { + "epoch": 9.146722426563208, + "grad_norm": 0.7416576147079468, + "learning_rate": 0.0001, + "loss": 65.1557, + "step": 5680 + }, + { + "epoch": 9.162845770141583, + "grad_norm": 0.7723799347877502, + "learning_rate": 0.0001, + "loss": 66.2871, + "step": 5690 + }, + { + "epoch": 9.178969113719958, + "grad_norm": 0.7461305856704712, + "learning_rate": 0.0001, + "loss": 65.5795, + "step": 5700 + }, + { + "epoch": 9.195092457298331, + "grad_norm": 0.6663411259651184, + "learning_rate": 0.0001, + "loss": 66.4094, + "step": 5710 + }, + { + "epoch": 9.211215800876706, + "grad_norm": 0.9340134859085083, + "learning_rate": 0.0001, + "loss": 63.5753, + "step": 5720 + }, + { + "epoch": 9.227339144455081, + "grad_norm": 0.7643963694572449, + "learning_rate": 0.0001, + "loss": 65.3359, + "step": 5730 + }, + { + "epoch": 9.243462488033456, + "grad_norm": 0.6931570768356323, + "learning_rate": 0.0001, + "loss": 65.0755, + "step": 5740 + }, + { + "epoch": 9.259585831611831, + "grad_norm": 0.9096988439559937, + "learning_rate": 0.0001, + "loss": 64.7983, + "step": 5750 + }, + { + "epoch": 9.275709175190205, + "grad_norm": 0.8360128402709961, + "learning_rate": 0.0001, + "loss": 64.3604, + "step": 5760 + }, + { + "epoch": 9.29183251876858, + "grad_norm": 0.9302591681480408, + "learning_rate": 0.0001, + "loss": 66.2754, + "step": 5770 + }, + { + "epoch": 9.307955862346954, + "grad_norm": 0.7000735998153687, + "learning_rate": 0.0001, + "loss": 65.2241, + "step": 5780 + }, + { + "epoch": 9.32407920592533, + "grad_norm": 0.7321175932884216, + "learning_rate": 0.0001, + "loss": 65.3558, + "step": 5790 + }, + { + "epoch": 9.340202549503703, + "grad_norm": 0.7948715090751648, + "learning_rate": 0.0001, + "loss": 64.2793, + "step": 5800 + }, + { + "epoch": 9.356325893082078, + "grad_norm": 0.7676129341125488, + "learning_rate": 0.0001, + "loss": 66.4778, + "step": 5810 + }, + { + "epoch": 9.372449236660453, + "grad_norm": 0.977236270904541, + "learning_rate": 0.0001, + "loss": 64.5587, + "step": 5820 + }, + { + "epoch": 9.388572580238828, + "grad_norm": 0.86808842420578, + "learning_rate": 0.0001, + "loss": 65.8026, + "step": 5830 + }, + { + "epoch": 9.4046959238172, + "grad_norm": 0.7455324530601501, + "learning_rate": 0.0001, + "loss": 65.066, + "step": 5840 + }, + { + "epoch": 9.420819267395576, + "grad_norm": 0.6784212589263916, + "learning_rate": 0.0001, + "loss": 65.4167, + "step": 5850 + }, + { + "epoch": 9.43694261097395, + "grad_norm": 1.033731460571289, + "learning_rate": 0.0001, + "loss": 66.4502, + "step": 5860 + }, + { + "epoch": 9.453065954552326, + "grad_norm": 0.7946107983589172, + "learning_rate": 0.0001, + "loss": 63.7567, + "step": 5870 + }, + { + "epoch": 9.4691892981307, + "grad_norm": 0.7826364636421204, + "learning_rate": 0.0001, + "loss": 65.4484, + "step": 5880 + }, + { + "epoch": 9.485312641709074, + "grad_norm": 0.7605016827583313, + "learning_rate": 0.0001, + "loss": 66.8459, + "step": 5890 + }, + { + "epoch": 9.501435985287449, + "grad_norm": 0.7383126020431519, + "learning_rate": 0.0001, + "loss": 64.6873, + "step": 5900 + }, + { + "epoch": 9.517559328865824, + "grad_norm": 0.8355686664581299, + "learning_rate": 0.0001, + "loss": 63.5145, + "step": 5910 + }, + { + "epoch": 9.533682672444199, + "grad_norm": 0.8827091455459595, + "learning_rate": 0.0001, + "loss": 67.2207, + "step": 5920 + }, + { + "epoch": 9.549806016022572, + "grad_norm": 0.6988142728805542, + "learning_rate": 0.0001, + "loss": 64.2289, + "step": 5930 + }, + { + "epoch": 9.565929359600947, + "grad_norm": 0.7609000205993652, + "learning_rate": 0.0001, + "loss": 63.9324, + "step": 5940 + }, + { + "epoch": 9.582052703179322, + "grad_norm": 0.8053853511810303, + "learning_rate": 0.0001, + "loss": 62.5564, + "step": 5950 + }, + { + "epoch": 9.598176046757697, + "grad_norm": 0.7684671878814697, + "learning_rate": 0.0001, + "loss": 66.2669, + "step": 5960 + }, + { + "epoch": 9.614299390336072, + "grad_norm": 0.8411065340042114, + "learning_rate": 0.0001, + "loss": 64.5087, + "step": 5970 + }, + { + "epoch": 9.630422733914445, + "grad_norm": 0.963498592376709, + "learning_rate": 0.0001, + "loss": 68.1736, + "step": 5980 + }, + { + "epoch": 9.64654607749282, + "grad_norm": 0.8715981841087341, + "learning_rate": 0.0001, + "loss": 64.1686, + "step": 5990 + }, + { + "epoch": 9.662669421071195, + "grad_norm": 0.8217800855636597, + "learning_rate": 0.0001, + "loss": 64.2211, + "step": 6000 + }, + { + "epoch": 9.67879276464957, + "grad_norm": 0.7824742197990417, + "learning_rate": 0.0001, + "loss": 65.9052, + "step": 6010 + }, + { + "epoch": 9.694916108227943, + "grad_norm": 0.7440009117126465, + "learning_rate": 0.0001, + "loss": 63.3749, + "step": 6020 + }, + { + "epoch": 9.711039451806318, + "grad_norm": 0.8759479522705078, + "learning_rate": 0.0001, + "loss": 63.4488, + "step": 6030 + }, + { + "epoch": 9.727162795384693, + "grad_norm": 3.662144184112549, + "learning_rate": 0.0001, + "loss": 62.8787, + "step": 6040 + }, + { + "epoch": 9.743286138963068, + "grad_norm": 1.007149577140808, + "learning_rate": 0.0001, + "loss": 66.0053, + "step": 6050 + }, + { + "epoch": 9.759409482541441, + "grad_norm": 0.8369242548942566, + "learning_rate": 0.0001, + "loss": 67.233, + "step": 6060 + }, + { + "epoch": 9.775532826119816, + "grad_norm": 0.8160434365272522, + "learning_rate": 0.0001, + "loss": 65.2089, + "step": 6070 + }, + { + "epoch": 9.791656169698191, + "grad_norm": 0.8482319712638855, + "learning_rate": 0.0001, + "loss": 65.0718, + "step": 6080 + }, + { + "epoch": 9.807779513276566, + "grad_norm": 0.8800129890441895, + "learning_rate": 0.0001, + "loss": 65.145, + "step": 6090 + }, + { + "epoch": 9.82390285685494, + "grad_norm": 0.8911405801773071, + "learning_rate": 0.0001, + "loss": 65.7952, + "step": 6100 + }, + { + "epoch": 9.840026200433314, + "grad_norm": 0.8108358979225159, + "learning_rate": 0.0001, + "loss": 65.1928, + "step": 6110 + }, + { + "epoch": 9.85614954401169, + "grad_norm": 0.8919743895530701, + "learning_rate": 0.0001, + "loss": 65.8129, + "step": 6120 + }, + { + "epoch": 9.872272887590064, + "grad_norm": 0.73152756690979, + "learning_rate": 0.0001, + "loss": 65.7141, + "step": 6130 + }, + { + "epoch": 9.88839623116844, + "grad_norm": 0.8583903312683105, + "learning_rate": 0.0001, + "loss": 64.8231, + "step": 6140 + }, + { + "epoch": 9.904519574746812, + "grad_norm": 0.798543393611908, + "learning_rate": 0.0001, + "loss": 64.5073, + "step": 6150 + }, + { + "epoch": 9.920642918325187, + "grad_norm": 0.8657358288764954, + "learning_rate": 0.0001, + "loss": 64.1207, + "step": 6160 + }, + { + "epoch": 9.936766261903562, + "grad_norm": 0.9324677586555481, + "learning_rate": 0.0001, + "loss": 64.7129, + "step": 6170 + }, + { + "epoch": 9.952889605481937, + "grad_norm": 0.7371154427528381, + "learning_rate": 0.0001, + "loss": 64.0251, + "step": 6180 + }, + { + "epoch": 9.96901294906031, + "grad_norm": 0.8273237347602844, + "learning_rate": 0.0001, + "loss": 65.8633, + "step": 6190 + }, + { + "epoch": 9.985136292638686, + "grad_norm": 0.7219365239143372, + "learning_rate": 0.0001, + "loss": 63.4029, + "step": 6200 + }, + { + "epoch": 10.0, + "grad_norm": 0.76242995262146, + "learning_rate": 0.0001, + "loss": 60.9994, + "step": 6210 + }, + { + "epoch": 10.016123343578375, + "grad_norm": 0.8677287697792053, + "learning_rate": 0.0001, + "loss": 65.9687, + "step": 6220 + }, + { + "epoch": 10.03224668715675, + "grad_norm": 0.7073521018028259, + "learning_rate": 0.0001, + "loss": 63.0083, + "step": 6230 + }, + { + "epoch": 10.048370030735123, + "grad_norm": 0.794535756111145, + "learning_rate": 0.0001, + "loss": 63.0616, + "step": 6240 + }, + { + "epoch": 10.064493374313498, + "grad_norm": 0.6974631547927856, + "learning_rate": 0.0001, + "loss": 64.6579, + "step": 6250 + }, + { + "epoch": 10.080616717891873, + "grad_norm": 0.8279150128364563, + "learning_rate": 0.0001, + "loss": 65.1069, + "step": 6260 + }, + { + "epoch": 10.096740061470248, + "grad_norm": 0.8682932257652283, + "learning_rate": 0.0001, + "loss": 64.5019, + "step": 6270 + }, + { + "epoch": 10.112863405048621, + "grad_norm": 1.0158190727233887, + "learning_rate": 0.0001, + "loss": 65.8854, + "step": 6280 + }, + { + "epoch": 10.128986748626996, + "grad_norm": 0.6911011934280396, + "learning_rate": 0.0001, + "loss": 62.4879, + "step": 6290 + }, + { + "epoch": 10.145110092205371, + "grad_norm": 0.7394022941589355, + "learning_rate": 0.0001, + "loss": 64.2604, + "step": 6300 + }, + { + "epoch": 10.161233435783746, + "grad_norm": 0.7941621541976929, + "learning_rate": 0.0001, + "loss": 64.5133, + "step": 6310 + }, + { + "epoch": 10.17735677936212, + "grad_norm": 1.0238994359970093, + "learning_rate": 0.0001, + "loss": 63.8594, + "step": 6320 + }, + { + "epoch": 10.193480122940494, + "grad_norm": 0.7332146763801575, + "learning_rate": 0.0001, + "loss": 66.5434, + "step": 6330 + }, + { + "epoch": 10.20960346651887, + "grad_norm": 0.9375078082084656, + "learning_rate": 0.0001, + "loss": 63.4793, + "step": 6340 + }, + { + "epoch": 10.225726810097244, + "grad_norm": 0.715931236743927, + "learning_rate": 0.0001, + "loss": 65.8812, + "step": 6350 + }, + { + "epoch": 10.24185015367562, + "grad_norm": 1.291536808013916, + "learning_rate": 0.0001, + "loss": 64.9125, + "step": 6360 + }, + { + "epoch": 10.257973497253992, + "grad_norm": 0.8942781686782837, + "learning_rate": 0.0001, + "loss": 67.1706, + "step": 6370 + }, + { + "epoch": 10.274096840832367, + "grad_norm": 0.8272124528884888, + "learning_rate": 0.0001, + "loss": 62.1721, + "step": 6380 + }, + { + "epoch": 10.290220184410742, + "grad_norm": 0.8191152215003967, + "learning_rate": 0.0001, + "loss": 64.8931, + "step": 6390 + }, + { + "epoch": 10.306343527989117, + "grad_norm": 1.7500241994857788, + "learning_rate": 0.0001, + "loss": 62.9693, + "step": 6400 + }, + { + "epoch": 10.32246687156749, + "grad_norm": 1.0159430503845215, + "learning_rate": 0.0001, + "loss": 63.0889, + "step": 6410 + }, + { + "epoch": 10.338590215145866, + "grad_norm": 0.7300348281860352, + "learning_rate": 0.0001, + "loss": 65.1742, + "step": 6420 + }, + { + "epoch": 10.35471355872424, + "grad_norm": 1.3132387399673462, + "learning_rate": 0.0001, + "loss": 64.8867, + "step": 6430 + }, + { + "epoch": 10.370836902302615, + "grad_norm": 0.7502451539039612, + "learning_rate": 0.0001, + "loss": 66.5094, + "step": 6440 + }, + { + "epoch": 10.38696024588099, + "grad_norm": 0.9408686757087708, + "learning_rate": 0.0001, + "loss": 65.1515, + "step": 6450 + }, + { + "epoch": 10.403083589459364, + "grad_norm": 0.769598126411438, + "learning_rate": 0.0001, + "loss": 65.1478, + "step": 6460 + }, + { + "epoch": 10.419206933037739, + "grad_norm": 0.9086162447929382, + "learning_rate": 0.0001, + "loss": 63.5263, + "step": 6470 + }, + { + "epoch": 10.435330276616114, + "grad_norm": 0.7899322509765625, + "learning_rate": 0.0001, + "loss": 63.3939, + "step": 6480 + }, + { + "epoch": 10.451453620194489, + "grad_norm": 0.9135434031486511, + "learning_rate": 0.0001, + "loss": 63.867, + "step": 6490 + }, + { + "epoch": 10.467576963772862, + "grad_norm": 0.9312737584114075, + "learning_rate": 0.0001, + "loss": 63.6939, + "step": 6500 + }, + { + "epoch": 10.483700307351237, + "grad_norm": 0.861764132976532, + "learning_rate": 0.0001, + "loss": 63.4523, + "step": 6510 + }, + { + "epoch": 10.499823650929612, + "grad_norm": 0.7056366801261902, + "learning_rate": 0.0001, + "loss": 63.2889, + "step": 6520 + }, + { + "epoch": 10.515946994507987, + "grad_norm": 0.7007710337638855, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 6530 + }, + { + "epoch": 10.53207033808636, + "grad_norm": 0.7091413736343384, + "learning_rate": 0.0001, + "loss": 63.4185, + "step": 6540 + }, + { + "epoch": 10.548193681664735, + "grad_norm": 0.7147234678268433, + "learning_rate": 0.0001, + "loss": 65.0726, + "step": 6550 + }, + { + "epoch": 10.56431702524311, + "grad_norm": 0.7954673767089844, + "learning_rate": 0.0001, + "loss": 65.0334, + "step": 6560 + }, + { + "epoch": 10.580440368821485, + "grad_norm": 0.725352942943573, + "learning_rate": 0.0001, + "loss": 62.6663, + "step": 6570 + }, + { + "epoch": 10.59656371239986, + "grad_norm": 0.9563508629798889, + "learning_rate": 0.0001, + "loss": 64.0705, + "step": 6580 + }, + { + "epoch": 10.612687055978233, + "grad_norm": 0.7778734564781189, + "learning_rate": 0.0001, + "loss": 63.7619, + "step": 6590 + }, + { + "epoch": 10.628810399556608, + "grad_norm": 0.8414369821548462, + "learning_rate": 0.0001, + "loss": 63.9238, + "step": 6600 + }, + { + "epoch": 10.644933743134983, + "grad_norm": 0.8070562481880188, + "learning_rate": 0.0001, + "loss": 63.9591, + "step": 6610 + }, + { + "epoch": 10.661057086713358, + "grad_norm": 0.6493734121322632, + "learning_rate": 0.0001, + "loss": 63.7237, + "step": 6620 + }, + { + "epoch": 10.677180430291731, + "grad_norm": 0.8263372778892517, + "learning_rate": 0.0001, + "loss": 64.2262, + "step": 6630 + }, + { + "epoch": 10.693303773870106, + "grad_norm": 1.0090227127075195, + "learning_rate": 0.0001, + "loss": 63.5469, + "step": 6640 + }, + { + "epoch": 10.709427117448481, + "grad_norm": 0.998319685459137, + "learning_rate": 0.0001, + "loss": 62.9113, + "step": 6650 + }, + { + "epoch": 10.725550461026856, + "grad_norm": 1.006489634513855, + "learning_rate": 0.0001, + "loss": 66.1458, + "step": 6660 + }, + { + "epoch": 10.741673804605231, + "grad_norm": 0.9575757384300232, + "learning_rate": 0.0001, + "loss": 63.0913, + "step": 6670 + }, + { + "epoch": 10.757797148183604, + "grad_norm": 0.6832238435745239, + "learning_rate": 0.0001, + "loss": 66.7501, + "step": 6680 + }, + { + "epoch": 10.773920491761979, + "grad_norm": 0.8075717687606812, + "learning_rate": 0.0001, + "loss": 64.2839, + "step": 6690 + }, + { + "epoch": 10.790043835340354, + "grad_norm": 0.9402624368667603, + "learning_rate": 0.0001, + "loss": 64.9296, + "step": 6700 + }, + { + "epoch": 10.806167178918729, + "grad_norm": 0.7681158185005188, + "learning_rate": 0.0001, + "loss": 63.5686, + "step": 6710 + }, + { + "epoch": 10.822290522497102, + "grad_norm": 0.8451285362243652, + "learning_rate": 0.0001, + "loss": 64.5404, + "step": 6720 + }, + { + "epoch": 10.838413866075477, + "grad_norm": 0.7278769016265869, + "learning_rate": 0.0001, + "loss": 64.1341, + "step": 6730 + }, + { + "epoch": 10.854537209653852, + "grad_norm": 0.8524075746536255, + "learning_rate": 0.0001, + "loss": 64.0483, + "step": 6740 + }, + { + "epoch": 10.870660553232227, + "grad_norm": 0.703598141670227, + "learning_rate": 0.0001, + "loss": 64.0072, + "step": 6750 + }, + { + "epoch": 10.8867838968106, + "grad_norm": 0.7740384936332703, + "learning_rate": 0.0001, + "loss": 63.4171, + "step": 6760 + }, + { + "epoch": 10.902907240388975, + "grad_norm": 0.8302356004714966, + "learning_rate": 0.0001, + "loss": 65.5159, + "step": 6770 + }, + { + "epoch": 10.91903058396735, + "grad_norm": 1.0289733409881592, + "learning_rate": 0.0001, + "loss": 64.9785, + "step": 6780 + }, + { + "epoch": 10.935153927545725, + "grad_norm": 0.6721987128257751, + "learning_rate": 0.0001, + "loss": 63.965, + "step": 6790 + }, + { + "epoch": 10.951277271124098, + "grad_norm": 0.7614948749542236, + "learning_rate": 0.0001, + "loss": 64.1375, + "step": 6800 + }, + { + "epoch": 10.967400614702473, + "grad_norm": 0.7342216372489929, + "learning_rate": 0.0001, + "loss": 64.4342, + "step": 6810 + }, + { + "epoch": 10.983523958280848, + "grad_norm": 0.8808079957962036, + "learning_rate": 0.0001, + "loss": 65.2864, + "step": 6820 + }, + { + "epoch": 10.999647301859223, + "grad_norm": 0.818477988243103, + "learning_rate": 0.0001, + "loss": 63.9931, + "step": 6830 + }, + { + "epoch": 11.014511009220538, + "grad_norm": 0.7867474555969238, + "learning_rate": 0.0001, + "loss": 59.2426, + "step": 6840 + }, + { + "epoch": 11.030634352798911, + "grad_norm": 0.6163678765296936, + "learning_rate": 0.0001, + "loss": 65.0149, + "step": 6850 + }, + { + "epoch": 11.046757696377286, + "grad_norm": 0.8334609866142273, + "learning_rate": 0.0001, + "loss": 63.5017, + "step": 6860 + }, + { + "epoch": 11.062881039955661, + "grad_norm": 0.7335264086723328, + "learning_rate": 0.0001, + "loss": 64.6307, + "step": 6870 + }, + { + "epoch": 11.079004383534036, + "grad_norm": 0.7723846435546875, + "learning_rate": 0.0001, + "loss": 64.316, + "step": 6880 + }, + { + "epoch": 11.09512772711241, + "grad_norm": 0.7972145080566406, + "learning_rate": 0.0001, + "loss": 63.1469, + "step": 6890 + }, + { + "epoch": 11.111251070690784, + "grad_norm": 0.7696133852005005, + "learning_rate": 0.0001, + "loss": 64.0338, + "step": 6900 + }, + { + "epoch": 11.127374414269159, + "grad_norm": 0.8972133994102478, + "learning_rate": 0.0001, + "loss": 65.5727, + "step": 6910 + }, + { + "epoch": 11.143497757847534, + "grad_norm": 0.6776295900344849, + "learning_rate": 0.0001, + "loss": 66.176, + "step": 6920 + }, + { + "epoch": 11.159621101425909, + "grad_norm": 0.7345874905586243, + "learning_rate": 0.0001, + "loss": 64.0949, + "step": 6930 + }, + { + "epoch": 11.175744445004282, + "grad_norm": 0.7988374829292297, + "learning_rate": 0.0001, + "loss": 64.7721, + "step": 6940 + }, + { + "epoch": 11.191867788582657, + "grad_norm": 0.7552387714385986, + "learning_rate": 0.0001, + "loss": 63.1053, + "step": 6950 + }, + { + "epoch": 11.207991132161032, + "grad_norm": 0.6388214230537415, + "learning_rate": 0.0001, + "loss": 63.7871, + "step": 6960 + }, + { + "epoch": 11.224114475739407, + "grad_norm": 0.6837960481643677, + "learning_rate": 0.0001, + "loss": 63.906, + "step": 6970 + }, + { + "epoch": 11.24023781931778, + "grad_norm": 0.7050679326057434, + "learning_rate": 0.0001, + "loss": 64.2803, + "step": 6980 + }, + { + "epoch": 11.256361162896155, + "grad_norm": 0.8619478940963745, + "learning_rate": 0.0001, + "loss": 62.8786, + "step": 6990 + }, + { + "epoch": 11.27248450647453, + "grad_norm": 0.9759813547134399, + "learning_rate": 0.0001, + "loss": 65.2299, + "step": 7000 + }, + { + "epoch": 11.288607850052905, + "grad_norm": 0.7584514021873474, + "learning_rate": 0.0001, + "loss": 61.4178, + "step": 7010 + }, + { + "epoch": 11.304731193631278, + "grad_norm": 0.8467874526977539, + "learning_rate": 0.0001, + "loss": 65.6229, + "step": 7020 + }, + { + "epoch": 11.320854537209653, + "grad_norm": 0.7121654152870178, + "learning_rate": 0.0001, + "loss": 63.9061, + "step": 7030 + }, + { + "epoch": 11.336977880788028, + "grad_norm": 0.8885557055473328, + "learning_rate": 0.0001, + "loss": 65.1436, + "step": 7040 + }, + { + "epoch": 11.353101224366403, + "grad_norm": 0.6320177912712097, + "learning_rate": 0.0001, + "loss": 62.8535, + "step": 7050 + }, + { + "epoch": 11.369224567944778, + "grad_norm": 0.9287629723548889, + "learning_rate": 0.0001, + "loss": 62.7407, + "step": 7060 + }, + { + "epoch": 11.385347911523152, + "grad_norm": 0.7705608606338501, + "learning_rate": 0.0001, + "loss": 63.469, + "step": 7070 + }, + { + "epoch": 11.401471255101526, + "grad_norm": 0.6687246561050415, + "learning_rate": 0.0001, + "loss": 62.722, + "step": 7080 + }, + { + "epoch": 11.417594598679901, + "grad_norm": 0.7774648666381836, + "learning_rate": 0.0001, + "loss": 63.1145, + "step": 7090 + }, + { + "epoch": 11.433717942258276, + "grad_norm": 0.7808363437652588, + "learning_rate": 0.0001, + "loss": 63.9342, + "step": 7100 + }, + { + "epoch": 11.44984128583665, + "grad_norm": 0.8033562302589417, + "learning_rate": 0.0001, + "loss": 65.7876, + "step": 7110 + }, + { + "epoch": 11.465964629415025, + "grad_norm": 0.8143486976623535, + "learning_rate": 0.0001, + "loss": 64.8782, + "step": 7120 + }, + { + "epoch": 11.4820879729934, + "grad_norm": 0.7855561971664429, + "learning_rate": 0.0001, + "loss": 63.4944, + "step": 7130 + }, + { + "epoch": 11.498211316571775, + "grad_norm": 0.7525351047515869, + "learning_rate": 0.0001, + "loss": 62.4088, + "step": 7140 + }, + { + "epoch": 11.514334660150148, + "grad_norm": 0.7193494439125061, + "learning_rate": 0.0001, + "loss": 64.2386, + "step": 7150 + }, + { + "epoch": 11.530458003728523, + "grad_norm": 0.892281174659729, + "learning_rate": 0.0001, + "loss": 63.8075, + "step": 7160 + }, + { + "epoch": 11.546581347306898, + "grad_norm": 0.720287561416626, + "learning_rate": 0.0001, + "loss": 63.3259, + "step": 7170 + }, + { + "epoch": 11.562704690885273, + "grad_norm": 1.4404689073562622, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 7180 + }, + { + "epoch": 11.578828034463648, + "grad_norm": 0.9082726240158081, + "learning_rate": 0.0001, + "loss": 64.6877, + "step": 7190 + }, + { + "epoch": 11.59495137804202, + "grad_norm": 0.7137670516967773, + "learning_rate": 0.0001, + "loss": 63.4832, + "step": 7200 + }, + { + "epoch": 11.611074721620396, + "grad_norm": 0.6848503947257996, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 7210 + }, + { + "epoch": 11.62719806519877, + "grad_norm": 0.7483687996864319, + "learning_rate": 0.0001, + "loss": 63.294, + "step": 7220 + }, + { + "epoch": 11.643321408777146, + "grad_norm": 0.935321569442749, + "learning_rate": 0.0001, + "loss": 62.8669, + "step": 7230 + }, + { + "epoch": 11.659444752355519, + "grad_norm": 0.8359326124191284, + "learning_rate": 0.0001, + "loss": 64.4482, + "step": 7240 + }, + { + "epoch": 11.675568095933894, + "grad_norm": 0.7838515043258667, + "learning_rate": 0.0001, + "loss": 64.1975, + "step": 7250 + }, + { + "epoch": 11.691691439512269, + "grad_norm": 0.9128443598747253, + "learning_rate": 0.0001, + "loss": 63.1822, + "step": 7260 + }, + { + "epoch": 11.707814783090644, + "grad_norm": 1.0545419454574585, + "learning_rate": 0.0001, + "loss": 63.3112, + "step": 7270 + }, + { + "epoch": 11.723938126669019, + "grad_norm": 0.8572486639022827, + "learning_rate": 0.0001, + "loss": 64.9605, + "step": 7280 + }, + { + "epoch": 11.740061470247392, + "grad_norm": 1.0894116163253784, + "learning_rate": 0.0001, + "loss": 64.3671, + "step": 7290 + }, + { + "epoch": 11.756184813825767, + "grad_norm": 0.7500355839729309, + "learning_rate": 0.0001, + "loss": 62.6381, + "step": 7300 + }, + { + "epoch": 11.772308157404142, + "grad_norm": 0.7315185070037842, + "learning_rate": 0.0001, + "loss": 66.1438, + "step": 7310 + }, + { + "epoch": 11.788431500982517, + "grad_norm": 0.8123629093170166, + "learning_rate": 0.0001, + "loss": 62.6012, + "step": 7320 + }, + { + "epoch": 11.80455484456089, + "grad_norm": 0.8072763085365295, + "learning_rate": 0.0001, + "loss": 63.4424, + "step": 7330 + }, + { + "epoch": 11.820678188139265, + "grad_norm": 1.1077874898910522, + "learning_rate": 0.0001, + "loss": 63.2812, + "step": 7340 + }, + { + "epoch": 11.83680153171764, + "grad_norm": 0.7116726636886597, + "learning_rate": 0.0001, + "loss": 63.6077, + "step": 7350 + }, + { + "epoch": 11.852924875296015, + "grad_norm": 0.8817790746688843, + "learning_rate": 0.0001, + "loss": 63.1938, + "step": 7360 + }, + { + "epoch": 11.86904821887439, + "grad_norm": 1.021730661392212, + "learning_rate": 0.0001, + "loss": 65.2215, + "step": 7370 + }, + { + "epoch": 11.885171562452763, + "grad_norm": 0.9871250987052917, + "learning_rate": 0.0001, + "loss": 64.6468, + "step": 7380 + }, + { + "epoch": 11.901294906031138, + "grad_norm": 1.3591430187225342, + "learning_rate": 0.0001, + "loss": 61.5546, + "step": 7390 + }, + { + "epoch": 11.917418249609513, + "grad_norm": 0.8114227056503296, + "learning_rate": 0.0001, + "loss": 61.8215, + "step": 7400 + }, + { + "epoch": 11.933541593187888, + "grad_norm": 0.7151288986206055, + "learning_rate": 0.0001, + "loss": 61.4856, + "step": 7410 + }, + { + "epoch": 11.949664936766261, + "grad_norm": 0.7494287490844727, + "learning_rate": 0.0001, + "loss": 63.6451, + "step": 7420 + }, + { + "epoch": 11.965788280344636, + "grad_norm": 0.7377745509147644, + "learning_rate": 0.0001, + "loss": 63.002, + "step": 7430 + }, + { + "epoch": 11.981911623923011, + "grad_norm": 0.6534659266471863, + "learning_rate": 0.0001, + "loss": 63.6404, + "step": 7440 + }, + { + "epoch": 11.998034967501386, + "grad_norm": 0.7548736333847046, + "learning_rate": 0.0001, + "loss": 64.1986, + "step": 7450 + }, + { + "epoch": 12.012898674862699, + "grad_norm": 0.7227553129196167, + "learning_rate": 0.0001, + "loss": 56.8327, + "step": 7460 + }, + { + "epoch": 12.029022018441074, + "grad_norm": 0.6348695158958435, + "learning_rate": 0.0001, + "loss": 62.4472, + "step": 7470 + }, + { + "epoch": 12.045145362019449, + "grad_norm": 0.8249921202659607, + "learning_rate": 0.0001, + "loss": 65.1877, + "step": 7480 + }, + { + "epoch": 12.061268705597824, + "grad_norm": 0.7152777314186096, + "learning_rate": 0.0001, + "loss": 64.2609, + "step": 7490 + }, + { + "epoch": 12.077392049176197, + "grad_norm": 0.7399184703826904, + "learning_rate": 0.0001, + "loss": 63.3997, + "step": 7500 + }, + { + "epoch": 12.093515392754572, + "grad_norm": 0.6972354650497437, + "learning_rate": 0.0001, + "loss": 61.3573, + "step": 7510 + }, + { + "epoch": 12.109638736332947, + "grad_norm": 0.7666720151901245, + "learning_rate": 0.0001, + "loss": 63.8438, + "step": 7520 + }, + { + "epoch": 12.125762079911322, + "grad_norm": 0.7309302687644958, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 7530 + }, + { + "epoch": 12.141885423489697, + "grad_norm": 1.3805177211761475, + "learning_rate": 0.0001, + "loss": 64.68, + "step": 7540 + }, + { + "epoch": 12.15800876706807, + "grad_norm": 1.0315922498703003, + "learning_rate": 0.0001, + "loss": 63.6372, + "step": 7550 + }, + { + "epoch": 12.174132110646445, + "grad_norm": 1.2012155055999756, + "learning_rate": 0.0001, + "loss": 62.5765, + "step": 7560 + }, + { + "epoch": 12.19025545422482, + "grad_norm": 0.7769846320152283, + "learning_rate": 0.0001, + "loss": 63.9737, + "step": 7570 + }, + { + "epoch": 12.206378797803195, + "grad_norm": 0.6587379574775696, + "learning_rate": 0.0001, + "loss": 63.382, + "step": 7580 + }, + { + "epoch": 12.222502141381568, + "grad_norm": 0.8309435248374939, + "learning_rate": 0.0001, + "loss": 64.4027, + "step": 7590 + }, + { + "epoch": 12.238625484959943, + "grad_norm": 0.8221393823623657, + "learning_rate": 0.0001, + "loss": 62.8561, + "step": 7600 + }, + { + "epoch": 12.254748828538318, + "grad_norm": 0.8629311323165894, + "learning_rate": 0.0001, + "loss": 64.3673, + "step": 7610 + }, + { + "epoch": 12.270872172116693, + "grad_norm": 0.698113739490509, + "learning_rate": 0.0001, + "loss": 63.6153, + "step": 7620 + }, + { + "epoch": 12.286995515695068, + "grad_norm": 0.72906094789505, + "learning_rate": 0.0001, + "loss": 64.8439, + "step": 7630 + }, + { + "epoch": 12.303118859273441, + "grad_norm": 0.7402384877204895, + "learning_rate": 0.0001, + "loss": 64.2103, + "step": 7640 + }, + { + "epoch": 12.319242202851816, + "grad_norm": 0.9972009062767029, + "learning_rate": 0.0001, + "loss": 63.803, + "step": 7650 + }, + { + "epoch": 12.335365546430191, + "grad_norm": 0.7852274775505066, + "learning_rate": 0.0001, + "loss": 64.8144, + "step": 7660 + }, + { + "epoch": 12.351488890008566, + "grad_norm": 0.7433444857597351, + "learning_rate": 0.0001, + "loss": 63.0123, + "step": 7670 + }, + { + "epoch": 12.36761223358694, + "grad_norm": 0.8548759818077087, + "learning_rate": 0.0001, + "loss": 63.2215, + "step": 7680 + }, + { + "epoch": 12.383735577165314, + "grad_norm": 0.6759565472602844, + "learning_rate": 0.0001, + "loss": 62.1246, + "step": 7690 + }, + { + "epoch": 12.39985892074369, + "grad_norm": 0.9079536199569702, + "learning_rate": 0.0001, + "loss": 64.1692, + "step": 7700 + }, + { + "epoch": 12.415982264322064, + "grad_norm": 0.6890071034431458, + "learning_rate": 0.0001, + "loss": 62.5602, + "step": 7710 + }, + { + "epoch": 12.432105607900438, + "grad_norm": 0.8096348643302917, + "learning_rate": 0.0001, + "loss": 61.0103, + "step": 7720 + }, + { + "epoch": 12.448228951478812, + "grad_norm": 0.7359873056411743, + "learning_rate": 0.0001, + "loss": 62.1914, + "step": 7730 + }, + { + "epoch": 12.464352295057187, + "grad_norm": 1.0701091289520264, + "learning_rate": 0.0001, + "loss": 61.7295, + "step": 7740 + }, + { + "epoch": 12.480475638635562, + "grad_norm": 0.791928768157959, + "learning_rate": 0.0001, + "loss": 62.6812, + "step": 7750 + }, + { + "epoch": 12.496598982213937, + "grad_norm": 0.7719446420669556, + "learning_rate": 0.0001, + "loss": 61.9114, + "step": 7760 + }, + { + "epoch": 12.51272232579231, + "grad_norm": 0.8720338344573975, + "learning_rate": 0.0001, + "loss": 64.8107, + "step": 7770 + }, + { + "epoch": 12.528845669370686, + "grad_norm": 0.9999920129776001, + "learning_rate": 0.0001, + "loss": 61.198, + "step": 7780 + }, + { + "epoch": 12.54496901294906, + "grad_norm": 0.710754930973053, + "learning_rate": 0.0001, + "loss": 64.1462, + "step": 7790 + }, + { + "epoch": 12.561092356527435, + "grad_norm": 0.7401363849639893, + "learning_rate": 0.0001, + "loss": 61.8988, + "step": 7800 + }, + { + "epoch": 12.577215700105809, + "grad_norm": 0.9183729290962219, + "learning_rate": 0.0001, + "loss": 62.6392, + "step": 7810 + }, + { + "epoch": 12.593339043684184, + "grad_norm": 0.7341399788856506, + "learning_rate": 0.0001, + "loss": 62.7068, + "step": 7820 + }, + { + "epoch": 12.609462387262559, + "grad_norm": 0.8066722750663757, + "learning_rate": 0.0001, + "loss": 64.0625, + "step": 7830 + }, + { + "epoch": 12.625585730840934, + "grad_norm": 0.6088835597038269, + "learning_rate": 0.0001, + "loss": 64.3208, + "step": 7840 + }, + { + "epoch": 12.641709074419307, + "grad_norm": 0.6969302892684937, + "learning_rate": 0.0001, + "loss": 62.5817, + "step": 7850 + }, + { + "epoch": 12.657832417997682, + "grad_norm": 0.82635098695755, + "learning_rate": 0.0001, + "loss": 62.0061, + "step": 7860 + }, + { + "epoch": 12.673955761576057, + "grad_norm": 0.7016922235488892, + "learning_rate": 0.0001, + "loss": 62.5166, + "step": 7870 + }, + { + "epoch": 12.690079105154432, + "grad_norm": 0.8035777807235718, + "learning_rate": 0.0001, + "loss": 63.6543, + "step": 7880 + }, + { + "epoch": 12.706202448732807, + "grad_norm": 0.8573809862136841, + "learning_rate": 0.0001, + "loss": 62.5258, + "step": 7890 + }, + { + "epoch": 12.72232579231118, + "grad_norm": 0.9497472643852234, + "learning_rate": 0.0001, + "loss": 63.1133, + "step": 7900 + }, + { + "epoch": 12.738449135889555, + "grad_norm": 0.7494457960128784, + "learning_rate": 0.0001, + "loss": 63.8157, + "step": 7910 + }, + { + "epoch": 12.75457247946793, + "grad_norm": 0.7657570242881775, + "learning_rate": 0.0001, + "loss": 62.6255, + "step": 7920 + }, + { + "epoch": 12.770695823046305, + "grad_norm": 0.7848380208015442, + "learning_rate": 0.0001, + "loss": 62.9095, + "step": 7930 + }, + { + "epoch": 12.786819166624678, + "grad_norm": 0.8386877775192261, + "learning_rate": 0.0001, + "loss": 63.2254, + "step": 7940 + }, + { + "epoch": 12.802942510203053, + "grad_norm": 0.6774776577949524, + "learning_rate": 0.0001, + "loss": 64.3929, + "step": 7950 + }, + { + "epoch": 12.819065853781428, + "grad_norm": 0.798547625541687, + "learning_rate": 0.0001, + "loss": 62.9884, + "step": 7960 + }, + { + "epoch": 12.835189197359803, + "grad_norm": 0.9340888261795044, + "learning_rate": 0.0001, + "loss": 62.6197, + "step": 7970 + }, + { + "epoch": 12.851312540938178, + "grad_norm": 0.8211432695388794, + "learning_rate": 0.0001, + "loss": 62.7756, + "step": 7980 + }, + { + "epoch": 12.867435884516551, + "grad_norm": 0.7612001299858093, + "learning_rate": 0.0001, + "loss": 62.0792, + "step": 7990 + }, + { + "epoch": 12.883559228094926, + "grad_norm": 0.7743386626243591, + "learning_rate": 0.0001, + "loss": 63.9203, + "step": 8000 + }, + { + "epoch": 12.899682571673301, + "grad_norm": 0.7419672608375549, + "learning_rate": 0.0001, + "loss": 62.1705, + "step": 8010 + }, + { + "epoch": 12.915805915251676, + "grad_norm": 0.6556256413459778, + "learning_rate": 0.0001, + "loss": 64.5863, + "step": 8020 + }, + { + "epoch": 12.93192925883005, + "grad_norm": 1.136175274848938, + "learning_rate": 0.0001, + "loss": 63.2619, + "step": 8030 + }, + { + "epoch": 12.948052602408424, + "grad_norm": 0.8631499409675598, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8040 + }, + { + "epoch": 12.9641759459868, + "grad_norm": 0.8386172652244568, + "learning_rate": 0.0001, + "loss": 63.0598, + "step": 8050 + }, + { + "epoch": 12.980299289565174, + "grad_norm": 0.8078306913375854, + "learning_rate": 0.0001, + "loss": 62.6877, + "step": 8060 + }, + { + "epoch": 12.996422633143549, + "grad_norm": 0.8486189842224121, + "learning_rate": 0.0001, + "loss": 62.8643, + "step": 8070 + }, + { + "epoch": 13.011286340504862, + "grad_norm": 0.8717893958091736, + "learning_rate": 0.0001, + "loss": 57.6947, + "step": 8080 + }, + { + "epoch": 13.027409684083237, + "grad_norm": 1.2023789882659912, + "learning_rate": 0.0001, + "loss": 60.4728, + "step": 8090 + }, + { + "epoch": 13.043533027661612, + "grad_norm": 0.6879557967185974, + "learning_rate": 0.0001, + "loss": 61.7643, + "step": 8100 + }, + { + "epoch": 13.059656371239987, + "grad_norm": 1.0918916463851929, + "learning_rate": 0.0001, + "loss": 65.3682, + "step": 8110 + }, + { + "epoch": 13.07577971481836, + "grad_norm": 0.85753333568573, + "learning_rate": 0.0001, + "loss": 63.0815, + "step": 8120 + }, + { + "epoch": 13.091903058396735, + "grad_norm": 0.6568583846092224, + "learning_rate": 0.0001, + "loss": 64.7167, + "step": 8130 + }, + { + "epoch": 13.10802640197511, + "grad_norm": 0.6178144812583923, + "learning_rate": 0.0001, + "loss": 62.4294, + "step": 8140 + }, + { + "epoch": 13.124149745553485, + "grad_norm": 0.7930784821510315, + "learning_rate": 0.0001, + "loss": 64.542, + "step": 8150 + }, + { + "epoch": 13.140273089131858, + "grad_norm": 0.7675037980079651, + "learning_rate": 0.0001, + "loss": 62.2645, + "step": 8160 + }, + { + "epoch": 13.156396432710233, + "grad_norm": 0.786997377872467, + "learning_rate": 0.0001, + "loss": 64.1495, + "step": 8170 + }, + { + "epoch": 13.172519776288608, + "grad_norm": 0.7752560377120972, + "learning_rate": 0.0001, + "loss": 61.6483, + "step": 8180 + }, + { + "epoch": 13.188643119866983, + "grad_norm": 0.8455203771591187, + "learning_rate": 0.0001, + "loss": 63.3551, + "step": 8190 + }, + { + "epoch": 13.204766463445356, + "grad_norm": 0.7223097681999207, + "learning_rate": 0.0001, + "loss": 63.1473, + "step": 8200 + }, + { + "epoch": 13.220889807023731, + "grad_norm": 0.7688442468643188, + "learning_rate": 0.0001, + "loss": 62.6723, + "step": 8210 + }, + { + "epoch": 13.237013150602106, + "grad_norm": 0.7730395793914795, + "learning_rate": 0.0001, + "loss": 64.3424, + "step": 8220 + }, + { + "epoch": 13.253136494180481, + "grad_norm": 0.8479727506637573, + "learning_rate": 0.0001, + "loss": 61.4336, + "step": 8230 + }, + { + "epoch": 13.269259837758856, + "grad_norm": 0.9070560932159424, + "learning_rate": 0.0001, + "loss": 63.1221, + "step": 8240 + }, + { + "epoch": 13.28538318133723, + "grad_norm": 0.8924600481987, + "learning_rate": 0.0001, + "loss": 63.9172, + "step": 8250 + }, + { + "epoch": 13.301506524915604, + "grad_norm": 1.0688443183898926, + "learning_rate": 0.0001, + "loss": 63.7944, + "step": 8260 + }, + { + "epoch": 13.31762986849398, + "grad_norm": 0.6866253018379211, + "learning_rate": 0.0001, + "loss": 65.5847, + "step": 8270 + }, + { + "epoch": 13.333753212072354, + "grad_norm": 0.711171567440033, + "learning_rate": 0.0001, + "loss": 62.698, + "step": 8280 + }, + { + "epoch": 13.349876555650727, + "grad_norm": 0.8128448724746704, + "learning_rate": 0.0001, + "loss": 61.5992, + "step": 8290 + }, + { + "epoch": 13.365999899229102, + "grad_norm": 0.7483755350112915, + "learning_rate": 0.0001, + "loss": 63.782, + "step": 8300 + }, + { + "epoch": 13.382123242807477, + "grad_norm": 0.7872599363327026, + "learning_rate": 0.0001, + "loss": 65.4922, + "step": 8310 + }, + { + "epoch": 13.398246586385852, + "grad_norm": 0.6861543655395508, + "learning_rate": 0.0001, + "loss": 62.6946, + "step": 8320 + }, + { + "epoch": 13.414369929964227, + "grad_norm": 0.7192599773406982, + "learning_rate": 0.0001, + "loss": 62.894, + "step": 8330 + }, + { + "epoch": 13.4304932735426, + "grad_norm": 0.7665286064147949, + "learning_rate": 0.0001, + "loss": 62.6253, + "step": 8340 + }, + { + "epoch": 13.446616617120975, + "grad_norm": 0.7909309267997742, + "learning_rate": 0.0001, + "loss": 62.5355, + "step": 8350 + }, + { + "epoch": 13.46273996069935, + "grad_norm": 0.7339541912078857, + "learning_rate": 0.0001, + "loss": 62.8514, + "step": 8360 + }, + { + "epoch": 13.478863304277725, + "grad_norm": 0.9893004298210144, + "learning_rate": 0.0001, + "loss": 63.7751, + "step": 8370 + }, + { + "epoch": 13.494986647856098, + "grad_norm": 0.9109547734260559, + "learning_rate": 0.0001, + "loss": 62.6697, + "step": 8380 + }, + { + "epoch": 13.511109991434473, + "grad_norm": 0.6807034015655518, + "learning_rate": 0.0001, + "loss": 62.1418, + "step": 8390 + }, + { + "epoch": 13.527233335012848, + "grad_norm": 0.915533185005188, + "learning_rate": 0.0001, + "loss": 61.7645, + "step": 8400 + }, + { + "epoch": 13.543356678591223, + "grad_norm": 0.8662852048873901, + "learning_rate": 0.0001, + "loss": 62.5546, + "step": 8410 + }, + { + "epoch": 13.559480022169598, + "grad_norm": 0.8678365349769592, + "learning_rate": 0.0001, + "loss": 63.8795, + "step": 8420 + }, + { + "epoch": 13.575603365747972, + "grad_norm": 0.6808590888977051, + "learning_rate": 0.0001, + "loss": 64.158, + "step": 8430 + }, + { + "epoch": 13.591726709326347, + "grad_norm": 0.7896480560302734, + "learning_rate": 0.0001, + "loss": 62.1052, + "step": 8440 + }, + { + "epoch": 13.607850052904721, + "grad_norm": 0.8973979949951172, + "learning_rate": 0.0001, + "loss": 64.478, + "step": 8450 + }, + { + "epoch": 13.623973396483096, + "grad_norm": 0.8228474259376526, + "learning_rate": 0.0001, + "loss": 61.2416, + "step": 8460 + }, + { + "epoch": 13.64009674006147, + "grad_norm": 0.9414492845535278, + "learning_rate": 0.0001, + "loss": 62.816, + "step": 8470 + }, + { + "epoch": 13.656220083639845, + "grad_norm": 0.9369436502456665, + "learning_rate": 0.0001, + "loss": 63.4951, + "step": 8480 + }, + { + "epoch": 13.67234342721822, + "grad_norm": 0.7152880430221558, + "learning_rate": 0.0001, + "loss": 62.4737, + "step": 8490 + }, + { + "epoch": 13.688466770796595, + "grad_norm": 0.7364594340324402, + "learning_rate": 0.0001, + "loss": 63.8688, + "step": 8500 + }, + { + "epoch": 13.704590114374968, + "grad_norm": 0.8499143123626709, + "learning_rate": 0.0001, + "loss": 64.5654, + "step": 8510 + }, + { + "epoch": 13.720713457953343, + "grad_norm": 0.7699156999588013, + "learning_rate": 0.0001, + "loss": 63.3726, + "step": 8520 + }, + { + "epoch": 13.736836801531718, + "grad_norm": 0.9146804809570312, + "learning_rate": 0.0001, + "loss": 63.6565, + "step": 8530 + }, + { + "epoch": 13.752960145110093, + "grad_norm": 0.773318350315094, + "learning_rate": 0.0001, + "loss": 65.0719, + "step": 8540 + }, + { + "epoch": 13.769083488688466, + "grad_norm": 0.6158450245857239, + "learning_rate": 0.0001, + "loss": 60.7291, + "step": 8550 + }, + { + "epoch": 13.78520683226684, + "grad_norm": 0.8053523302078247, + "learning_rate": 0.0001, + "loss": 62.5622, + "step": 8560 + }, + { + "epoch": 13.801330175845216, + "grad_norm": 0.8983457684516907, + "learning_rate": 0.0001, + "loss": 64.2921, + "step": 8570 + }, + { + "epoch": 13.81745351942359, + "grad_norm": 0.8214695453643799, + "learning_rate": 0.0001, + "loss": 62.2649, + "step": 8580 + }, + { + "epoch": 13.833576863001966, + "grad_norm": 0.8079172372817993, + "learning_rate": 0.0001, + "loss": 64.0828, + "step": 8590 + }, + { + "epoch": 13.849700206580339, + "grad_norm": 1.126568078994751, + "learning_rate": 0.0001, + "loss": 62.3927, + "step": 8600 + }, + { + "epoch": 13.865823550158714, + "grad_norm": 0.7597793936729431, + "learning_rate": 0.0001, + "loss": 62.5922, + "step": 8610 + }, + { + "epoch": 13.881946893737089, + "grad_norm": 0.671531617641449, + "learning_rate": 0.0001, + "loss": 61.9885, + "step": 8620 + }, + { + "epoch": 13.898070237315464, + "grad_norm": 0.823423445224762, + "learning_rate": 0.0001, + "loss": 62.0632, + "step": 8630 + }, + { + "epoch": 13.914193580893837, + "grad_norm": 0.7694658637046814, + "learning_rate": 0.0001, + "loss": 64.5485, + "step": 8640 + }, + { + "epoch": 13.930316924472212, + "grad_norm": 0.7394612431526184, + "learning_rate": 0.0001, + "loss": 61.9626, + "step": 8650 + }, + { + "epoch": 13.946440268050587, + "grad_norm": 0.724054217338562, + "learning_rate": 0.0001, + "loss": 62.3761, + "step": 8660 + }, + { + "epoch": 13.962563611628962, + "grad_norm": 0.7160128951072693, + "learning_rate": 0.0001, + "loss": 60.5142, + "step": 8670 + }, + { + "epoch": 13.978686955207337, + "grad_norm": 1.0045746564865112, + "learning_rate": 0.0001, + "loss": 62.7059, + "step": 8680 + }, + { + "epoch": 13.99481029878571, + "grad_norm": 0.6980776190757751, + "learning_rate": 0.0001, + "loss": 62.9427, + "step": 8690 + }, + { + "epoch": 14.009674006147025, + "grad_norm": 0.810961127281189, + "learning_rate": 0.0001, + "loss": 58.7552, + "step": 8700 + }, + { + "epoch": 14.0257973497254, + "grad_norm": 0.9210551381111145, + "learning_rate": 0.0001, + "loss": 63.7412, + "step": 8710 + }, + { + "epoch": 14.041920693303775, + "grad_norm": 0.7805785536766052, + "learning_rate": 0.0001, + "loss": 61.477, + "step": 8720 + }, + { + "epoch": 14.058044036882148, + "grad_norm": 0.8528208136558533, + "learning_rate": 0.0001, + "loss": 62.6746, + "step": 8730 + }, + { + "epoch": 14.074167380460523, + "grad_norm": 0.7467619776725769, + "learning_rate": 0.0001, + "loss": 63.8744, + "step": 8740 + }, + { + "epoch": 14.090290724038898, + "grad_norm": 0.6434402465820312, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 8750 + }, + { + "epoch": 14.106414067617273, + "grad_norm": 0.7217757701873779, + "learning_rate": 0.0001, + "loss": 64.0428, + "step": 8760 + }, + { + "epoch": 14.122537411195646, + "grad_norm": 0.8665125966072083, + "learning_rate": 0.0001, + "loss": 62.2772, + "step": 8770 + }, + { + "epoch": 14.13866075477402, + "grad_norm": 0.7868321537971497, + "learning_rate": 0.0001, + "loss": 62.0509, + "step": 8780 + }, + { + "epoch": 14.154784098352396, + "grad_norm": 0.8716760873794556, + "learning_rate": 0.0001, + "loss": 61.6021, + "step": 8790 + }, + { + "epoch": 14.17090744193077, + "grad_norm": 1.0922521352767944, + "learning_rate": 0.0001, + "loss": 64.4197, + "step": 8800 + }, + { + "epoch": 14.187030785509146, + "grad_norm": 0.6545577049255371, + "learning_rate": 0.0001, + "loss": 62.9259, + "step": 8810 + }, + { + "epoch": 14.203154129087519, + "grad_norm": 0.7634063959121704, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 8820 + }, + { + "epoch": 14.219277472665894, + "grad_norm": 0.9075300097465515, + "learning_rate": 0.0001, + "loss": 62.9128, + "step": 8830 + }, + { + "epoch": 14.235400816244269, + "grad_norm": 3.475320816040039, + "learning_rate": 0.0001, + "loss": 62.4841, + "step": 8840 + }, + { + "epoch": 14.251524159822644, + "grad_norm": 0.6839455962181091, + "learning_rate": 0.0001, + "loss": 61.5898, + "step": 8850 + }, + { + "epoch": 14.267647503401017, + "grad_norm": 0.8024231791496277, + "learning_rate": 0.0001, + "loss": 61.1875, + "step": 8860 + }, + { + "epoch": 14.283770846979392, + "grad_norm": 0.8948174118995667, + "learning_rate": 0.0001, + "loss": 62.3386, + "step": 8870 + }, + { + "epoch": 14.299894190557767, + "grad_norm": 0.7571985721588135, + "learning_rate": 0.0001, + "loss": 60.579, + "step": 8880 + }, + { + "epoch": 14.316017534136142, + "grad_norm": 1.432371973991394, + "learning_rate": 0.0001, + "loss": 62.4405, + "step": 8890 + }, + { + "epoch": 14.332140877714515, + "grad_norm": 0.6696875691413879, + "learning_rate": 0.0001, + "loss": 62.9984, + "step": 8900 + }, + { + "epoch": 14.34826422129289, + "grad_norm": 0.749396026134491, + "learning_rate": 0.0001, + "loss": 61.492, + "step": 8910 + }, + { + "epoch": 14.364387564871265, + "grad_norm": 0.664193332195282, + "learning_rate": 0.0001, + "loss": 62.9893, + "step": 8920 + }, + { + "epoch": 14.38051090844964, + "grad_norm": 0.7032697796821594, + "learning_rate": 0.0001, + "loss": 61.1682, + "step": 8930 + }, + { + "epoch": 14.396634252028015, + "grad_norm": 0.813119649887085, + "learning_rate": 0.0001, + "loss": 62.906, + "step": 8940 + }, + { + "epoch": 14.412757595606388, + "grad_norm": 0.7699596881866455, + "learning_rate": 0.0001, + "loss": 64.8087, + "step": 8950 + }, + { + "epoch": 14.428880939184763, + "grad_norm": 0.6424354910850525, + "learning_rate": 0.0001, + "loss": 62.2418, + "step": 8960 + }, + { + "epoch": 14.445004282763138, + "grad_norm": 0.8741731643676758, + "learning_rate": 0.0001, + "loss": 62.656, + "step": 8970 + }, + { + "epoch": 14.461127626341513, + "grad_norm": 0.710617184638977, + "learning_rate": 0.0001, + "loss": 60.8979, + "step": 8980 + }, + { + "epoch": 14.477250969919886, + "grad_norm": 0.6493995785713196, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 8990 + }, + { + "epoch": 14.493374313498261, + "grad_norm": 0.6655835509300232, + "learning_rate": 0.0001, + "loss": 64.151, + "step": 9000 + }, + { + "epoch": 14.509497657076636, + "grad_norm": 0.8140881657600403, + "learning_rate": 0.0001, + "loss": 62.4447, + "step": 9010 + }, + { + "epoch": 14.525621000655011, + "grad_norm": 0.7288498878479004, + "learning_rate": 0.0001, + "loss": 62.5189, + "step": 9020 + }, + { + "epoch": 14.541744344233386, + "grad_norm": 0.6972017884254456, + "learning_rate": 0.0001, + "loss": 61.5441, + "step": 9030 + }, + { + "epoch": 14.55786768781176, + "grad_norm": 0.7444366216659546, + "learning_rate": 0.0001, + "loss": 62.3233, + "step": 9040 + }, + { + "epoch": 14.573991031390134, + "grad_norm": 0.6602845191955566, + "learning_rate": 0.0001, + "loss": 61.4176, + "step": 9050 + }, + { + "epoch": 14.59011437496851, + "grad_norm": 0.7149770855903625, + "learning_rate": 0.0001, + "loss": 62.159, + "step": 9060 + }, + { + "epoch": 14.606237718546884, + "grad_norm": 0.6975772976875305, + "learning_rate": 0.0001, + "loss": 63.1857, + "step": 9070 + }, + { + "epoch": 14.622361062125258, + "grad_norm": 0.7835262417793274, + "learning_rate": 0.0001, + "loss": 61.6795, + "step": 9080 + }, + { + "epoch": 14.638484405703633, + "grad_norm": 5.790656566619873, + "learning_rate": 0.0001, + "loss": 62.7087, + "step": 9090 + }, + { + "epoch": 14.654607749282007, + "grad_norm": 1.0082288980484009, + "learning_rate": 0.0001, + "loss": 61.6808, + "step": 9100 + }, + { + "epoch": 14.670731092860382, + "grad_norm": 0.7758098840713501, + "learning_rate": 0.0001, + "loss": 61.0734, + "step": 9110 + }, + { + "epoch": 14.686854436438756, + "grad_norm": 0.7841722965240479, + "learning_rate": 0.0001, + "loss": 62.8882, + "step": 9120 + }, + { + "epoch": 14.70297778001713, + "grad_norm": 0.7695890069007874, + "learning_rate": 0.0001, + "loss": 60.7811, + "step": 9130 + }, + { + "epoch": 14.719101123595506, + "grad_norm": 0.7444761395454407, + "learning_rate": 0.0001, + "loss": 61.692, + "step": 9140 + }, + { + "epoch": 14.73522446717388, + "grad_norm": 0.7036821246147156, + "learning_rate": 0.0001, + "loss": 63.1698, + "step": 9150 + }, + { + "epoch": 14.751347810752256, + "grad_norm": 0.7891719937324524, + "learning_rate": 0.0001, + "loss": 63.8575, + "step": 9160 + }, + { + "epoch": 14.767471154330629, + "grad_norm": 0.6371109485626221, + "learning_rate": 0.0001, + "loss": 61.3358, + "step": 9170 + }, + { + "epoch": 14.783594497909004, + "grad_norm": 0.6551769375801086, + "learning_rate": 0.0001, + "loss": 62.2705, + "step": 9180 + }, + { + "epoch": 14.799717841487379, + "grad_norm": 0.6922310590744019, + "learning_rate": 0.0001, + "loss": 61.6442, + "step": 9190 + }, + { + "epoch": 14.815841185065754, + "grad_norm": 0.9615464210510254, + "learning_rate": 0.0001, + "loss": 60.932, + "step": 9200 + }, + { + "epoch": 14.831964528644127, + "grad_norm": 0.6367932558059692, + "learning_rate": 0.0001, + "loss": 61.3475, + "step": 9210 + }, + { + "epoch": 14.848087872222502, + "grad_norm": 0.5634505748748779, + "learning_rate": 0.0001, + "loss": 61.7562, + "step": 9220 + }, + { + "epoch": 14.864211215800877, + "grad_norm": 0.7021980285644531, + "learning_rate": 0.0001, + "loss": 62.0672, + "step": 9230 + }, + { + "epoch": 14.880334559379252, + "grad_norm": 0.8832064867019653, + "learning_rate": 0.0001, + "loss": 60.9882, + "step": 9240 + }, + { + "epoch": 14.896457902957625, + "grad_norm": 0.9043561220169067, + "learning_rate": 0.0001, + "loss": 63.0968, + "step": 9250 + }, + { + "epoch": 14.912581246536, + "grad_norm": 0.8958032131195068, + "learning_rate": 0.0001, + "loss": 61.5431, + "step": 9260 + }, + { + "epoch": 14.928704590114375, + "grad_norm": 0.7181615829467773, + "learning_rate": 0.0001, + "loss": 62.3626, + "step": 9270 + }, + { + "epoch": 14.94482793369275, + "grad_norm": 0.7541075348854065, + "learning_rate": 0.0001, + "loss": 61.7186, + "step": 9280 + }, + { + "epoch": 14.960951277271125, + "grad_norm": 1.5627169609069824, + "learning_rate": 0.0001, + "loss": 61.654, + "step": 9290 + }, + { + "epoch": 14.977074620849498, + "grad_norm": 0.7126879096031189, + "learning_rate": 0.0001, + "loss": 61.6806, + "step": 9300 + }, + { + "epoch": 14.993197964427873, + "grad_norm": 0.7397864460945129, + "learning_rate": 0.0001, + "loss": 61.2669, + "step": 9310 + }, + { + "epoch": 15.008061671789187, + "grad_norm": 0.7282766699790955, + "learning_rate": 0.0001, + "loss": 56.2289, + "step": 9320 + }, + { + "epoch": 15.024185015367562, + "grad_norm": 0.8346580862998962, + "learning_rate": 0.0001, + "loss": 61.4335, + "step": 9330 + }, + { + "epoch": 15.040308358945936, + "grad_norm": 0.9117271304130554, + "learning_rate": 0.0001, + "loss": 60.4636, + "step": 9340 + }, + { + "epoch": 15.05643170252431, + "grad_norm": 0.8483272194862366, + "learning_rate": 0.0001, + "loss": 61.3003, + "step": 9350 + }, + { + "epoch": 15.072555046102686, + "grad_norm": 1.1898932456970215, + "learning_rate": 0.0001, + "loss": 62.7169, + "step": 9360 + }, + { + "epoch": 15.08867838968106, + "grad_norm": 0.7258167862892151, + "learning_rate": 0.0001, + "loss": 63.5285, + "step": 9370 + }, + { + "epoch": 15.104801733259436, + "grad_norm": 0.7810865044593811, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 9380 + }, + { + "epoch": 15.120925076837809, + "grad_norm": 1.0359688997268677, + "learning_rate": 0.0001, + "loss": 62.5435, + "step": 9390 + }, + { + "epoch": 15.137048420416184, + "grad_norm": 0.7371546030044556, + "learning_rate": 0.0001, + "loss": 60.5583, + "step": 9400 + }, + { + "epoch": 15.153171763994559, + "grad_norm": 0.7578926086425781, + "learning_rate": 0.0001, + "loss": 61.2853, + "step": 9410 + }, + { + "epoch": 15.169295107572934, + "grad_norm": 0.6161713004112244, + "learning_rate": 0.0001, + "loss": 62.9725, + "step": 9420 + }, + { + "epoch": 15.185418451151307, + "grad_norm": 0.8325598835945129, + "learning_rate": 0.0001, + "loss": 62.857, + "step": 9430 + }, + { + "epoch": 15.201541794729682, + "grad_norm": 1.1438454389572144, + "learning_rate": 0.0001, + "loss": 61.8167, + "step": 9440 + }, + { + "epoch": 15.217665138308057, + "grad_norm": 1.1820837259292603, + "learning_rate": 0.0001, + "loss": 60.9899, + "step": 9450 + }, + { + "epoch": 15.233788481886432, + "grad_norm": 0.6057641506195068, + "learning_rate": 0.0001, + "loss": 62.5144, + "step": 9460 + }, + { + "epoch": 15.249911825464805, + "grad_norm": 0.6661948561668396, + "learning_rate": 0.0001, + "loss": 62.0835, + "step": 9470 + }, + { + "epoch": 15.26603516904318, + "grad_norm": 0.7623168230056763, + "learning_rate": 0.0001, + "loss": 61.9491, + "step": 9480 + }, + { + "epoch": 15.282158512621555, + "grad_norm": 0.6784029006958008, + "learning_rate": 0.0001, + "loss": 63.6006, + "step": 9490 + }, + { + "epoch": 15.29828185619993, + "grad_norm": 0.9433781504631042, + "learning_rate": 0.0001, + "loss": 61.6677, + "step": 9500 + }, + { + "epoch": 15.314405199778305, + "grad_norm": 0.6544079184532166, + "learning_rate": 0.0001, + "loss": 63.1439, + "step": 9510 + }, + { + "epoch": 15.330528543356678, + "grad_norm": 0.7979472279548645, + "learning_rate": 0.0001, + "loss": 62.719, + "step": 9520 + }, + { + "epoch": 15.346651886935053, + "grad_norm": 0.6641252040863037, + "learning_rate": 0.0001, + "loss": 63.2543, + "step": 9530 + }, + { + "epoch": 15.362775230513428, + "grad_norm": 0.7780525088310242, + "learning_rate": 0.0001, + "loss": 62.5659, + "step": 9540 + }, + { + "epoch": 15.378898574091803, + "grad_norm": 0.842349648475647, + "learning_rate": 0.0001, + "loss": 60.041, + "step": 9550 + }, + { + "epoch": 15.395021917670176, + "grad_norm": 0.8991017937660217, + "learning_rate": 0.0001, + "loss": 62.9075, + "step": 9560 + }, + { + "epoch": 15.411145261248551, + "grad_norm": 0.6734052300453186, + "learning_rate": 0.0001, + "loss": 60.8647, + "step": 9570 + }, + { + "epoch": 15.427268604826926, + "grad_norm": 0.7585113644599915, + "learning_rate": 0.0001, + "loss": 62.105, + "step": 9580 + }, + { + "epoch": 15.443391948405301, + "grad_norm": 0.7915840148925781, + "learning_rate": 0.0001, + "loss": 62.2741, + "step": 9590 + }, + { + "epoch": 15.459515291983674, + "grad_norm": 0.8676218390464783, + "learning_rate": 0.0001, + "loss": 62.4548, + "step": 9600 + }, + { + "epoch": 15.47563863556205, + "grad_norm": 0.7698650360107422, + "learning_rate": 0.0001, + "loss": 61.3412, + "step": 9610 + }, + { + "epoch": 15.491761979140424, + "grad_norm": 0.6600908637046814, + "learning_rate": 0.0001, + "loss": 61.1561, + "step": 9620 + }, + { + "epoch": 15.5078853227188, + "grad_norm": 0.5995001792907715, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 9630 + }, + { + "epoch": 15.524008666297174, + "grad_norm": 0.7258557677268982, + "learning_rate": 0.0001, + "loss": 62.4659, + "step": 9640 + }, + { + "epoch": 15.540132009875547, + "grad_norm": 0.6562618613243103, + "learning_rate": 0.0001, + "loss": 63.288, + "step": 9650 + }, + { + "epoch": 15.556255353453922, + "grad_norm": 0.9224995374679565, + "learning_rate": 0.0001, + "loss": 61.4861, + "step": 9660 + }, + { + "epoch": 15.572378697032297, + "grad_norm": 0.7053484916687012, + "learning_rate": 0.0001, + "loss": 62.9339, + "step": 9670 + }, + { + "epoch": 15.588502040610672, + "grad_norm": 1.095229148864746, + "learning_rate": 0.0001, + "loss": 62.1542, + "step": 9680 + }, + { + "epoch": 15.604625384189045, + "grad_norm": 0.7563759088516235, + "learning_rate": 0.0001, + "loss": 61.1644, + "step": 9690 + }, + { + "epoch": 15.62074872776742, + "grad_norm": 0.6508835554122925, + "learning_rate": 0.0001, + "loss": 60.1374, + "step": 9700 + }, + { + "epoch": 15.636872071345795, + "grad_norm": 0.664799153804779, + "learning_rate": 0.0001, + "loss": 60.2331, + "step": 9710 + }, + { + "epoch": 15.65299541492417, + "grad_norm": 0.6525771617889404, + "learning_rate": 0.0001, + "loss": 60.6142, + "step": 9720 + }, + { + "epoch": 15.669118758502545, + "grad_norm": 0.7395381331443787, + "learning_rate": 0.0001, + "loss": 63.5623, + "step": 9730 + }, + { + "epoch": 15.685242102080919, + "grad_norm": 0.7063724398612976, + "learning_rate": 0.0001, + "loss": 63.7133, + "step": 9740 + }, + { + "epoch": 15.701365445659293, + "grad_norm": 0.7918921113014221, + "learning_rate": 0.0001, + "loss": 62.5957, + "step": 9750 + }, + { + "epoch": 15.717488789237668, + "grad_norm": 0.8135486245155334, + "learning_rate": 0.0001, + "loss": 62.9146, + "step": 9760 + }, + { + "epoch": 15.733612132816043, + "grad_norm": 0.8422091007232666, + "learning_rate": 0.0001, + "loss": 61.1466, + "step": 9770 + }, + { + "epoch": 15.749735476394417, + "grad_norm": 0.6329447627067566, + "learning_rate": 0.0001, + "loss": 61.7247, + "step": 9780 + }, + { + "epoch": 15.765858819972792, + "grad_norm": 1.0627039670944214, + "learning_rate": 0.0001, + "loss": 60.6789, + "step": 9790 + }, + { + "epoch": 15.781982163551167, + "grad_norm": 0.7316949367523193, + "learning_rate": 0.0001, + "loss": 61.9896, + "step": 9800 + }, + { + "epoch": 15.798105507129542, + "grad_norm": 0.6968954205513, + "learning_rate": 0.0001, + "loss": 60.8571, + "step": 9810 + }, + { + "epoch": 15.814228850707915, + "grad_norm": 0.7226986289024353, + "learning_rate": 0.0001, + "loss": 62.2664, + "step": 9820 + }, + { + "epoch": 15.83035219428629, + "grad_norm": 0.6542951464653015, + "learning_rate": 0.0001, + "loss": 61.3316, + "step": 9830 + }, + { + "epoch": 15.846475537864665, + "grad_norm": 0.7662289142608643, + "learning_rate": 0.0001, + "loss": 60.6209, + "step": 9840 + }, + { + "epoch": 15.86259888144304, + "grad_norm": 0.7060043215751648, + "learning_rate": 0.0001, + "loss": 62.2647, + "step": 9850 + }, + { + "epoch": 15.878722225021415, + "grad_norm": 0.6948843002319336, + "learning_rate": 0.0001, + "loss": 62.8678, + "step": 9860 + }, + { + "epoch": 15.894845568599788, + "grad_norm": 0.7787327766418457, + "learning_rate": 0.0001, + "loss": 62.9154, + "step": 9870 + }, + { + "epoch": 15.910968912178163, + "grad_norm": 0.7077026963233948, + "learning_rate": 0.0001, + "loss": 62.9537, + "step": 9880 + }, + { + "epoch": 15.927092255756538, + "grad_norm": 0.9957109093666077, + "learning_rate": 0.0001, + "loss": 60.9461, + "step": 9890 + }, + { + "epoch": 15.943215599334913, + "grad_norm": 0.9767161011695862, + "learning_rate": 0.0001, + "loss": 62.5944, + "step": 9900 + }, + { + "epoch": 15.959338942913286, + "grad_norm": 0.6016744375228882, + "learning_rate": 0.0001, + "loss": 62.5889, + "step": 9910 + }, + { + "epoch": 15.975462286491661, + "grad_norm": 0.5464991331100464, + "learning_rate": 0.0001, + "loss": 61.3138, + "step": 9920 + }, + { + "epoch": 15.991585630070036, + "grad_norm": 0.8242359757423401, + "learning_rate": 0.0001, + "loss": 61.8091, + "step": 9930 + }, + { + "epoch": 16.00644933743135, + "grad_norm": 0.7031534314155579, + "learning_rate": 0.0001, + "loss": 57.4847, + "step": 9940 + }, + { + "epoch": 16.022572681009724, + "grad_norm": 0.8429487943649292, + "learning_rate": 0.0001, + "loss": 61.7286, + "step": 9950 + }, + { + "epoch": 16.0386960245881, + "grad_norm": 0.7200850248336792, + "learning_rate": 0.0001, + "loss": 62.6968, + "step": 9960 + }, + { + "epoch": 16.054819368166473, + "grad_norm": 0.7412559390068054, + "learning_rate": 0.0001, + "loss": 62.413, + "step": 9970 + }, + { + "epoch": 16.07094271174485, + "grad_norm": 0.8179170489311218, + "learning_rate": 0.0001, + "loss": 62.7044, + "step": 9980 + }, + { + "epoch": 16.087066055323223, + "grad_norm": 0.6939452886581421, + "learning_rate": 0.0001, + "loss": 61.9062, + "step": 9990 + }, + { + "epoch": 16.1031893989016, + "grad_norm": 0.7145045399665833, + "learning_rate": 0.0001, + "loss": 61.6201, + "step": 10000 + }, + { + "epoch": 16.119312742479973, + "grad_norm": 0.7874398231506348, + "learning_rate": 0.0001, + "loss": 60.4299, + "step": 10010 + }, + { + "epoch": 16.135436086058345, + "grad_norm": 0.6640581488609314, + "learning_rate": 0.0001, + "loss": 62.0195, + "step": 10020 + }, + { + "epoch": 16.15155942963672, + "grad_norm": 0.7903703451156616, + "learning_rate": 0.0001, + "loss": 62.501, + "step": 10030 + }, + { + "epoch": 16.167682773215095, + "grad_norm": 0.6669818162918091, + "learning_rate": 0.0001, + "loss": 59.1489, + "step": 10040 + }, + { + "epoch": 16.18380611679347, + "grad_norm": 0.8380771279335022, + "learning_rate": 0.0001, + "loss": 61.0985, + "step": 10050 + }, + { + "epoch": 16.199929460371845, + "grad_norm": 0.7925304174423218, + "learning_rate": 0.0001, + "loss": 62.9251, + "step": 10060 + }, + { + "epoch": 16.21605280395022, + "grad_norm": 1.2176182270050049, + "learning_rate": 0.0001, + "loss": 61.6772, + "step": 10070 + }, + { + "epoch": 16.232176147528595, + "grad_norm": 0.828171968460083, + "learning_rate": 0.0001, + "loss": 61.3296, + "step": 10080 + }, + { + "epoch": 16.24829949110697, + "grad_norm": 0.6884972453117371, + "learning_rate": 0.0001, + "loss": 61.5371, + "step": 10090 + }, + { + "epoch": 16.264422834685345, + "grad_norm": 0.6455051302909851, + "learning_rate": 0.0001, + "loss": 60.6548, + "step": 10100 + }, + { + "epoch": 16.280546178263716, + "grad_norm": 0.687982976436615, + "learning_rate": 0.0001, + "loss": 62.0357, + "step": 10110 + }, + { + "epoch": 16.29666952184209, + "grad_norm": 0.811744213104248, + "learning_rate": 0.0001, + "loss": 60.5664, + "step": 10120 + }, + { + "epoch": 16.312792865420466, + "grad_norm": 0.7140675187110901, + "learning_rate": 0.0001, + "loss": 62.9491, + "step": 10130 + }, + { + "epoch": 16.32891620899884, + "grad_norm": 0.9068045020103455, + "learning_rate": 0.0001, + "loss": 61.118, + "step": 10140 + }, + { + "epoch": 16.345039552577216, + "grad_norm": 0.9028809666633606, + "learning_rate": 0.0001, + "loss": 61.4308, + "step": 10150 + }, + { + "epoch": 16.36116289615559, + "grad_norm": 0.7502166628837585, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10160 + }, + { + "epoch": 16.377286239733966, + "grad_norm": 0.807391345500946, + "learning_rate": 0.0001, + "loss": 61.6825, + "step": 10170 + }, + { + "epoch": 16.39340958331234, + "grad_norm": 0.762725830078125, + "learning_rate": 0.0001, + "loss": 62.7879, + "step": 10180 + }, + { + "epoch": 16.409532926890712, + "grad_norm": 0.7973660230636597, + "learning_rate": 0.0001, + "loss": 61.0727, + "step": 10190 + }, + { + "epoch": 16.425656270469087, + "grad_norm": 1.004622459411621, + "learning_rate": 0.0001, + "loss": 59.8048, + "step": 10200 + }, + { + "epoch": 16.441779614047462, + "grad_norm": 0.8243395686149597, + "learning_rate": 0.0001, + "loss": 63.9312, + "step": 10210 + }, + { + "epoch": 16.457902957625837, + "grad_norm": 0.7006281614303589, + "learning_rate": 0.0001, + "loss": 61.3766, + "step": 10220 + }, + { + "epoch": 16.474026301204212, + "grad_norm": 0.8672197461128235, + "learning_rate": 0.0001, + "loss": 61.6659, + "step": 10230 + }, + { + "epoch": 16.490149644782587, + "grad_norm": 0.7836477160453796, + "learning_rate": 0.0001, + "loss": 61.3686, + "step": 10240 + }, + { + "epoch": 16.506272988360962, + "grad_norm": 0.5842607617378235, + "learning_rate": 0.0001, + "loss": 62.616, + "step": 10250 + }, + { + "epoch": 16.522396331939337, + "grad_norm": 0.6777102947235107, + "learning_rate": 0.0001, + "loss": 62.1135, + "step": 10260 + }, + { + "epoch": 16.538519675517712, + "grad_norm": 0.856697678565979, + "learning_rate": 0.0001, + "loss": 61.849, + "step": 10270 + }, + { + "epoch": 16.554643019096083, + "grad_norm": 0.7300705313682556, + "learning_rate": 0.0001, + "loss": 60.7678, + "step": 10280 + }, + { + "epoch": 16.57076636267446, + "grad_norm": 0.799802839756012, + "learning_rate": 0.0001, + "loss": 62.4888, + "step": 10290 + }, + { + "epoch": 16.586889706252833, + "grad_norm": 0.791251540184021, + "learning_rate": 0.0001, + "loss": 61.7589, + "step": 10300 + }, + { + "epoch": 16.60301304983121, + "grad_norm": 0.6498433947563171, + "learning_rate": 0.0001, + "loss": 61.087, + "step": 10310 + }, + { + "epoch": 16.619136393409583, + "grad_norm": 0.702925443649292, + "learning_rate": 0.0001, + "loss": 59.3493, + "step": 10320 + }, + { + "epoch": 16.63525973698796, + "grad_norm": 0.8508023619651794, + "learning_rate": 0.0001, + "loss": 61.2188, + "step": 10330 + }, + { + "epoch": 16.651383080566333, + "grad_norm": 0.8325870633125305, + "learning_rate": 0.0001, + "loss": 62.9164, + "step": 10340 + }, + { + "epoch": 16.667506424144708, + "grad_norm": 0.7140557765960693, + "learning_rate": 0.0001, + "loss": 62.5838, + "step": 10350 + }, + { + "epoch": 16.683629767723083, + "grad_norm": 0.5574952363967896, + "learning_rate": 0.0001, + "loss": 59.767, + "step": 10360 + }, + { + "epoch": 16.699753111301455, + "grad_norm": 0.8022965788841248, + "learning_rate": 0.0001, + "loss": 60.2198, + "step": 10370 + }, + { + "epoch": 16.71587645487983, + "grad_norm": 1.1291698217391968, + "learning_rate": 0.0001, + "loss": 61.4908, + "step": 10380 + }, + { + "epoch": 16.731999798458205, + "grad_norm": 0.8651410937309265, + "learning_rate": 0.0001, + "loss": 60.5228, + "step": 10390 + }, + { + "epoch": 16.74812314203658, + "grad_norm": 0.8426535129547119, + "learning_rate": 0.0001, + "loss": 62.3786, + "step": 10400 + }, + { + "epoch": 16.764246485614954, + "grad_norm": 0.6329453587532043, + "learning_rate": 0.0001, + "loss": 60.9621, + "step": 10410 + }, + { + "epoch": 16.78036982919333, + "grad_norm": 0.6745129227638245, + "learning_rate": 0.0001, + "loss": 59.3405, + "step": 10420 + }, + { + "epoch": 16.796493172771704, + "grad_norm": 0.8483593463897705, + "learning_rate": 0.0001, + "loss": 61.1527, + "step": 10430 + }, + { + "epoch": 16.81261651635008, + "grad_norm": 0.8649617433547974, + "learning_rate": 0.0001, + "loss": 61.051, + "step": 10440 + }, + { + "epoch": 16.828739859928454, + "grad_norm": 0.7309728264808655, + "learning_rate": 0.0001, + "loss": 60.3626, + "step": 10450 + }, + { + "epoch": 16.844863203506826, + "grad_norm": 0.6467884182929993, + "learning_rate": 0.0001, + "loss": 63.8254, + "step": 10460 + }, + { + "epoch": 16.8609865470852, + "grad_norm": 0.7628102898597717, + "learning_rate": 0.0001, + "loss": 59.5604, + "step": 10470 + }, + { + "epoch": 16.877109890663576, + "grad_norm": 0.78266841173172, + "learning_rate": 0.0001, + "loss": 60.9387, + "step": 10480 + }, + { + "epoch": 16.89323323424195, + "grad_norm": 0.6936920285224915, + "learning_rate": 0.0001, + "loss": 62.5492, + "step": 10490 + }, + { + "epoch": 16.909356577820326, + "grad_norm": 0.8343939185142517, + "learning_rate": 0.0001, + "loss": 60.9975, + "step": 10500 + }, + { + "epoch": 16.9254799213987, + "grad_norm": 0.798984169960022, + "learning_rate": 0.0001, + "loss": 59.5835, + "step": 10510 + }, + { + "epoch": 16.941603264977076, + "grad_norm": 1.369275450706482, + "learning_rate": 0.0001, + "loss": 61.749, + "step": 10520 + }, + { + "epoch": 16.95772660855545, + "grad_norm": 0.5993408560752869, + "learning_rate": 0.0001, + "loss": 62.1093, + "step": 10530 + }, + { + "epoch": 16.973849952133826, + "grad_norm": 0.8153544664382935, + "learning_rate": 0.0001, + "loss": 61.857, + "step": 10540 + }, + { + "epoch": 16.989973295712197, + "grad_norm": 0.7042689919471741, + "learning_rate": 0.0001, + "loss": 60.5178, + "step": 10550 + }, + { + "epoch": 17.00483700307351, + "grad_norm": 1.1931812763214111, + "learning_rate": 0.0001, + "loss": 56.4815, + "step": 10560 + }, + { + "epoch": 17.020960346651886, + "grad_norm": 0.9051854610443115, + "learning_rate": 0.0001, + "loss": 60.7729, + "step": 10570 + }, + { + "epoch": 17.03708369023026, + "grad_norm": 1.093518614768982, + "learning_rate": 0.0001, + "loss": 60.904, + "step": 10580 + }, + { + "epoch": 17.053207033808636, + "grad_norm": 0.7545284032821655, + "learning_rate": 0.0001, + "loss": 59.7165, + "step": 10590 + }, + { + "epoch": 17.06933037738701, + "grad_norm": 0.8542641401290894, + "learning_rate": 0.0001, + "loss": 60.8947, + "step": 10600 + }, + { + "epoch": 17.085453720965386, + "grad_norm": 0.7393799424171448, + "learning_rate": 0.0001, + "loss": 62.8384, + "step": 10610 + }, + { + "epoch": 17.10157706454376, + "grad_norm": 0.6756435632705688, + "learning_rate": 0.0001, + "loss": 61.164, + "step": 10620 + }, + { + "epoch": 17.117700408122133, + "grad_norm": 0.725150465965271, + "learning_rate": 0.0001, + "loss": 60.3789, + "step": 10630 + }, + { + "epoch": 17.133823751700508, + "grad_norm": 1.1529417037963867, + "learning_rate": 0.0001, + "loss": 62.2465, + "step": 10640 + }, + { + "epoch": 17.149947095278883, + "grad_norm": 0.8266693353652954, + "learning_rate": 0.0001, + "loss": 61.6755, + "step": 10650 + }, + { + "epoch": 17.166070438857258, + "grad_norm": 0.7444040775299072, + "learning_rate": 0.0001, + "loss": 62.7084, + "step": 10660 + }, + { + "epoch": 17.182193782435633, + "grad_norm": 0.8164665102958679, + "learning_rate": 0.0001, + "loss": 61.6032, + "step": 10670 + }, + { + "epoch": 17.198317126014008, + "grad_norm": 0.8215296268463135, + "learning_rate": 0.0001, + "loss": 61.033, + "step": 10680 + }, + { + "epoch": 17.214440469592382, + "grad_norm": 0.7939956188201904, + "learning_rate": 0.0001, + "loss": 60.5016, + "step": 10690 + }, + { + "epoch": 17.230563813170757, + "grad_norm": 0.9533108472824097, + "learning_rate": 0.0001, + "loss": 62.6995, + "step": 10700 + }, + { + "epoch": 17.246687156749132, + "grad_norm": 0.6602820754051208, + "learning_rate": 0.0001, + "loss": 61.2709, + "step": 10710 + }, + { + "epoch": 17.262810500327504, + "grad_norm": 0.8369201421737671, + "learning_rate": 0.0001, + "loss": 63.1138, + "step": 10720 + }, + { + "epoch": 17.27893384390588, + "grad_norm": 0.7431164979934692, + "learning_rate": 0.0001, + "loss": 62.8588, + "step": 10730 + }, + { + "epoch": 17.295057187484254, + "grad_norm": 0.7957271337509155, + "learning_rate": 0.0001, + "loss": 60.5119, + "step": 10740 + }, + { + "epoch": 17.31118053106263, + "grad_norm": 0.9407948851585388, + "learning_rate": 0.0001, + "loss": 61.1877, + "step": 10750 + }, + { + "epoch": 17.327303874641004, + "grad_norm": 0.6641581654548645, + "learning_rate": 0.0001, + "loss": 60.542, + "step": 10760 + }, + { + "epoch": 17.34342721821938, + "grad_norm": 0.7786923050880432, + "learning_rate": 0.0001, + "loss": 62.1582, + "step": 10770 + }, + { + "epoch": 17.359550561797754, + "grad_norm": 0.8150411248207092, + "learning_rate": 0.0001, + "loss": 61.12, + "step": 10780 + }, + { + "epoch": 17.37567390537613, + "grad_norm": 0.8315487504005432, + "learning_rate": 0.0001, + "loss": 60.4382, + "step": 10790 + }, + { + "epoch": 17.391797248954504, + "grad_norm": 0.8305006623268127, + "learning_rate": 0.0001, + "loss": 61.3262, + "step": 10800 + }, + { + "epoch": 17.407920592532875, + "grad_norm": 0.897731363773346, + "learning_rate": 0.0001, + "loss": 60.7809, + "step": 10810 + }, + { + "epoch": 17.42404393611125, + "grad_norm": 0.7461619973182678, + "learning_rate": 0.0001, + "loss": 61.3858, + "step": 10820 + }, + { + "epoch": 17.440167279689625, + "grad_norm": 0.6750472187995911, + "learning_rate": 0.0001, + "loss": 60.2692, + "step": 10830 + }, + { + "epoch": 17.456290623268, + "grad_norm": 0.5412933230400085, + "learning_rate": 0.0001, + "loss": 61.7872, + "step": 10840 + }, + { + "epoch": 17.472413966846375, + "grad_norm": 0.6485120058059692, + "learning_rate": 0.0001, + "loss": 59.5407, + "step": 10850 + }, + { + "epoch": 17.48853731042475, + "grad_norm": 0.818708598613739, + "learning_rate": 0.0001, + "loss": 61.8066, + "step": 10860 + }, + { + "epoch": 17.504660654003125, + "grad_norm": 0.7121092081069946, + "learning_rate": 0.0001, + "loss": 61.8086, + "step": 10870 + }, + { + "epoch": 17.5207839975815, + "grad_norm": 0.7031684517860413, + "learning_rate": 0.0001, + "loss": 60.0706, + "step": 10880 + }, + { + "epoch": 17.536907341159875, + "grad_norm": 0.6259323358535767, + "learning_rate": 0.0001, + "loss": 61.9625, + "step": 10890 + }, + { + "epoch": 17.553030684738246, + "grad_norm": 0.8526529669761658, + "learning_rate": 0.0001, + "loss": 61.3299, + "step": 10900 + }, + { + "epoch": 17.56915402831662, + "grad_norm": 0.6860859990119934, + "learning_rate": 0.0001, + "loss": 61.3193, + "step": 10910 + }, + { + "epoch": 17.585277371894996, + "grad_norm": 0.7715994119644165, + "learning_rate": 0.0001, + "loss": 61.9111, + "step": 10920 + }, + { + "epoch": 17.60140071547337, + "grad_norm": 0.7171186804771423, + "learning_rate": 0.0001, + "loss": 60.3638, + "step": 10930 + }, + { + "epoch": 17.617524059051746, + "grad_norm": 0.7796649932861328, + "learning_rate": 0.0001, + "loss": 61.9437, + "step": 10940 + }, + { + "epoch": 17.63364740263012, + "grad_norm": 0.7930101752281189, + "learning_rate": 0.0001, + "loss": 60.1085, + "step": 10950 + }, + { + "epoch": 17.649770746208496, + "grad_norm": 0.6370466351509094, + "learning_rate": 0.0001, + "loss": 61.8433, + "step": 10960 + }, + { + "epoch": 17.66589408978687, + "grad_norm": 0.7830466628074646, + "learning_rate": 0.0001, + "loss": 61.3403, + "step": 10970 + }, + { + "epoch": 17.682017433365242, + "grad_norm": 0.873456597328186, + "learning_rate": 0.0001, + "loss": 62.4567, + "step": 10980 + }, + { + "epoch": 17.698140776943617, + "grad_norm": 0.7560754418373108, + "learning_rate": 0.0001, + "loss": 60.9808, + "step": 10990 + }, + { + "epoch": 17.714264120521992, + "grad_norm": 0.6832230091094971, + "learning_rate": 0.0001, + "loss": 61.5059, + "step": 11000 + }, + { + "epoch": 17.730387464100367, + "grad_norm": 0.7701742649078369, + "learning_rate": 0.0001, + "loss": 62.4963, + "step": 11010 + }, + { + "epoch": 17.746510807678742, + "grad_norm": 0.7088269591331482, + "learning_rate": 0.0001, + "loss": 62.6467, + "step": 11020 + }, + { + "epoch": 17.762634151257117, + "grad_norm": 0.8424612879753113, + "learning_rate": 0.0001, + "loss": 60.1213, + "step": 11030 + }, + { + "epoch": 17.778757494835492, + "grad_norm": 0.9974421858787537, + "learning_rate": 0.0001, + "loss": 61.6051, + "step": 11040 + }, + { + "epoch": 17.794880838413867, + "grad_norm": 0.701970636844635, + "learning_rate": 0.0001, + "loss": 62.4155, + "step": 11050 + }, + { + "epoch": 17.811004181992242, + "grad_norm": 0.6826842427253723, + "learning_rate": 0.0001, + "loss": 60.2127, + "step": 11060 + }, + { + "epoch": 17.827127525570614, + "grad_norm": 0.7019745707511902, + "learning_rate": 0.0001, + "loss": 60.9281, + "step": 11070 + }, + { + "epoch": 17.84325086914899, + "grad_norm": 0.836280345916748, + "learning_rate": 0.0001, + "loss": 61.1217, + "step": 11080 + }, + { + "epoch": 17.859374212727364, + "grad_norm": 0.6405923366546631, + "learning_rate": 0.0001, + "loss": 61.6393, + "step": 11090 + }, + { + "epoch": 17.87549755630574, + "grad_norm": 0.6809528470039368, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 11100 + }, + { + "epoch": 17.891620899884114, + "grad_norm": 0.6983410120010376, + "learning_rate": 0.0001, + "loss": 59.933, + "step": 11110 + }, + { + "epoch": 17.90774424346249, + "grad_norm": 1.0240110158920288, + "learning_rate": 0.0001, + "loss": 61.1964, + "step": 11120 + }, + { + "epoch": 17.923867587040863, + "grad_norm": 0.860657811164856, + "learning_rate": 0.0001, + "loss": 61.1613, + "step": 11130 + }, + { + "epoch": 17.93999093061924, + "grad_norm": 0.6434107422828674, + "learning_rate": 0.0001, + "loss": 60.2435, + "step": 11140 + }, + { + "epoch": 17.956114274197613, + "grad_norm": 0.7400848269462585, + "learning_rate": 0.0001, + "loss": 62.452, + "step": 11150 + }, + { + "epoch": 17.972237617775985, + "grad_norm": 0.6526095867156982, + "learning_rate": 0.0001, + "loss": 62.3344, + "step": 11160 + }, + { + "epoch": 17.98836096135436, + "grad_norm": 0.6971097588539124, + "learning_rate": 0.0001, + "loss": 60.2415, + "step": 11170 + }, + { + "epoch": 18.003224668715674, + "grad_norm": 0.724881649017334, + "learning_rate": 0.0001, + "loss": 56.0513, + "step": 11180 + }, + { + "epoch": 18.01934801229405, + "grad_norm": 0.653710663318634, + "learning_rate": 0.0001, + "loss": 60.8364, + "step": 11190 + }, + { + "epoch": 18.035471355872424, + "grad_norm": 0.7356674075126648, + "learning_rate": 0.0001, + "loss": 61.6681, + "step": 11200 + }, + { + "epoch": 18.0515946994508, + "grad_norm": 0.6726879477500916, + "learning_rate": 0.0001, + "loss": 60.6682, + "step": 11210 + }, + { + "epoch": 18.067718043029174, + "grad_norm": 0.7367722392082214, + "learning_rate": 0.0001, + "loss": 63.0462, + "step": 11220 + }, + { + "epoch": 18.08384138660755, + "grad_norm": 0.6687121987342834, + "learning_rate": 0.0001, + "loss": 60.8707, + "step": 11230 + }, + { + "epoch": 18.09996473018592, + "grad_norm": 1.0027060508728027, + "learning_rate": 0.0001, + "loss": 62.4703, + "step": 11240 + }, + { + "epoch": 18.116088073764296, + "grad_norm": 1.0447720289230347, + "learning_rate": 0.0001, + "loss": 62.1746, + "step": 11250 + }, + { + "epoch": 18.13221141734267, + "grad_norm": 0.7687053084373474, + "learning_rate": 0.0001, + "loss": 60.1413, + "step": 11260 + }, + { + "epoch": 18.148334760921045, + "grad_norm": 0.7250781059265137, + "learning_rate": 0.0001, + "loss": 60.1143, + "step": 11270 + }, + { + "epoch": 18.16445810449942, + "grad_norm": 0.7356208562850952, + "learning_rate": 0.0001, + "loss": 62.0788, + "step": 11280 + }, + { + "epoch": 18.180581448077795, + "grad_norm": 0.7838225364685059, + "learning_rate": 0.0001, + "loss": 61.7046, + "step": 11290 + }, + { + "epoch": 18.19670479165617, + "grad_norm": 0.7600042819976807, + "learning_rate": 0.0001, + "loss": 61.4646, + "step": 11300 + }, + { + "epoch": 18.212828135234545, + "grad_norm": 0.7339571714401245, + "learning_rate": 0.0001, + "loss": 61.0983, + "step": 11310 + }, + { + "epoch": 18.22895147881292, + "grad_norm": 0.6982674598693848, + "learning_rate": 0.0001, + "loss": 60.2853, + "step": 11320 + }, + { + "epoch": 18.24507482239129, + "grad_norm": 0.7572301030158997, + "learning_rate": 0.0001, + "loss": 61.8967, + "step": 11330 + }, + { + "epoch": 18.261198165969667, + "grad_norm": 0.6206921935081482, + "learning_rate": 0.0001, + "loss": 62.1467, + "step": 11340 + }, + { + "epoch": 18.27732150954804, + "grad_norm": 0.658437967300415, + "learning_rate": 0.0001, + "loss": 61.2117, + "step": 11350 + }, + { + "epoch": 18.293444853126417, + "grad_norm": 0.7880764603614807, + "learning_rate": 0.0001, + "loss": 61.6246, + "step": 11360 + }, + { + "epoch": 18.30956819670479, + "grad_norm": 0.6601800918579102, + "learning_rate": 0.0001, + "loss": 61.4818, + "step": 11370 + }, + { + "epoch": 18.325691540283167, + "grad_norm": 0.6816290616989136, + "learning_rate": 0.0001, + "loss": 60.4834, + "step": 11380 + }, + { + "epoch": 18.34181488386154, + "grad_norm": 0.7667511105537415, + "learning_rate": 0.0001, + "loss": 60.9106, + "step": 11390 + }, + { + "epoch": 18.357938227439917, + "grad_norm": 0.6490039825439453, + "learning_rate": 0.0001, + "loss": 59.6968, + "step": 11400 + }, + { + "epoch": 18.37406157101829, + "grad_norm": 0.6865658760070801, + "learning_rate": 0.0001, + "loss": 61.7898, + "step": 11410 + }, + { + "epoch": 18.390184914596663, + "grad_norm": 0.8605737090110779, + "learning_rate": 0.0001, + "loss": 60.924, + "step": 11420 + }, + { + "epoch": 18.406308258175038, + "grad_norm": 1.0474110841751099, + "learning_rate": 0.0001, + "loss": 61.0381, + "step": 11430 + }, + { + "epoch": 18.422431601753413, + "grad_norm": 0.9241340160369873, + "learning_rate": 0.0001, + "loss": 61.2609, + "step": 11440 + }, + { + "epoch": 18.438554945331788, + "grad_norm": 1.1441621780395508, + "learning_rate": 0.0001, + "loss": 60.5495, + "step": 11450 + }, + { + "epoch": 18.454678288910163, + "grad_norm": 0.7277412414550781, + "learning_rate": 0.0001, + "loss": 60.4654, + "step": 11460 + }, + { + "epoch": 18.470801632488538, + "grad_norm": 0.6668717861175537, + "learning_rate": 0.0001, + "loss": 60.057, + "step": 11470 + }, + { + "epoch": 18.486924976066913, + "grad_norm": 0.7335819005966187, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 11480 + }, + { + "epoch": 18.503048319645288, + "grad_norm": 0.7261807918548584, + "learning_rate": 0.0001, + "loss": 61.1508, + "step": 11490 + }, + { + "epoch": 18.519171663223663, + "grad_norm": 0.7715283036231995, + "learning_rate": 0.0001, + "loss": 61.5492, + "step": 11500 + }, + { + "epoch": 18.535295006802034, + "grad_norm": 0.9603212475776672, + "learning_rate": 0.0001, + "loss": 60.4321, + "step": 11510 + }, + { + "epoch": 18.55141835038041, + "grad_norm": 0.6130046844482422, + "learning_rate": 0.0001, + "loss": 61.2942, + "step": 11520 + }, + { + "epoch": 18.567541693958784, + "grad_norm": 0.8147348165512085, + "learning_rate": 0.0001, + "loss": 61.4313, + "step": 11530 + }, + { + "epoch": 18.58366503753716, + "grad_norm": 0.7909948825836182, + "learning_rate": 0.0001, + "loss": 60.3547, + "step": 11540 + }, + { + "epoch": 18.599788381115534, + "grad_norm": 0.6470443606376648, + "learning_rate": 0.0001, + "loss": 61.4144, + "step": 11550 + }, + { + "epoch": 18.61591172469391, + "grad_norm": 0.7132186889648438, + "learning_rate": 0.0001, + "loss": 61.8102, + "step": 11560 + }, + { + "epoch": 18.632035068272284, + "grad_norm": 0.7054323554039001, + "learning_rate": 0.0001, + "loss": 62.1431, + "step": 11570 + }, + { + "epoch": 18.64815841185066, + "grad_norm": 0.8784761428833008, + "learning_rate": 0.0001, + "loss": 61.6653, + "step": 11580 + }, + { + "epoch": 18.66428175542903, + "grad_norm": 0.7143248319625854, + "learning_rate": 0.0001, + "loss": 60.1275, + "step": 11590 + }, + { + "epoch": 18.680405099007405, + "grad_norm": 7.87012243270874, + "learning_rate": 0.0001, + "loss": 61.3484, + "step": 11600 + }, + { + "epoch": 18.69652844258578, + "grad_norm": 0.7263020873069763, + "learning_rate": 0.0001, + "loss": 59.906, + "step": 11610 + }, + { + "epoch": 18.712651786164155, + "grad_norm": 0.6468373537063599, + "learning_rate": 0.0001, + "loss": 61.8781, + "step": 11620 + }, + { + "epoch": 18.72877512974253, + "grad_norm": 3.34307861328125, + "learning_rate": 0.0001, + "loss": 61.0841, + "step": 11630 + }, + { + "epoch": 18.744898473320905, + "grad_norm": 0.7348340153694153, + "learning_rate": 0.0001, + "loss": 61.3201, + "step": 11640 + }, + { + "epoch": 18.76102181689928, + "grad_norm": 0.7646427154541016, + "learning_rate": 0.0001, + "loss": 60.1858, + "step": 11650 + }, + { + "epoch": 18.777145160477655, + "grad_norm": 0.6659495234489441, + "learning_rate": 0.0001, + "loss": 60.8999, + "step": 11660 + }, + { + "epoch": 18.79326850405603, + "grad_norm": 0.9913583993911743, + "learning_rate": 0.0001, + "loss": 59.0732, + "step": 11670 + }, + { + "epoch": 18.8093918476344, + "grad_norm": 0.7480027079582214, + "learning_rate": 0.0001, + "loss": 58.6846, + "step": 11680 + }, + { + "epoch": 18.825515191212777, + "grad_norm": 0.8000630736351013, + "learning_rate": 0.0001, + "loss": 62.8883, + "step": 11690 + }, + { + "epoch": 18.84163853479115, + "grad_norm": 0.8450509309768677, + "learning_rate": 0.0001, + "loss": 63.4962, + "step": 11700 + }, + { + "epoch": 18.857761878369526, + "grad_norm": 0.8365609645843506, + "learning_rate": 0.0001, + "loss": 62.2701, + "step": 11710 + }, + { + "epoch": 18.8738852219479, + "grad_norm": 0.7197144031524658, + "learning_rate": 0.0001, + "loss": 61.3158, + "step": 11720 + }, + { + "epoch": 18.890008565526276, + "grad_norm": 0.6460379958152771, + "learning_rate": 0.0001, + "loss": 62.6645, + "step": 11730 + }, + { + "epoch": 18.90613190910465, + "grad_norm": 0.8203967213630676, + "learning_rate": 0.0001, + "loss": 62.3257, + "step": 11740 + }, + { + "epoch": 18.922255252683026, + "grad_norm": 0.682134211063385, + "learning_rate": 0.0001, + "loss": 59.9144, + "step": 11750 + }, + { + "epoch": 18.9383785962614, + "grad_norm": 0.7080821394920349, + "learning_rate": 0.0001, + "loss": 61.4325, + "step": 11760 + }, + { + "epoch": 18.954501939839773, + "grad_norm": 0.7963712811470032, + "learning_rate": 0.0001, + "loss": 58.6136, + "step": 11770 + }, + { + "epoch": 18.970625283418148, + "grad_norm": 0.7317046523094177, + "learning_rate": 0.0001, + "loss": 61.3507, + "step": 11780 + }, + { + "epoch": 18.986748626996523, + "grad_norm": 0.6586548686027527, + "learning_rate": 0.0001, + "loss": 61.4898, + "step": 11790 + }, + { + "epoch": 19.001612334357837, + "grad_norm": 0.9010424613952637, + "learning_rate": 0.0001, + "loss": 56.8915, + "step": 11800 + }, + { + "epoch": 19.017735677936212, + "grad_norm": 0.7102377414703369, + "learning_rate": 0.0001, + "loss": 62.2097, + "step": 11810 + }, + { + "epoch": 19.033859021514587, + "grad_norm": 0.7959222793579102, + "learning_rate": 0.0001, + "loss": 61.4484, + "step": 11820 + }, + { + "epoch": 19.049982365092962, + "grad_norm": 0.6483829617500305, + "learning_rate": 0.0001, + "loss": 60.3563, + "step": 11830 + }, + { + "epoch": 19.066105708671337, + "grad_norm": 1.1301988363265991, + "learning_rate": 0.0001, + "loss": 61.3698, + "step": 11840 + }, + { + "epoch": 19.082229052249712, + "grad_norm": 0.7304167151451111, + "learning_rate": 0.0001, + "loss": 59.2096, + "step": 11850 + }, + { + "epoch": 19.098352395828083, + "grad_norm": 0.7757825255393982, + "learning_rate": 0.0001, + "loss": 59.914, + "step": 11860 + }, + { + "epoch": 19.11447573940646, + "grad_norm": 0.7009251713752747, + "learning_rate": 0.0001, + "loss": 61.0768, + "step": 11870 + }, + { + "epoch": 19.130599082984833, + "grad_norm": 0.6015574336051941, + "learning_rate": 0.0001, + "loss": 61.2698, + "step": 11880 + }, + { + "epoch": 19.14672242656321, + "grad_norm": 0.8028033971786499, + "learning_rate": 0.0001, + "loss": 61.422, + "step": 11890 + }, + { + "epoch": 19.162845770141583, + "grad_norm": 0.5753505825996399, + "learning_rate": 0.0001, + "loss": 61.5066, + "step": 11900 + }, + { + "epoch": 19.17896911371996, + "grad_norm": 0.7702978849411011, + "learning_rate": 0.0001, + "loss": 60.0737, + "step": 11910 + }, + { + "epoch": 19.195092457298333, + "grad_norm": 0.6469365358352661, + "learning_rate": 0.0001, + "loss": 59.5091, + "step": 11920 + }, + { + "epoch": 19.211215800876708, + "grad_norm": 0.8400216698646545, + "learning_rate": 0.0001, + "loss": 62.3739, + "step": 11930 + }, + { + "epoch": 19.22733914445508, + "grad_norm": 1.9919443130493164, + "learning_rate": 0.0001, + "loss": 59.7271, + "step": 11940 + }, + { + "epoch": 19.243462488033455, + "grad_norm": 1.0247725248336792, + "learning_rate": 0.0001, + "loss": 59.3804, + "step": 11950 + }, + { + "epoch": 19.25958583161183, + "grad_norm": 1.0717105865478516, + "learning_rate": 0.0001, + "loss": 61.3617, + "step": 11960 + }, + { + "epoch": 19.275709175190205, + "grad_norm": 0.6324111223220825, + "learning_rate": 0.0001, + "loss": 59.6094, + "step": 11970 + }, + { + "epoch": 19.29183251876858, + "grad_norm": 0.6665195226669312, + "learning_rate": 0.0001, + "loss": 61.5879, + "step": 11980 + }, + { + "epoch": 19.307955862346954, + "grad_norm": 0.7933892011642456, + "learning_rate": 0.0001, + "loss": 60.4011, + "step": 11990 + }, + { + "epoch": 19.32407920592533, + "grad_norm": 0.8135427236557007, + "learning_rate": 0.0001, + "loss": 61.2936, + "step": 12000 + }, + { + "epoch": 19.340202549503704, + "grad_norm": 0.7154449820518494, + "learning_rate": 0.0001, + "loss": 62.3996, + "step": 12010 + }, + { + "epoch": 19.35632589308208, + "grad_norm": 0.9926077127456665, + "learning_rate": 0.0001, + "loss": 60.5219, + "step": 12020 + }, + { + "epoch": 19.37244923666045, + "grad_norm": 1.013094425201416, + "learning_rate": 0.0001, + "loss": 59.9494, + "step": 12030 + }, + { + "epoch": 19.388572580238826, + "grad_norm": 0.984582245349884, + "learning_rate": 0.0001, + "loss": 59.7877, + "step": 12040 + }, + { + "epoch": 19.4046959238172, + "grad_norm": 1.256519079208374, + "learning_rate": 0.0001, + "loss": 60.2608, + "step": 12050 + }, + { + "epoch": 19.420819267395576, + "grad_norm": 0.7013972997665405, + "learning_rate": 0.0001, + "loss": 63.4573, + "step": 12060 + }, + { + "epoch": 19.43694261097395, + "grad_norm": 0.9027557373046875, + "learning_rate": 0.0001, + "loss": 61.3252, + "step": 12070 + }, + { + "epoch": 19.453065954552326, + "grad_norm": 0.9057058691978455, + "learning_rate": 0.0001, + "loss": 60.4992, + "step": 12080 + }, + { + "epoch": 19.4691892981307, + "grad_norm": 0.6828131079673767, + "learning_rate": 0.0001, + "loss": 61.5369, + "step": 12090 + }, + { + "epoch": 19.485312641709076, + "grad_norm": 0.617858350276947, + "learning_rate": 0.0001, + "loss": 61.4837, + "step": 12100 + }, + { + "epoch": 19.50143598528745, + "grad_norm": 0.7722935676574707, + "learning_rate": 0.0001, + "loss": 61.0886, + "step": 12110 + }, + { + "epoch": 19.517559328865822, + "grad_norm": 0.7039631009101868, + "learning_rate": 0.0001, + "loss": 59.9055, + "step": 12120 + }, + { + "epoch": 19.533682672444197, + "grad_norm": 0.8353239297866821, + "learning_rate": 0.0001, + "loss": 62.0864, + "step": 12130 + }, + { + "epoch": 19.549806016022572, + "grad_norm": 0.764098048210144, + "learning_rate": 0.0001, + "loss": 58.9591, + "step": 12140 + }, + { + "epoch": 19.565929359600947, + "grad_norm": 0.7842984795570374, + "learning_rate": 0.0001, + "loss": 59.0494, + "step": 12150 + }, + { + "epoch": 19.582052703179322, + "grad_norm": 0.7140147686004639, + "learning_rate": 0.0001, + "loss": 60.8067, + "step": 12160 + }, + { + "epoch": 19.598176046757697, + "grad_norm": 0.8182031512260437, + "learning_rate": 0.0001, + "loss": 59.6013, + "step": 12170 + }, + { + "epoch": 19.614299390336072, + "grad_norm": 0.7428567409515381, + "learning_rate": 0.0001, + "loss": 61.8644, + "step": 12180 + }, + { + "epoch": 19.630422733914447, + "grad_norm": 0.6174675822257996, + "learning_rate": 0.0001, + "loss": 60.2086, + "step": 12190 + }, + { + "epoch": 19.646546077492822, + "grad_norm": 0.6416913866996765, + "learning_rate": 0.0001, + "loss": 61.3452, + "step": 12200 + }, + { + "epoch": 19.662669421071193, + "grad_norm": 1.3036755323410034, + "learning_rate": 0.0001, + "loss": 60.5332, + "step": 12210 + }, + { + "epoch": 19.678792764649568, + "grad_norm": 1.230710506439209, + "learning_rate": 0.0001, + "loss": 62.5671, + "step": 12220 + }, + { + "epoch": 19.694916108227943, + "grad_norm": 0.6914290189743042, + "learning_rate": 0.0001, + "loss": 60.7302, + "step": 12230 + }, + { + "epoch": 19.711039451806318, + "grad_norm": 0.6492108106613159, + "learning_rate": 0.0001, + "loss": 60.6465, + "step": 12240 + }, + { + "epoch": 19.727162795384693, + "grad_norm": 0.7104936838150024, + "learning_rate": 0.0001, + "loss": 61.14, + "step": 12250 + }, + { + "epoch": 19.743286138963068, + "grad_norm": 1.0740787982940674, + "learning_rate": 0.0001, + "loss": 60.1977, + "step": 12260 + }, + { + "epoch": 19.759409482541443, + "grad_norm": 0.5764663815498352, + "learning_rate": 0.0001, + "loss": 59.994, + "step": 12270 + }, + { + "epoch": 19.775532826119818, + "grad_norm": 0.6204820871353149, + "learning_rate": 0.0001, + "loss": 60.3091, + "step": 12280 + }, + { + "epoch": 19.791656169698193, + "grad_norm": 0.702944815158844, + "learning_rate": 0.0001, + "loss": 60.8637, + "step": 12290 + }, + { + "epoch": 19.807779513276564, + "grad_norm": 0.7591354250907898, + "learning_rate": 0.0001, + "loss": 59.54, + "step": 12300 + }, + { + "epoch": 19.82390285685494, + "grad_norm": 0.755224883556366, + "learning_rate": 0.0001, + "loss": 60.0285, + "step": 12310 + }, + { + "epoch": 19.840026200433314, + "grad_norm": 0.7563093900680542, + "learning_rate": 0.0001, + "loss": 60.2669, + "step": 12320 + }, + { + "epoch": 19.85614954401169, + "grad_norm": 0.8339024782180786, + "learning_rate": 0.0001, + "loss": 60.606, + "step": 12330 + }, + { + "epoch": 19.872272887590064, + "grad_norm": 0.7017751932144165, + "learning_rate": 0.0001, + "loss": 59.579, + "step": 12340 + }, + { + "epoch": 19.88839623116844, + "grad_norm": 1.3561909198760986, + "learning_rate": 0.0001, + "loss": 59.7727, + "step": 12350 + }, + { + "epoch": 19.904519574746814, + "grad_norm": 0.727020263671875, + "learning_rate": 0.0001, + "loss": 60.6587, + "step": 12360 + }, + { + "epoch": 19.92064291832519, + "grad_norm": 0.6882230043411255, + "learning_rate": 0.0001, + "loss": 60.3916, + "step": 12370 + }, + { + "epoch": 19.93676626190356, + "grad_norm": 0.6410917639732361, + "learning_rate": 0.0001, + "loss": 61.6649, + "step": 12380 + }, + { + "epoch": 19.952889605481936, + "grad_norm": 0.9747995138168335, + "learning_rate": 0.0001, + "loss": 60.4622, + "step": 12390 + }, + { + "epoch": 19.96901294906031, + "grad_norm": 1.260621190071106, + "learning_rate": 0.0001, + "loss": 60.4228, + "step": 12400 + }, + { + "epoch": 19.985136292638686, + "grad_norm": 0.6525571942329407, + "learning_rate": 0.0001, + "loss": 59.0612, + "step": 12410 + }, + { + "epoch": 20.0, + "grad_norm": 0.7864645719528198, + "learning_rate": 0.0001, + "loss": 57.2347, + "step": 12420 + }, + { + "epoch": 20.016123343578375, + "grad_norm": 0.7479318380355835, + "learning_rate": 0.0001, + "loss": 60.9263, + "step": 12430 + }, + { + "epoch": 20.03224668715675, + "grad_norm": 0.6433089375495911, + "learning_rate": 0.0001, + "loss": 59.8846, + "step": 12440 + }, + { + "epoch": 20.048370030735125, + "grad_norm": 0.7594776153564453, + "learning_rate": 0.0001, + "loss": 61.7793, + "step": 12450 + }, + { + "epoch": 20.0644933743135, + "grad_norm": 0.8710535764694214, + "learning_rate": 0.0001, + "loss": 60.5331, + "step": 12460 + }, + { + "epoch": 20.08061671789187, + "grad_norm": 0.7302114963531494, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 12470 + }, + { + "epoch": 20.096740061470246, + "grad_norm": 0.6039112210273743, + "learning_rate": 0.0001, + "loss": 60.8272, + "step": 12480 + }, + { + "epoch": 20.11286340504862, + "grad_norm": 0.7339144945144653, + "learning_rate": 0.0001, + "loss": 61.4149, + "step": 12490 + }, + { + "epoch": 20.128986748626996, + "grad_norm": 0.7278878092765808, + "learning_rate": 0.0001, + "loss": 62.631, + "step": 12500 + }, + { + "epoch": 20.14511009220537, + "grad_norm": 0.7241142988204956, + "learning_rate": 0.0001, + "loss": 59.9328, + "step": 12510 + }, + { + "epoch": 20.161233435783746, + "grad_norm": 0.6967542767524719, + "learning_rate": 0.0001, + "loss": 58.7463, + "step": 12520 + }, + { + "epoch": 20.17735677936212, + "grad_norm": 0.7326555848121643, + "learning_rate": 0.0001, + "loss": 59.7082, + "step": 12530 + }, + { + "epoch": 20.193480122940496, + "grad_norm": 0.6706270575523376, + "learning_rate": 0.0001, + "loss": 60.5978, + "step": 12540 + }, + { + "epoch": 20.20960346651887, + "grad_norm": 0.8541543483734131, + "learning_rate": 0.0001, + "loss": 58.9632, + "step": 12550 + }, + { + "epoch": 20.225726810097242, + "grad_norm": 0.6599560379981995, + "learning_rate": 0.0001, + "loss": 58.4572, + "step": 12560 + }, + { + "epoch": 20.241850153675617, + "grad_norm": 0.6668654680252075, + "learning_rate": 0.0001, + "loss": 61.8741, + "step": 12570 + }, + { + "epoch": 20.257973497253992, + "grad_norm": 0.6453808546066284, + "learning_rate": 0.0001, + "loss": 60.2538, + "step": 12580 + }, + { + "epoch": 20.274096840832367, + "grad_norm": 0.7429870963096619, + "learning_rate": 0.0001, + "loss": 60.5795, + "step": 12590 + }, + { + "epoch": 20.290220184410742, + "grad_norm": 0.7588251829147339, + "learning_rate": 0.0001, + "loss": 60.6261, + "step": 12600 + }, + { + "epoch": 20.306343527989117, + "grad_norm": 0.6430977582931519, + "learning_rate": 0.0001, + "loss": 61.0344, + "step": 12610 + }, + { + "epoch": 20.322466871567492, + "grad_norm": 0.6879712343215942, + "learning_rate": 0.0001, + "loss": 62.2668, + "step": 12620 + }, + { + "epoch": 20.338590215145867, + "grad_norm": 0.7650917172431946, + "learning_rate": 0.0001, + "loss": 60.3852, + "step": 12630 + }, + { + "epoch": 20.35471355872424, + "grad_norm": 0.5326449871063232, + "learning_rate": 0.0001, + "loss": 60.6138, + "step": 12640 + }, + { + "epoch": 20.370836902302614, + "grad_norm": 0.7018648982048035, + "learning_rate": 0.0001, + "loss": 62.1081, + "step": 12650 + }, + { + "epoch": 20.38696024588099, + "grad_norm": 0.7988570332527161, + "learning_rate": 0.0001, + "loss": 61.2661, + "step": 12660 + }, + { + "epoch": 20.403083589459364, + "grad_norm": 0.6800012588500977, + "learning_rate": 0.0001, + "loss": 59.9279, + "step": 12670 + }, + { + "epoch": 20.41920693303774, + "grad_norm": 0.6380598545074463, + "learning_rate": 0.0001, + "loss": 60.4877, + "step": 12680 + }, + { + "epoch": 20.435330276616114, + "grad_norm": 0.6755974888801575, + "learning_rate": 0.0001, + "loss": 60.9729, + "step": 12690 + }, + { + "epoch": 20.45145362019449, + "grad_norm": 0.8057810664176941, + "learning_rate": 0.0001, + "loss": 59.6747, + "step": 12700 + }, + { + "epoch": 20.467576963772864, + "grad_norm": 0.6942827105522156, + "learning_rate": 0.0001, + "loss": 60.7314, + "step": 12710 + }, + { + "epoch": 20.48370030735124, + "grad_norm": 1.107717752456665, + "learning_rate": 0.0001, + "loss": 59.1397, + "step": 12720 + }, + { + "epoch": 20.49982365092961, + "grad_norm": 2.6167991161346436, + "learning_rate": 0.0001, + "loss": 60.5604, + "step": 12730 + }, + { + "epoch": 20.515946994507985, + "grad_norm": 0.6041843891143799, + "learning_rate": 0.0001, + "loss": 60.6188, + "step": 12740 + }, + { + "epoch": 20.53207033808636, + "grad_norm": 0.5643273591995239, + "learning_rate": 0.0001, + "loss": 59.3172, + "step": 12750 + }, + { + "epoch": 20.548193681664735, + "grad_norm": 0.7615086436271667, + "learning_rate": 0.0001, + "loss": 60.9263, + "step": 12760 + }, + { + "epoch": 20.56431702524311, + "grad_norm": 0.692243754863739, + "learning_rate": 0.0001, + "loss": 58.3092, + "step": 12770 + }, + { + "epoch": 20.580440368821485, + "grad_norm": 0.6447681784629822, + "learning_rate": 0.0001, + "loss": 58.0169, + "step": 12780 + }, + { + "epoch": 20.59656371239986, + "grad_norm": 0.7043688297271729, + "learning_rate": 0.0001, + "loss": 58.8346, + "step": 12790 + }, + { + "epoch": 20.612687055978235, + "grad_norm": 0.8017414808273315, + "learning_rate": 0.0001, + "loss": 60.7451, + "step": 12800 + }, + { + "epoch": 20.62881039955661, + "grad_norm": 0.8983294367790222, + "learning_rate": 0.0001, + "loss": 59.6858, + "step": 12810 + }, + { + "epoch": 20.64493374313498, + "grad_norm": 1.0817819833755493, + "learning_rate": 0.0001, + "loss": 60.8232, + "step": 12820 + }, + { + "epoch": 20.661057086713356, + "grad_norm": 0.5739386081695557, + "learning_rate": 0.0001, + "loss": 60.7852, + "step": 12830 + }, + { + "epoch": 20.67718043029173, + "grad_norm": 1.007978916168213, + "learning_rate": 0.0001, + "loss": 61.7002, + "step": 12840 + }, + { + "epoch": 20.693303773870106, + "grad_norm": 0.7019670009613037, + "learning_rate": 0.0001, + "loss": 59.7211, + "step": 12850 + }, + { + "epoch": 20.70942711744848, + "grad_norm": 0.7891802191734314, + "learning_rate": 0.0001, + "loss": 61.4963, + "step": 12860 + }, + { + "epoch": 20.725550461026856, + "grad_norm": 0.6955318450927734, + "learning_rate": 0.0001, + "loss": 60.8528, + "step": 12870 + }, + { + "epoch": 20.74167380460523, + "grad_norm": 1.9988878965377808, + "learning_rate": 0.0001, + "loss": 58.7444, + "step": 12880 + }, + { + "epoch": 20.757797148183606, + "grad_norm": 0.7859933972358704, + "learning_rate": 0.0001, + "loss": 60.0696, + "step": 12890 + }, + { + "epoch": 20.77392049176198, + "grad_norm": 0.8253535628318787, + "learning_rate": 0.0001, + "loss": 61.3408, + "step": 12900 + }, + { + "epoch": 20.790043835340352, + "grad_norm": 0.6639575362205505, + "learning_rate": 0.0001, + "loss": 58.3044, + "step": 12910 + }, + { + "epoch": 20.806167178918727, + "grad_norm": 0.6754361391067505, + "learning_rate": 0.0001, + "loss": 59.8301, + "step": 12920 + }, + { + "epoch": 20.822290522497102, + "grad_norm": 0.7306725382804871, + "learning_rate": 0.0001, + "loss": 60.0021, + "step": 12930 + }, + { + "epoch": 20.838413866075477, + "grad_norm": 6.799670696258545, + "learning_rate": 0.0001, + "loss": 57.7934, + "step": 12940 + }, + { + "epoch": 20.854537209653852, + "grad_norm": 0.6870545744895935, + "learning_rate": 0.0001, + "loss": 59.2313, + "step": 12950 + }, + { + "epoch": 20.870660553232227, + "grad_norm": 0.6980645060539246, + "learning_rate": 0.0001, + "loss": 60.1903, + "step": 12960 + }, + { + "epoch": 20.886783896810602, + "grad_norm": 0.7006897926330566, + "learning_rate": 0.0001, + "loss": 60.9525, + "step": 12970 + }, + { + "epoch": 20.902907240388977, + "grad_norm": 0.7185907363891602, + "learning_rate": 0.0001, + "loss": 61.8833, + "step": 12980 + }, + { + "epoch": 20.91903058396735, + "grad_norm": 0.6029082536697388, + "learning_rate": 0.0001, + "loss": 60.3764, + "step": 12990 + }, + { + "epoch": 20.935153927545723, + "grad_norm": 0.8502999544143677, + "learning_rate": 0.0001, + "loss": 59.0051, + "step": 13000 + }, + { + "epoch": 20.9512772711241, + "grad_norm": 0.908899188041687, + "learning_rate": 0.0001, + "loss": 59.0677, + "step": 13010 + }, + { + "epoch": 20.967400614702473, + "grad_norm": 0.6983920335769653, + "learning_rate": 0.0001, + "loss": 60.8523, + "step": 13020 + }, + { + "epoch": 20.98352395828085, + "grad_norm": 0.9079043865203857, + "learning_rate": 0.0001, + "loss": 58.8014, + "step": 13030 + }, + { + "epoch": 20.999647301859223, + "grad_norm": 0.7139968872070312, + "learning_rate": 0.0001, + "loss": 59.3019, + "step": 13040 + }, + { + "epoch": 21.014511009220538, + "grad_norm": 0.7472447156906128, + "learning_rate": 0.0001, + "loss": 56.1178, + "step": 13050 + }, + { + "epoch": 21.030634352798913, + "grad_norm": 0.5645695328712463, + "learning_rate": 0.0001, + "loss": 60.5944, + "step": 13060 + }, + { + "epoch": 21.046757696377288, + "grad_norm": 0.6419529318809509, + "learning_rate": 0.0001, + "loss": 59.5792, + "step": 13070 + }, + { + "epoch": 21.06288103995566, + "grad_norm": 0.6508122682571411, + "learning_rate": 0.0001, + "loss": 58.995, + "step": 13080 + }, + { + "epoch": 21.079004383534034, + "grad_norm": 0.6605037450790405, + "learning_rate": 0.0001, + "loss": 60.9783, + "step": 13090 + }, + { + "epoch": 21.09512772711241, + "grad_norm": 0.6636376976966858, + "learning_rate": 0.0001, + "loss": 60.3516, + "step": 13100 + }, + { + "epoch": 21.111251070690784, + "grad_norm": 0.7671985030174255, + "learning_rate": 0.0001, + "loss": 61.8918, + "step": 13110 + }, + { + "epoch": 21.12737441426916, + "grad_norm": 0.859834611415863, + "learning_rate": 0.0001, + "loss": 61.0766, + "step": 13120 + }, + { + "epoch": 21.143497757847534, + "grad_norm": 0.7551945447921753, + "learning_rate": 0.0001, + "loss": 60.6674, + "step": 13130 + }, + { + "epoch": 21.15962110142591, + "grad_norm": 0.6225672960281372, + "learning_rate": 0.0001, + "loss": 60.4107, + "step": 13140 + }, + { + "epoch": 21.175744445004284, + "grad_norm": 0.6100388765335083, + "learning_rate": 0.0001, + "loss": 58.8803, + "step": 13150 + }, + { + "epoch": 21.19186778858266, + "grad_norm": 0.669247567653656, + "learning_rate": 0.0001, + "loss": 60.4534, + "step": 13160 + }, + { + "epoch": 21.20799113216103, + "grad_norm": 0.8899680972099304, + "learning_rate": 0.0001, + "loss": 59.3081, + "step": 13170 + }, + { + "epoch": 21.224114475739405, + "grad_norm": 0.6804965138435364, + "learning_rate": 0.0001, + "loss": 58.1406, + "step": 13180 + }, + { + "epoch": 21.24023781931778, + "grad_norm": 0.7442401051521301, + "learning_rate": 0.0001, + "loss": 61.9136, + "step": 13190 + }, + { + "epoch": 21.256361162896155, + "grad_norm": 0.6071797013282776, + "learning_rate": 0.0001, + "loss": 59.7979, + "step": 13200 + }, + { + "epoch": 21.27248450647453, + "grad_norm": 0.6128804683685303, + "learning_rate": 0.0001, + "loss": 58.802, + "step": 13210 + }, + { + "epoch": 21.288607850052905, + "grad_norm": 0.6430484652519226, + "learning_rate": 0.0001, + "loss": 59.9492, + "step": 13220 + }, + { + "epoch": 21.30473119363128, + "grad_norm": 0.6414241194725037, + "learning_rate": 0.0001, + "loss": 60.699, + "step": 13230 + }, + { + "epoch": 21.320854537209655, + "grad_norm": 0.7222250699996948, + "learning_rate": 0.0001, + "loss": 58.5909, + "step": 13240 + }, + { + "epoch": 21.33697788078803, + "grad_norm": 1.0913405418395996, + "learning_rate": 0.0001, + "loss": 59.9657, + "step": 13250 + }, + { + "epoch": 21.3531012243664, + "grad_norm": 0.8245854377746582, + "learning_rate": 0.0001, + "loss": 60.2276, + "step": 13260 + }, + { + "epoch": 21.369224567944777, + "grad_norm": 0.7204747200012207, + "learning_rate": 0.0001, + "loss": 60.1854, + "step": 13270 + }, + { + "epoch": 21.38534791152315, + "grad_norm": 0.5877585411071777, + "learning_rate": 0.0001, + "loss": 60.8509, + "step": 13280 + }, + { + "epoch": 21.401471255101526, + "grad_norm": 0.906605064868927, + "learning_rate": 0.0001, + "loss": 57.9792, + "step": 13290 + }, + { + "epoch": 21.4175945986799, + "grad_norm": 0.910301923751831, + "learning_rate": 0.0001, + "loss": 58.986, + "step": 13300 + }, + { + "epoch": 21.433717942258276, + "grad_norm": 0.7098278403282166, + "learning_rate": 0.0001, + "loss": 60.6727, + "step": 13310 + }, + { + "epoch": 21.44984128583665, + "grad_norm": 0.842569887638092, + "learning_rate": 0.0001, + "loss": 59.3789, + "step": 13320 + }, + { + "epoch": 21.465964629415026, + "grad_norm": 0.7320014834403992, + "learning_rate": 0.0001, + "loss": 60.7055, + "step": 13330 + }, + { + "epoch": 21.482087972993398, + "grad_norm": 0.7192698121070862, + "learning_rate": 0.0001, + "loss": 59.7494, + "step": 13340 + }, + { + "epoch": 21.498211316571773, + "grad_norm": 0.6317388415336609, + "learning_rate": 0.0001, + "loss": 59.2083, + "step": 13350 + }, + { + "epoch": 21.514334660150148, + "grad_norm": 0.7012689709663391, + "learning_rate": 0.0001, + "loss": 58.6733, + "step": 13360 + }, + { + "epoch": 21.530458003728523, + "grad_norm": 0.8148412704467773, + "learning_rate": 0.0001, + "loss": 60.6708, + "step": 13370 + }, + { + "epoch": 21.546581347306898, + "grad_norm": 0.6740885376930237, + "learning_rate": 0.0001, + "loss": 60.2317, + "step": 13380 + }, + { + "epoch": 21.562704690885273, + "grad_norm": 0.6835291981697083, + "learning_rate": 0.0001, + "loss": 60.1807, + "step": 13390 + }, + { + "epoch": 21.578828034463648, + "grad_norm": 0.9604494571685791, + "learning_rate": 0.0001, + "loss": 60.811, + "step": 13400 + }, + { + "epoch": 21.594951378042023, + "grad_norm": 0.765917956829071, + "learning_rate": 0.0001, + "loss": 60.4539, + "step": 13410 + }, + { + "epoch": 21.611074721620398, + "grad_norm": 0.7830843329429626, + "learning_rate": 0.0001, + "loss": 59.2483, + "step": 13420 + }, + { + "epoch": 21.62719806519877, + "grad_norm": 0.6538785099983215, + "learning_rate": 0.0001, + "loss": 60.4062, + "step": 13430 + }, + { + "epoch": 21.643321408777144, + "grad_norm": 0.7341951727867126, + "learning_rate": 0.0001, + "loss": 59.3654, + "step": 13440 + }, + { + "epoch": 21.65944475235552, + "grad_norm": 0.684467077255249, + "learning_rate": 0.0001, + "loss": 58.9634, + "step": 13450 + }, + { + "epoch": 21.675568095933894, + "grad_norm": 0.6942781805992126, + "learning_rate": 0.0001, + "loss": 60.3677, + "step": 13460 + }, + { + "epoch": 21.69169143951227, + "grad_norm": 0.7780037522315979, + "learning_rate": 0.0001, + "loss": 61.6705, + "step": 13470 + }, + { + "epoch": 21.707814783090644, + "grad_norm": 0.8363171219825745, + "learning_rate": 0.0001, + "loss": 59.7802, + "step": 13480 + }, + { + "epoch": 21.72393812666902, + "grad_norm": 0.7392274737358093, + "learning_rate": 0.0001, + "loss": 59.8264, + "step": 13490 + }, + { + "epoch": 21.740061470247394, + "grad_norm": 0.6051466464996338, + "learning_rate": 0.0001, + "loss": 59.6685, + "step": 13500 + }, + { + "epoch": 21.75618481382577, + "grad_norm": 0.7772730588912964, + "learning_rate": 0.0001, + "loss": 58.261, + "step": 13510 + }, + { + "epoch": 21.77230815740414, + "grad_norm": 0.7481459379196167, + "learning_rate": 0.0001, + "loss": 59.3212, + "step": 13520 + }, + { + "epoch": 21.788431500982515, + "grad_norm": 0.8020729422569275, + "learning_rate": 0.0001, + "loss": 59.0113, + "step": 13530 + }, + { + "epoch": 21.80455484456089, + "grad_norm": 0.8504058122634888, + "learning_rate": 0.0001, + "loss": 59.7243, + "step": 13540 + }, + { + "epoch": 21.820678188139265, + "grad_norm": 0.7256322503089905, + "learning_rate": 0.0001, + "loss": 60.7478, + "step": 13550 + }, + { + "epoch": 21.83680153171764, + "grad_norm": 0.6632773876190186, + "learning_rate": 0.0001, + "loss": 59.2331, + "step": 13560 + }, + { + "epoch": 21.852924875296015, + "grad_norm": 0.6225188970565796, + "learning_rate": 0.0001, + "loss": 59.8263, + "step": 13570 + }, + { + "epoch": 21.86904821887439, + "grad_norm": 0.6302409768104553, + "learning_rate": 0.0001, + "loss": 60.8574, + "step": 13580 + }, + { + "epoch": 21.885171562452765, + "grad_norm": 0.7654368877410889, + "learning_rate": 0.0001, + "loss": 59.8633, + "step": 13590 + }, + { + "epoch": 21.90129490603114, + "grad_norm": 1.227220892906189, + "learning_rate": 0.0001, + "loss": 60.7098, + "step": 13600 + }, + { + "epoch": 21.91741824960951, + "grad_norm": 0.6679421067237854, + "learning_rate": 0.0001, + "loss": 59.0663, + "step": 13610 + }, + { + "epoch": 21.933541593187886, + "grad_norm": 0.8533850312232971, + "learning_rate": 0.0001, + "loss": 60.8597, + "step": 13620 + }, + { + "epoch": 21.94966493676626, + "grad_norm": 0.6417121887207031, + "learning_rate": 0.0001, + "loss": 60.1464, + "step": 13630 + }, + { + "epoch": 21.965788280344636, + "grad_norm": 0.679053544998169, + "learning_rate": 0.0001, + "loss": 60.5066, + "step": 13640 + }, + { + "epoch": 21.98191162392301, + "grad_norm": 0.6914607286453247, + "learning_rate": 0.0001, + "loss": 59.6815, + "step": 13650 + }, + { + "epoch": 21.998034967501386, + "grad_norm": 0.5661413669586182, + "learning_rate": 0.0001, + "loss": 61.6076, + "step": 13660 + }, + { + "epoch": 22.0128986748627, + "grad_norm": 1.4387016296386719, + "learning_rate": 0.0001, + "loss": 55.0652, + "step": 13670 + }, + { + "epoch": 22.029022018441076, + "grad_norm": 0.7774295210838318, + "learning_rate": 0.0001, + "loss": 61.4395, + "step": 13680 + }, + { + "epoch": 22.045145362019447, + "grad_norm": 0.7438349723815918, + "learning_rate": 0.0001, + "loss": 60.998, + "step": 13690 + }, + { + "epoch": 22.061268705597822, + "grad_norm": 0.7506164312362671, + "learning_rate": 0.0001, + "loss": 60.6929, + "step": 13700 + }, + { + "epoch": 22.077392049176197, + "grad_norm": 0.6262985467910767, + "learning_rate": 0.0001, + "loss": 59.4921, + "step": 13710 + }, + { + "epoch": 22.093515392754572, + "grad_norm": 1.826015591621399, + "learning_rate": 0.0001, + "loss": 61.3791, + "step": 13720 + }, + { + "epoch": 22.109638736332947, + "grad_norm": 0.6410622596740723, + "learning_rate": 0.0001, + "loss": 62.5485, + "step": 13730 + }, + { + "epoch": 22.125762079911322, + "grad_norm": 0.7042442560195923, + "learning_rate": 0.0001, + "loss": 60.0654, + "step": 13740 + }, + { + "epoch": 22.141885423489697, + "grad_norm": 0.741142988204956, + "learning_rate": 0.0001, + "loss": 59.9831, + "step": 13750 + }, + { + "epoch": 22.158008767068072, + "grad_norm": 0.836445152759552, + "learning_rate": 0.0001, + "loss": 59.6805, + "step": 13760 + }, + { + "epoch": 22.174132110646447, + "grad_norm": 0.8353415727615356, + "learning_rate": 0.0001, + "loss": 60.1912, + "step": 13770 + }, + { + "epoch": 22.19025545422482, + "grad_norm": 0.6416126489639282, + "learning_rate": 0.0001, + "loss": 59.5518, + "step": 13780 + }, + { + "epoch": 22.206378797803193, + "grad_norm": 0.8702544569969177, + "learning_rate": 0.0001, + "loss": 57.7428, + "step": 13790 + }, + { + "epoch": 22.222502141381568, + "grad_norm": 0.6681593656539917, + "learning_rate": 0.0001, + "loss": 59.9499, + "step": 13800 + }, + { + "epoch": 22.238625484959943, + "grad_norm": 0.6185410022735596, + "learning_rate": 0.0001, + "loss": 59.8062, + "step": 13810 + }, + { + "epoch": 22.254748828538318, + "grad_norm": 0.7695328593254089, + "learning_rate": 0.0001, + "loss": 61.3512, + "step": 13820 + }, + { + "epoch": 22.270872172116693, + "grad_norm": 0.6550581455230713, + "learning_rate": 0.0001, + "loss": 59.3379, + "step": 13830 + }, + { + "epoch": 22.286995515695068, + "grad_norm": 0.7764340043067932, + "learning_rate": 0.0001, + "loss": 59.984, + "step": 13840 + }, + { + "epoch": 22.303118859273443, + "grad_norm": 0.6377913355827332, + "learning_rate": 0.0001, + "loss": 59.1862, + "step": 13850 + }, + { + "epoch": 22.319242202851818, + "grad_norm": 0.7837334275245667, + "learning_rate": 0.0001, + "loss": 60.8087, + "step": 13860 + }, + { + "epoch": 22.33536554643019, + "grad_norm": 0.7966547012329102, + "learning_rate": 0.0001, + "loss": 59.9109, + "step": 13870 + }, + { + "epoch": 22.351488890008564, + "grad_norm": 0.6303540468215942, + "learning_rate": 0.0001, + "loss": 61.0486, + "step": 13880 + }, + { + "epoch": 22.36761223358694, + "grad_norm": 0.6628289818763733, + "learning_rate": 0.0001, + "loss": 61.7723, + "step": 13890 + }, + { + "epoch": 22.383735577165314, + "grad_norm": 1.2156575918197632, + "learning_rate": 0.0001, + "loss": 58.0992, + "step": 13900 + }, + { + "epoch": 22.39985892074369, + "grad_norm": 1.0548591613769531, + "learning_rate": 0.0001, + "loss": 58.9139, + "step": 13910 + }, + { + "epoch": 22.415982264322064, + "grad_norm": 0.6389120817184448, + "learning_rate": 0.0001, + "loss": 59.1494, + "step": 13920 + }, + { + "epoch": 22.43210560790044, + "grad_norm": 0.6384537220001221, + "learning_rate": 0.0001, + "loss": 59.7135, + "step": 13930 + }, + { + "epoch": 22.448228951478814, + "grad_norm": 0.7434266209602356, + "learning_rate": 0.0001, + "loss": 60.7941, + "step": 13940 + }, + { + "epoch": 22.46435229505719, + "grad_norm": 0.7207310199737549, + "learning_rate": 0.0001, + "loss": 57.9466, + "step": 13950 + }, + { + "epoch": 22.48047563863556, + "grad_norm": 0.9680810570716858, + "learning_rate": 0.0001, + "loss": 57.5014, + "step": 13960 + }, + { + "epoch": 22.496598982213936, + "grad_norm": 0.5354718565940857, + "learning_rate": 0.0001, + "loss": 59.8771, + "step": 13970 + }, + { + "epoch": 22.51272232579231, + "grad_norm": 1.1624388694763184, + "learning_rate": 0.0001, + "loss": 60.2305, + "step": 13980 + }, + { + "epoch": 22.528845669370686, + "grad_norm": 0.6650974154472351, + "learning_rate": 0.0001, + "loss": 58.1649, + "step": 13990 + }, + { + "epoch": 22.54496901294906, + "grad_norm": 0.6151170134544373, + "learning_rate": 0.0001, + "loss": 60.4392, + "step": 14000 + }, + { + "epoch": 22.561092356527435, + "grad_norm": 0.6685694456100464, + "learning_rate": 0.0001, + "loss": 60.7055, + "step": 14010 + }, + { + "epoch": 22.57721570010581, + "grad_norm": 0.8092939257621765, + "learning_rate": 0.0001, + "loss": 57.8347, + "step": 14020 + }, + { + "epoch": 22.593339043684185, + "grad_norm": 0.7520709037780762, + "learning_rate": 0.0001, + "loss": 60.359, + "step": 14030 + }, + { + "epoch": 22.609462387262557, + "grad_norm": 0.8913780450820923, + "learning_rate": 0.0001, + "loss": 59.0833, + "step": 14040 + }, + { + "epoch": 22.625585730840932, + "grad_norm": 0.5508324503898621, + "learning_rate": 0.0001, + "loss": 60.8168, + "step": 14050 + }, + { + "epoch": 22.641709074419307, + "grad_norm": 0.7124258279800415, + "learning_rate": 0.0001, + "loss": 60.9433, + "step": 14060 + }, + { + "epoch": 22.65783241799768, + "grad_norm": 0.7286089062690735, + "learning_rate": 0.0001, + "loss": 59.8127, + "step": 14070 + }, + { + "epoch": 22.673955761576057, + "grad_norm": 0.8104858994483948, + "learning_rate": 0.0001, + "loss": 59.764, + "step": 14080 + }, + { + "epoch": 22.69007910515443, + "grad_norm": 0.7618247270584106, + "learning_rate": 0.0001, + "loss": 59.5563, + "step": 14090 + }, + { + "epoch": 22.706202448732807, + "grad_norm": 0.8512730598449707, + "learning_rate": 0.0001, + "loss": 59.3122, + "step": 14100 + }, + { + "epoch": 22.72232579231118, + "grad_norm": 0.6115924715995789, + "learning_rate": 0.0001, + "loss": 61.2874, + "step": 14110 + }, + { + "epoch": 22.738449135889557, + "grad_norm": 0.6883788704872131, + "learning_rate": 0.0001, + "loss": 59.427, + "step": 14120 + }, + { + "epoch": 22.754572479467928, + "grad_norm": 0.6240411400794983, + "learning_rate": 0.0001, + "loss": 58.9808, + "step": 14130 + }, + { + "epoch": 22.770695823046303, + "grad_norm": 0.6807436347007751, + "learning_rate": 0.0001, + "loss": 58.9609, + "step": 14140 + }, + { + "epoch": 22.786819166624678, + "grad_norm": 0.6661691665649414, + "learning_rate": 0.0001, + "loss": 59.8573, + "step": 14150 + }, + { + "epoch": 22.802942510203053, + "grad_norm": 0.7038732171058655, + "learning_rate": 0.0001, + "loss": 59.0114, + "step": 14160 + }, + { + "epoch": 22.819065853781428, + "grad_norm": 0.7736789584159851, + "learning_rate": 0.0001, + "loss": 59.6709, + "step": 14170 + }, + { + "epoch": 22.835189197359803, + "grad_norm": 0.6095895767211914, + "learning_rate": 0.0001, + "loss": 60.4468, + "step": 14180 + }, + { + "epoch": 22.851312540938178, + "grad_norm": 0.6531018018722534, + "learning_rate": 0.0001, + "loss": 61.4419, + "step": 14190 + }, + { + "epoch": 22.867435884516553, + "grad_norm": 0.7522090077400208, + "learning_rate": 0.0001, + "loss": 57.6574, + "step": 14200 + }, + { + "epoch": 22.883559228094928, + "grad_norm": 0.8028358817100525, + "learning_rate": 0.0001, + "loss": 59.0353, + "step": 14210 + }, + { + "epoch": 22.8996825716733, + "grad_norm": 0.7619246244430542, + "learning_rate": 0.0001, + "loss": 59.3586, + "step": 14220 + }, + { + "epoch": 22.915805915251674, + "grad_norm": 0.8312122821807861, + "learning_rate": 0.0001, + "loss": 60.8772, + "step": 14230 + }, + { + "epoch": 22.93192925883005, + "grad_norm": 0.6536790132522583, + "learning_rate": 0.0001, + "loss": 60.5941, + "step": 14240 + }, + { + "epoch": 22.948052602408424, + "grad_norm": 0.6973451972007751, + "learning_rate": 0.0001, + "loss": 59.5332, + "step": 14250 + }, + { + "epoch": 22.9641759459868, + "grad_norm": 0.6776084899902344, + "learning_rate": 0.0001, + "loss": 60.2337, + "step": 14260 + }, + { + "epoch": 22.980299289565174, + "grad_norm": 0.5728079080581665, + "learning_rate": 0.0001, + "loss": 61.0065, + "step": 14270 + }, + { + "epoch": 22.99642263314355, + "grad_norm": 0.798130452632904, + "learning_rate": 0.0001, + "loss": 59.7552, + "step": 14280 + }, + { + "epoch": 23.011286340504864, + "grad_norm": 0.6825193166732788, + "learning_rate": 0.0001, + "loss": 54.8593, + "step": 14290 + }, + { + "epoch": 23.02740968408324, + "grad_norm": 0.794450044631958, + "learning_rate": 0.0001, + "loss": 58.1472, + "step": 14300 + }, + { + "epoch": 23.04353302766161, + "grad_norm": 0.8178662657737732, + "learning_rate": 0.0001, + "loss": 58.8868, + "step": 14310 + }, + { + "epoch": 23.059656371239985, + "grad_norm": 0.6562325358390808, + "learning_rate": 0.0001, + "loss": 59.2406, + "step": 14320 + }, + { + "epoch": 23.07577971481836, + "grad_norm": 0.7282053828239441, + "learning_rate": 0.0001, + "loss": 58.1195, + "step": 14330 + }, + { + "epoch": 23.091903058396735, + "grad_norm": 0.6804936528205872, + "learning_rate": 0.0001, + "loss": 60.3755, + "step": 14340 + }, + { + "epoch": 23.10802640197511, + "grad_norm": 0.5851950645446777, + "learning_rate": 0.0001, + "loss": 59.2353, + "step": 14350 + }, + { + "epoch": 23.124149745553485, + "grad_norm": 0.5784706473350525, + "learning_rate": 0.0001, + "loss": 59.351, + "step": 14360 + }, + { + "epoch": 23.14027308913186, + "grad_norm": 0.7353758215904236, + "learning_rate": 0.0001, + "loss": 60.7708, + "step": 14370 + }, + { + "epoch": 23.156396432710235, + "grad_norm": 0.7761120796203613, + "learning_rate": 0.0001, + "loss": 59.6935, + "step": 14380 + }, + { + "epoch": 23.172519776288606, + "grad_norm": 0.6392366290092468, + "learning_rate": 0.0001, + "loss": 61.0263, + "step": 14390 + }, + { + "epoch": 23.18864311986698, + "grad_norm": 0.5722920894622803, + "learning_rate": 0.0001, + "loss": 59.4247, + "step": 14400 + }, + { + "epoch": 23.204766463445356, + "grad_norm": 0.6938275694847107, + "learning_rate": 0.0001, + "loss": 61.0933, + "step": 14410 + }, + { + "epoch": 23.22088980702373, + "grad_norm": 0.9164982438087463, + "learning_rate": 0.0001, + "loss": 58.0045, + "step": 14420 + }, + { + "epoch": 23.237013150602106, + "grad_norm": 0.6641019582748413, + "learning_rate": 0.0001, + "loss": 62.2162, + "step": 14430 + }, + { + "epoch": 23.25313649418048, + "grad_norm": 0.6775721311569214, + "learning_rate": 0.0001, + "loss": 60.2123, + "step": 14440 + }, + { + "epoch": 23.269259837758856, + "grad_norm": 0.6306013464927673, + "learning_rate": 0.0001, + "loss": 60.0638, + "step": 14450 + }, + { + "epoch": 23.28538318133723, + "grad_norm": 0.7770274877548218, + "learning_rate": 0.0001, + "loss": 60.87, + "step": 14460 + }, + { + "epoch": 23.301506524915606, + "grad_norm": 0.6893128752708435, + "learning_rate": 0.0001, + "loss": 59.2247, + "step": 14470 + }, + { + "epoch": 23.317629868493977, + "grad_norm": 0.7028453946113586, + "learning_rate": 0.0001, + "loss": 60.2405, + "step": 14480 + }, + { + "epoch": 23.333753212072352, + "grad_norm": 0.6184173822402954, + "learning_rate": 0.0001, + "loss": 58.5765, + "step": 14490 + }, + { + "epoch": 23.349876555650727, + "grad_norm": 0.793502151966095, + "learning_rate": 0.0001, + "loss": 57.5978, + "step": 14500 + }, + { + "epoch": 23.365999899229102, + "grad_norm": 0.660885751247406, + "learning_rate": 0.0001, + "loss": 60.5535, + "step": 14510 + }, + { + "epoch": 23.382123242807477, + "grad_norm": 0.9039516448974609, + "learning_rate": 0.0001, + "loss": 58.9432, + "step": 14520 + }, + { + "epoch": 23.398246586385852, + "grad_norm": 0.7348603010177612, + "learning_rate": 0.0001, + "loss": 58.3708, + "step": 14530 + }, + { + "epoch": 23.414369929964227, + "grad_norm": 0.5571108460426331, + "learning_rate": 0.0001, + "loss": 60.2059, + "step": 14540 + }, + { + "epoch": 23.430493273542602, + "grad_norm": 0.77266526222229, + "learning_rate": 0.0001, + "loss": 59.9602, + "step": 14550 + }, + { + "epoch": 23.446616617120977, + "grad_norm": 0.7564616203308105, + "learning_rate": 0.0001, + "loss": 59.0114, + "step": 14560 + }, + { + "epoch": 23.46273996069935, + "grad_norm": 1.0693135261535645, + "learning_rate": 0.0001, + "loss": 60.6075, + "step": 14570 + }, + { + "epoch": 23.478863304277724, + "grad_norm": 0.7800086140632629, + "learning_rate": 0.0001, + "loss": 60.6608, + "step": 14580 + }, + { + "epoch": 23.4949866478561, + "grad_norm": 0.8552407026290894, + "learning_rate": 0.0001, + "loss": 60.2378, + "step": 14590 + }, + { + "epoch": 23.511109991434473, + "grad_norm": 0.7270957231521606, + "learning_rate": 0.0001, + "loss": 60.8936, + "step": 14600 + }, + { + "epoch": 23.52723333501285, + "grad_norm": 0.5652869343757629, + "learning_rate": 0.0001, + "loss": 59.9882, + "step": 14610 + }, + { + "epoch": 23.543356678591223, + "grad_norm": 0.5465036034584045, + "learning_rate": 0.0001, + "loss": 57.4725, + "step": 14620 + }, + { + "epoch": 23.5594800221696, + "grad_norm": 0.6404461860656738, + "learning_rate": 0.0001, + "loss": 60.3962, + "step": 14630 + }, + { + "epoch": 23.575603365747973, + "grad_norm": 0.7860730886459351, + "learning_rate": 0.0001, + "loss": 60.3245, + "step": 14640 + }, + { + "epoch": 23.591726709326345, + "grad_norm": 0.6626171469688416, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 14650 + }, + { + "epoch": 23.60785005290472, + "grad_norm": 1.5377638339996338, + "learning_rate": 0.0001, + "loss": 58.9615, + "step": 14660 + }, + { + "epoch": 23.623973396483095, + "grad_norm": 0.6656123995780945, + "learning_rate": 0.0001, + "loss": 60.7947, + "step": 14670 + }, + { + "epoch": 23.64009674006147, + "grad_norm": 0.5155975818634033, + "learning_rate": 0.0001, + "loss": 57.7742, + "step": 14680 + }, + { + "epoch": 23.656220083639845, + "grad_norm": 0.6278973817825317, + "learning_rate": 0.0001, + "loss": 58.6188, + "step": 14690 + }, + { + "epoch": 23.67234342721822, + "grad_norm": 0.7261143326759338, + "learning_rate": 0.0001, + "loss": 56.8495, + "step": 14700 + }, + { + "epoch": 23.688466770796595, + "grad_norm": 0.8105781078338623, + "learning_rate": 0.0001, + "loss": 57.3139, + "step": 14710 + }, + { + "epoch": 23.70459011437497, + "grad_norm": 0.7012073993682861, + "learning_rate": 0.0001, + "loss": 61.1186, + "step": 14720 + }, + { + "epoch": 23.720713457953345, + "grad_norm": 0.6352536082267761, + "learning_rate": 0.0001, + "loss": 59.0136, + "step": 14730 + }, + { + "epoch": 23.736836801531716, + "grad_norm": 0.7279717326164246, + "learning_rate": 0.0001, + "loss": 58.7408, + "step": 14740 + }, + { + "epoch": 23.75296014511009, + "grad_norm": 0.6714015603065491, + "learning_rate": 0.0001, + "loss": 58.6431, + "step": 14750 + }, + { + "epoch": 23.769083488688466, + "grad_norm": 1.3876261711120605, + "learning_rate": 0.0001, + "loss": 59.7586, + "step": 14760 + }, + { + "epoch": 23.78520683226684, + "grad_norm": 0.7192941904067993, + "learning_rate": 0.0001, + "loss": 58.8487, + "step": 14770 + }, + { + "epoch": 23.801330175845216, + "grad_norm": 0.7333327531814575, + "learning_rate": 0.0001, + "loss": 60.5608, + "step": 14780 + }, + { + "epoch": 23.81745351942359, + "grad_norm": 0.8186061382293701, + "learning_rate": 0.0001, + "loss": 59.4232, + "step": 14790 + }, + { + "epoch": 23.833576863001966, + "grad_norm": 1.037039041519165, + "learning_rate": 0.0001, + "loss": 58.2384, + "step": 14800 + }, + { + "epoch": 23.84970020658034, + "grad_norm": 0.6739209890365601, + "learning_rate": 0.0001, + "loss": 57.7256, + "step": 14810 + }, + { + "epoch": 23.865823550158716, + "grad_norm": 0.6515845656394958, + "learning_rate": 0.0001, + "loss": 61.0039, + "step": 14820 + }, + { + "epoch": 23.881946893737087, + "grad_norm": 0.559317946434021, + "learning_rate": 0.0001, + "loss": 60.4458, + "step": 14830 + }, + { + "epoch": 23.898070237315462, + "grad_norm": 0.663934588432312, + "learning_rate": 0.0001, + "loss": 59.1646, + "step": 14840 + }, + { + "epoch": 23.914193580893837, + "grad_norm": 0.6409069299697876, + "learning_rate": 0.0001, + "loss": 58.9508, + "step": 14850 + }, + { + "epoch": 23.930316924472212, + "grad_norm": 0.7624269127845764, + "learning_rate": 0.0001, + "loss": 60.7256, + "step": 14860 + }, + { + "epoch": 23.946440268050587, + "grad_norm": 0.6949140429496765, + "learning_rate": 0.0001, + "loss": 58.8987, + "step": 14870 + }, + { + "epoch": 23.962563611628962, + "grad_norm": 0.714063823223114, + "learning_rate": 0.0001, + "loss": 59.255, + "step": 14880 + }, + { + "epoch": 23.978686955207337, + "grad_norm": 0.6570878028869629, + "learning_rate": 0.0001, + "loss": 59.2637, + "step": 14890 + }, + { + "epoch": 23.994810298785712, + "grad_norm": 0.7898263335227966, + "learning_rate": 0.0001, + "loss": 61.0218, + "step": 14900 + }, + { + "epoch": 24.009674006147026, + "grad_norm": 0.6673877239227295, + "learning_rate": 0.0001, + "loss": 53.8562, + "step": 14910 + }, + { + "epoch": 24.025797349725398, + "grad_norm": 0.5548285245895386, + "learning_rate": 0.0001, + "loss": 59.4445, + "step": 14920 + }, + { + "epoch": 24.041920693303773, + "grad_norm": 0.6273561716079712, + "learning_rate": 0.0001, + "loss": 59.3292, + "step": 14930 + }, + { + "epoch": 24.058044036882148, + "grad_norm": 0.7050267457962036, + "learning_rate": 0.0001, + "loss": 60.4896, + "step": 14940 + }, + { + "epoch": 24.074167380460523, + "grad_norm": 0.7030066251754761, + "learning_rate": 0.0001, + "loss": 59.1622, + "step": 14950 + }, + { + "epoch": 24.090290724038898, + "grad_norm": 0.8014484643936157, + "learning_rate": 0.0001, + "loss": 59.746, + "step": 14960 + }, + { + "epoch": 24.106414067617273, + "grad_norm": 0.869323194026947, + "learning_rate": 0.0001, + "loss": 60.2156, + "step": 14970 + }, + { + "epoch": 24.122537411195648, + "grad_norm": 0.7157219648361206, + "learning_rate": 0.0001, + "loss": 58.7229, + "step": 14980 + }, + { + "epoch": 24.138660754774023, + "grad_norm": 0.6318342685699463, + "learning_rate": 0.0001, + "loss": 61.1832, + "step": 14990 + }, + { + "epoch": 24.154784098352394, + "grad_norm": 0.8689974546432495, + "learning_rate": 0.0001, + "loss": 58.8083, + "step": 15000 + }, + { + "epoch": 24.17090744193077, + "grad_norm": 1.1719586849212646, + "learning_rate": 0.0001, + "loss": 57.745, + "step": 15010 + }, + { + "epoch": 24.187030785509144, + "grad_norm": 0.676295280456543, + "learning_rate": 0.0001, + "loss": 60.6236, + "step": 15020 + }, + { + "epoch": 24.20315412908752, + "grad_norm": 0.6745544672012329, + "learning_rate": 0.0001, + "loss": 59.494, + "step": 15030 + }, + { + "epoch": 24.219277472665894, + "grad_norm": 0.6221221685409546, + "learning_rate": 0.0001, + "loss": 58.894, + "step": 15040 + }, + { + "epoch": 24.23540081624427, + "grad_norm": 0.7013781070709229, + "learning_rate": 0.0001, + "loss": 59.935, + "step": 15050 + }, + { + "epoch": 24.251524159822644, + "grad_norm": 0.6094849705696106, + "learning_rate": 0.0001, + "loss": 59.8276, + "step": 15060 + }, + { + "epoch": 24.26764750340102, + "grad_norm": 0.6969972848892212, + "learning_rate": 0.0001, + "loss": 59.7568, + "step": 15070 + }, + { + "epoch": 24.283770846979394, + "grad_norm": 0.7223725318908691, + "learning_rate": 0.0001, + "loss": 57.9065, + "step": 15080 + }, + { + "epoch": 24.299894190557765, + "grad_norm": 0.7008336186408997, + "learning_rate": 0.0001, + "loss": 59.5206, + "step": 15090 + }, + { + "epoch": 24.31601753413614, + "grad_norm": 0.6490783095359802, + "learning_rate": 0.0001, + "loss": 59.4457, + "step": 15100 + }, + { + "epoch": 24.332140877714515, + "grad_norm": 0.6096290349960327, + "learning_rate": 0.0001, + "loss": 60.8434, + "step": 15110 + }, + { + "epoch": 24.34826422129289, + "grad_norm": 0.771045446395874, + "learning_rate": 0.0001, + "loss": 60.3167, + "step": 15120 + }, + { + "epoch": 24.364387564871265, + "grad_norm": 0.7178729772567749, + "learning_rate": 0.0001, + "loss": 57.6667, + "step": 15130 + }, + { + "epoch": 24.38051090844964, + "grad_norm": 0.6484465003013611, + "learning_rate": 0.0001, + "loss": 60.1199, + "step": 15140 + }, + { + "epoch": 24.396634252028015, + "grad_norm": 1.0030865669250488, + "learning_rate": 0.0001, + "loss": 57.9288, + "step": 15150 + }, + { + "epoch": 24.41275759560639, + "grad_norm": 0.8297843933105469, + "learning_rate": 0.0001, + "loss": 61.0941, + "step": 15160 + }, + { + "epoch": 24.428880939184765, + "grad_norm": 0.9115694165229797, + "learning_rate": 0.0001, + "loss": 58.9574, + "step": 15170 + }, + { + "epoch": 24.445004282763136, + "grad_norm": 0.7758508920669556, + "learning_rate": 0.0001, + "loss": 58.8852, + "step": 15180 + }, + { + "epoch": 24.46112762634151, + "grad_norm": 0.5539108514785767, + "learning_rate": 0.0001, + "loss": 60.9549, + "step": 15190 + }, + { + "epoch": 24.477250969919886, + "grad_norm": 0.7292829751968384, + "learning_rate": 0.0001, + "loss": 58.0818, + "step": 15200 + }, + { + "epoch": 24.49337431349826, + "grad_norm": 0.6253948211669922, + "learning_rate": 0.0001, + "loss": 59.656, + "step": 15210 + }, + { + "epoch": 24.509497657076636, + "grad_norm": 0.8054394125938416, + "learning_rate": 0.0001, + "loss": 60.934, + "step": 15220 + }, + { + "epoch": 24.52562100065501, + "grad_norm": 0.8183079361915588, + "learning_rate": 0.0001, + "loss": 59.5303, + "step": 15230 + }, + { + "epoch": 24.541744344233386, + "grad_norm": 0.6063419580459595, + "learning_rate": 0.0001, + "loss": 59.2852, + "step": 15240 + }, + { + "epoch": 24.55786768781176, + "grad_norm": 0.6372854113578796, + "learning_rate": 0.0001, + "loss": 58.9647, + "step": 15250 + }, + { + "epoch": 24.573991031390136, + "grad_norm": 0.6270446181297302, + "learning_rate": 0.0001, + "loss": 58.5641, + "step": 15260 + }, + { + "epoch": 24.590114374968508, + "grad_norm": 0.6229680776596069, + "learning_rate": 0.0001, + "loss": 58.4142, + "step": 15270 + }, + { + "epoch": 24.606237718546883, + "grad_norm": 0.6567183136940002, + "learning_rate": 0.0001, + "loss": 60.1162, + "step": 15280 + }, + { + "epoch": 24.622361062125258, + "grad_norm": 0.6975446343421936, + "learning_rate": 0.0001, + "loss": 58.7862, + "step": 15290 + }, + { + "epoch": 24.638484405703633, + "grad_norm": 0.6961022615432739, + "learning_rate": 0.0001, + "loss": 58.3247, + "step": 15300 + }, + { + "epoch": 24.654607749282007, + "grad_norm": 5.25744104385376, + "learning_rate": 0.0001, + "loss": 59.3863, + "step": 15310 + }, + { + "epoch": 24.670731092860382, + "grad_norm": 0.8439975380897522, + "learning_rate": 0.0001, + "loss": 59.7157, + "step": 15320 + }, + { + "epoch": 24.686854436438757, + "grad_norm": 0.596845269203186, + "learning_rate": 0.0001, + "loss": 60.0859, + "step": 15330 + }, + { + "epoch": 24.702977780017132, + "grad_norm": 0.6758428812026978, + "learning_rate": 0.0001, + "loss": 59.9305, + "step": 15340 + }, + { + "epoch": 24.719101123595507, + "grad_norm": 0.7418860197067261, + "learning_rate": 0.0001, + "loss": 59.0929, + "step": 15350 + }, + { + "epoch": 24.73522446717388, + "grad_norm": 0.6856047511100769, + "learning_rate": 0.0001, + "loss": 58.7201, + "step": 15360 + }, + { + "epoch": 24.751347810752254, + "grad_norm": 0.5327897071838379, + "learning_rate": 0.0001, + "loss": 59.5899, + "step": 15370 + }, + { + "epoch": 24.76747115433063, + "grad_norm": 0.7127685546875, + "learning_rate": 0.0001, + "loss": 59.7729, + "step": 15380 + }, + { + "epoch": 24.783594497909004, + "grad_norm": 1.8885949850082397, + "learning_rate": 0.0001, + "loss": 58.3813, + "step": 15390 + }, + { + "epoch": 24.79971784148738, + "grad_norm": 0.7997881174087524, + "learning_rate": 0.0001, + "loss": 61.9402, + "step": 15400 + }, + { + "epoch": 24.815841185065754, + "grad_norm": 0.9729613661766052, + "learning_rate": 0.0001, + "loss": 59.7961, + "step": 15410 + }, + { + "epoch": 24.83196452864413, + "grad_norm": 0.8450629115104675, + "learning_rate": 0.0001, + "loss": 59.5969, + "step": 15420 + }, + { + "epoch": 24.848087872222504, + "grad_norm": 0.720668613910675, + "learning_rate": 0.0001, + "loss": 58.187, + "step": 15430 + }, + { + "epoch": 24.864211215800875, + "grad_norm": 0.638506293296814, + "learning_rate": 0.0001, + "loss": 59.4599, + "step": 15440 + }, + { + "epoch": 24.88033455937925, + "grad_norm": 0.6158926486968994, + "learning_rate": 0.0001, + "loss": 61.3858, + "step": 15450 + }, + { + "epoch": 24.896457902957625, + "grad_norm": 0.8130396604537964, + "learning_rate": 0.0001, + "loss": 59.3795, + "step": 15460 + }, + { + "epoch": 24.912581246536, + "grad_norm": 0.7966095805168152, + "learning_rate": 0.0001, + "loss": 60.3319, + "step": 15470 + }, + { + "epoch": 24.928704590114375, + "grad_norm": 0.6240444779396057, + "learning_rate": 0.0001, + "loss": 59.0407, + "step": 15480 + }, + { + "epoch": 24.94482793369275, + "grad_norm": 0.7439318895339966, + "learning_rate": 0.0001, + "loss": 57.7692, + "step": 15490 + }, + { + "epoch": 24.960951277271125, + "grad_norm": 0.6942011713981628, + "learning_rate": 0.0001, + "loss": 56.8418, + "step": 15500 + }, + { + "epoch": 24.9770746208495, + "grad_norm": 0.6470242738723755, + "learning_rate": 0.0001, + "loss": 59.8361, + "step": 15510 + }, + { + "epoch": 24.993197964427875, + "grad_norm": 0.6935222148895264, + "learning_rate": 0.0001, + "loss": 59.5357, + "step": 15520 + }, + { + "epoch": 25.008061671789186, + "grad_norm": 0.8943103551864624, + "learning_rate": 0.0001, + "loss": 54.8185, + "step": 15530 + }, + { + "epoch": 25.02418501536756, + "grad_norm": 0.7637899518013, + "learning_rate": 0.0001, + "loss": 58.4746, + "step": 15540 + }, + { + "epoch": 25.040308358945936, + "grad_norm": 0.8261918425559998, + "learning_rate": 0.0001, + "loss": 57.3993, + "step": 15550 + }, + { + "epoch": 25.05643170252431, + "grad_norm": 0.6395642161369324, + "learning_rate": 0.0001, + "loss": 58.123, + "step": 15560 + }, + { + "epoch": 25.072555046102686, + "grad_norm": 0.7899443507194519, + "learning_rate": 0.0001, + "loss": 57.1487, + "step": 15570 + }, + { + "epoch": 25.08867838968106, + "grad_norm": 0.6202112436294556, + "learning_rate": 0.0001, + "loss": 59.4995, + "step": 15580 + }, + { + "epoch": 25.104801733259436, + "grad_norm": 0.7683939933776855, + "learning_rate": 0.0001, + "loss": 59.1155, + "step": 15590 + }, + { + "epoch": 25.12092507683781, + "grad_norm": 0.6531215906143188, + "learning_rate": 0.0001, + "loss": 61.9998, + "step": 15600 + }, + { + "epoch": 25.137048420416185, + "grad_norm": 0.6354383826255798, + "learning_rate": 0.0001, + "loss": 59.3861, + "step": 15610 + }, + { + "epoch": 25.153171763994557, + "grad_norm": 0.6507435441017151, + "learning_rate": 0.0001, + "loss": 59.939, + "step": 15620 + }, + { + "epoch": 25.169295107572932, + "grad_norm": 0.8014065623283386, + "learning_rate": 0.0001, + "loss": 59.4927, + "step": 15630 + }, + { + "epoch": 25.185418451151307, + "grad_norm": 0.53805011510849, + "learning_rate": 0.0001, + "loss": 58.526, + "step": 15640 + }, + { + "epoch": 25.201541794729682, + "grad_norm": 4.835650444030762, + "learning_rate": 0.0001, + "loss": 58.6711, + "step": 15650 + }, + { + "epoch": 25.217665138308057, + "grad_norm": 0.723210871219635, + "learning_rate": 0.0001, + "loss": 60.3953, + "step": 15660 + }, + { + "epoch": 25.23378848188643, + "grad_norm": 0.7176834940910339, + "learning_rate": 0.0001, + "loss": 58.8576, + "step": 15670 + }, + { + "epoch": 25.249911825464807, + "grad_norm": 0.8280025124549866, + "learning_rate": 0.0001, + "loss": 58.7146, + "step": 15680 + }, + { + "epoch": 25.26603516904318, + "grad_norm": 0.6296973824501038, + "learning_rate": 0.0001, + "loss": 60.7341, + "step": 15690 + }, + { + "epoch": 25.282158512621557, + "grad_norm": 1.3184518814086914, + "learning_rate": 0.0001, + "loss": 60.6906, + "step": 15700 + }, + { + "epoch": 25.298281856199928, + "grad_norm": 1.0408564805984497, + "learning_rate": 0.0001, + "loss": 57.4703, + "step": 15710 + }, + { + "epoch": 25.314405199778303, + "grad_norm": 0.7140268683433533, + "learning_rate": 0.0001, + "loss": 59.8452, + "step": 15720 + }, + { + "epoch": 25.330528543356678, + "grad_norm": 0.7822073698043823, + "learning_rate": 0.0001, + "loss": 60.4394, + "step": 15730 + }, + { + "epoch": 25.346651886935053, + "grad_norm": 0.6579483151435852, + "learning_rate": 0.0001, + "loss": 59.5358, + "step": 15740 + }, + { + "epoch": 25.362775230513428, + "grad_norm": 0.6936199069023132, + "learning_rate": 0.0001, + "loss": 60.2117, + "step": 15750 + }, + { + "epoch": 25.378898574091803, + "grad_norm": 0.7166867852210999, + "learning_rate": 0.0001, + "loss": 58.6835, + "step": 15760 + }, + { + "epoch": 25.395021917670178, + "grad_norm": 0.7240379452705383, + "learning_rate": 0.0001, + "loss": 59.6293, + "step": 15770 + }, + { + "epoch": 25.411145261248553, + "grad_norm": 0.7465791702270508, + "learning_rate": 0.0001, + "loss": 58.2512, + "step": 15780 + }, + { + "epoch": 25.427268604826924, + "grad_norm": 0.6713439226150513, + "learning_rate": 0.0001, + "loss": 58.4149, + "step": 15790 + }, + { + "epoch": 25.4433919484053, + "grad_norm": 0.587433397769928, + "learning_rate": 0.0001, + "loss": 60.7315, + "step": 15800 + }, + { + "epoch": 25.459515291983674, + "grad_norm": 0.8216325044631958, + "learning_rate": 0.0001, + "loss": 58.5339, + "step": 15810 + }, + { + "epoch": 25.47563863556205, + "grad_norm": 0.8947033286094666, + "learning_rate": 0.0001, + "loss": 58.7762, + "step": 15820 + }, + { + "epoch": 25.491761979140424, + "grad_norm": 0.6659840941429138, + "learning_rate": 0.0001, + "loss": 58.9931, + "step": 15830 + }, + { + "epoch": 25.5078853227188, + "grad_norm": 0.7239252328872681, + "learning_rate": 0.0001, + "loss": 59.7113, + "step": 15840 + }, + { + "epoch": 25.524008666297174, + "grad_norm": 0.7233046889305115, + "learning_rate": 0.0001, + "loss": 58.973, + "step": 15850 + }, + { + "epoch": 25.54013200987555, + "grad_norm": 0.6620002388954163, + "learning_rate": 0.0001, + "loss": 57.1255, + "step": 15860 + }, + { + "epoch": 25.556255353453924, + "grad_norm": 0.6544366478919983, + "learning_rate": 0.0001, + "loss": 58.5897, + "step": 15870 + }, + { + "epoch": 25.572378697032295, + "grad_norm": 0.8955110311508179, + "learning_rate": 0.0001, + "loss": 58.3241, + "step": 15880 + }, + { + "epoch": 25.58850204061067, + "grad_norm": 0.6869794726371765, + "learning_rate": 0.0001, + "loss": 58.1951, + "step": 15890 + }, + { + "epoch": 25.604625384189045, + "grad_norm": 0.6819241642951965, + "learning_rate": 0.0001, + "loss": 58.6505, + "step": 15900 + }, + { + "epoch": 25.62074872776742, + "grad_norm": 0.6019999980926514, + "learning_rate": 0.0001, + "loss": 58.1397, + "step": 15910 + }, + { + "epoch": 25.636872071345795, + "grad_norm": 0.6664146184921265, + "learning_rate": 0.0001, + "loss": 58.6735, + "step": 15920 + }, + { + "epoch": 25.65299541492417, + "grad_norm": 0.7511261701583862, + "learning_rate": 0.0001, + "loss": 58.9189, + "step": 15930 + }, + { + "epoch": 25.669118758502545, + "grad_norm": 0.6886443495750427, + "learning_rate": 0.0001, + "loss": 60.1288, + "step": 15940 + }, + { + "epoch": 25.68524210208092, + "grad_norm": 0.7647057771682739, + "learning_rate": 0.0001, + "loss": 58.5042, + "step": 15950 + }, + { + "epoch": 25.701365445659295, + "grad_norm": 0.6636016368865967, + "learning_rate": 0.0001, + "loss": 56.7644, + "step": 15960 + }, + { + "epoch": 25.717488789237667, + "grad_norm": 0.6909965872764587, + "learning_rate": 0.0001, + "loss": 58.9248, + "step": 15970 + }, + { + "epoch": 25.73361213281604, + "grad_norm": 0.7853394150733948, + "learning_rate": 0.0001, + "loss": 58.7267, + "step": 15980 + }, + { + "epoch": 25.749735476394417, + "grad_norm": 0.5992239713668823, + "learning_rate": 0.0001, + "loss": 58.4463, + "step": 15990 + }, + { + "epoch": 25.76585881997279, + "grad_norm": 0.5730507969856262, + "learning_rate": 0.0001, + "loss": 58.2946, + "step": 16000 + }, + { + "epoch": 25.781982163551167, + "grad_norm": 0.8904218077659607, + "learning_rate": 0.0001, + "loss": 58.8344, + "step": 16010 + }, + { + "epoch": 25.79810550712954, + "grad_norm": 0.8680905103683472, + "learning_rate": 0.0001, + "loss": 58.8969, + "step": 16020 + }, + { + "epoch": 25.814228850707917, + "grad_norm": 0.7501125335693359, + "learning_rate": 0.0001, + "loss": 59.0709, + "step": 16030 + }, + { + "epoch": 25.83035219428629, + "grad_norm": 0.7211290597915649, + "learning_rate": 0.0001, + "loss": 58.1067, + "step": 16040 + }, + { + "epoch": 25.846475537864663, + "grad_norm": 0.6130759119987488, + "learning_rate": 0.0001, + "loss": 58.7301, + "step": 16050 + }, + { + "epoch": 25.862598881443038, + "grad_norm": 0.5713672041893005, + "learning_rate": 0.0001, + "loss": 59.7271, + "step": 16060 + }, + { + "epoch": 25.878722225021413, + "grad_norm": 0.5994897484779358, + "learning_rate": 0.0001, + "loss": 59.3829, + "step": 16070 + }, + { + "epoch": 25.894845568599788, + "grad_norm": 0.686657190322876, + "learning_rate": 0.0001, + "loss": 59.2881, + "step": 16080 + }, + { + "epoch": 25.910968912178163, + "grad_norm": 0.6650110483169556, + "learning_rate": 0.0001, + "loss": 56.2951, + "step": 16090 + }, + { + "epoch": 25.927092255756538, + "grad_norm": 0.7304360866546631, + "learning_rate": 0.0001, + "loss": 60.5764, + "step": 16100 + }, + { + "epoch": 25.943215599334913, + "grad_norm": 0.6502479910850525, + "learning_rate": 0.0001, + "loss": 59.3936, + "step": 16110 + }, + { + "epoch": 25.959338942913288, + "grad_norm": 0.7156543731689453, + "learning_rate": 0.0001, + "loss": 58.3354, + "step": 16120 + }, + { + "epoch": 25.975462286491663, + "grad_norm": 0.951116144657135, + "learning_rate": 0.0001, + "loss": 60.3694, + "step": 16130 + }, + { + "epoch": 25.991585630070034, + "grad_norm": 0.6068692207336426, + "learning_rate": 0.0001, + "loss": 58.4947, + "step": 16140 + }, + { + "epoch": 26.00644933743135, + "grad_norm": 0.7921227216720581, + "learning_rate": 0.0001, + "loss": 54.2985, + "step": 16150 + }, + { + "epoch": 26.022572681009724, + "grad_norm": 0.7406607866287231, + "learning_rate": 0.0001, + "loss": 59.1799, + "step": 16160 + }, + { + "epoch": 26.0386960245881, + "grad_norm": 0.6859458088874817, + "learning_rate": 0.0001, + "loss": 58.5136, + "step": 16170 + }, + { + "epoch": 26.054819368166473, + "grad_norm": 0.5251442790031433, + "learning_rate": 0.0001, + "loss": 58.0473, + "step": 16180 + }, + { + "epoch": 26.07094271174485, + "grad_norm": 0.8837170600891113, + "learning_rate": 0.0001, + "loss": 58.7922, + "step": 16190 + }, + { + "epoch": 26.087066055323223, + "grad_norm": 0.6178808808326721, + "learning_rate": 0.0001, + "loss": 58.5929, + "step": 16200 + }, + { + "epoch": 26.1031893989016, + "grad_norm": 0.7404752373695374, + "learning_rate": 0.0001, + "loss": 60.8605, + "step": 16210 + }, + { + "epoch": 26.119312742479973, + "grad_norm": 0.812877357006073, + "learning_rate": 0.0001, + "loss": 60.1625, + "step": 16220 + }, + { + "epoch": 26.135436086058345, + "grad_norm": 0.7631093859672546, + "learning_rate": 0.0001, + "loss": 59.0114, + "step": 16230 + }, + { + "epoch": 26.15155942963672, + "grad_norm": 0.5635222792625427, + "learning_rate": 0.0001, + "loss": 58.3207, + "step": 16240 + }, + { + "epoch": 26.167682773215095, + "grad_norm": 0.6111196875572205, + "learning_rate": 0.0001, + "loss": 60.1611, + "step": 16250 + }, + { + "epoch": 26.18380611679347, + "grad_norm": 0.6021577715873718, + "learning_rate": 0.0001, + "loss": 59.2199, + "step": 16260 + }, + { + "epoch": 26.199929460371845, + "grad_norm": 0.6264477968215942, + "learning_rate": 0.0001, + "loss": 58.9156, + "step": 16270 + }, + { + "epoch": 26.21605280395022, + "grad_norm": 0.5620694160461426, + "learning_rate": 0.0001, + "loss": 58.699, + "step": 16280 + }, + { + "epoch": 26.232176147528595, + "grad_norm": 0.6636842489242554, + "learning_rate": 0.0001, + "loss": 58.1521, + "step": 16290 + }, + { + "epoch": 26.24829949110697, + "grad_norm": 1.903132677078247, + "learning_rate": 0.0001, + "loss": 56.8953, + "step": 16300 + }, + { + "epoch": 26.264422834685345, + "grad_norm": 0.6112220883369446, + "learning_rate": 0.0001, + "loss": 58.6921, + "step": 16310 + }, + { + "epoch": 26.280546178263716, + "grad_norm": 0.6523616909980774, + "learning_rate": 0.0001, + "loss": 59.8097, + "step": 16320 + }, + { + "epoch": 26.29666952184209, + "grad_norm": 0.6033239364624023, + "learning_rate": 0.0001, + "loss": 58.2301, + "step": 16330 + }, + { + "epoch": 26.312792865420466, + "grad_norm": 0.6334905624389648, + "learning_rate": 0.0001, + "loss": 57.4851, + "step": 16340 + }, + { + "epoch": 26.32891620899884, + "grad_norm": 0.8884124159812927, + "learning_rate": 0.0001, + "loss": 58.4404, + "step": 16350 + }, + { + "epoch": 26.345039552577216, + "grad_norm": 0.6134583950042725, + "learning_rate": 0.0001, + "loss": 59.7355, + "step": 16360 + }, + { + "epoch": 26.36116289615559, + "grad_norm": 0.5804709792137146, + "learning_rate": 0.0001, + "loss": 59.3307, + "step": 16370 + }, + { + "epoch": 26.377286239733966, + "grad_norm": 1.3471691608428955, + "learning_rate": 0.0001, + "loss": 60.4356, + "step": 16380 + }, + { + "epoch": 26.39340958331234, + "grad_norm": 0.7593581080436707, + "learning_rate": 0.0001, + "loss": 60.6596, + "step": 16390 + }, + { + "epoch": 26.409532926890712, + "grad_norm": 0.7059800624847412, + "learning_rate": 0.0001, + "loss": 58.8393, + "step": 16400 + }, + { + "epoch": 26.425656270469087, + "grad_norm": 0.836167573928833, + "learning_rate": 0.0001, + "loss": 59.7399, + "step": 16410 + }, + { + "epoch": 26.441779614047462, + "grad_norm": 0.7547316551208496, + "learning_rate": 0.0001, + "loss": 57.8424, + "step": 16420 + }, + { + "epoch": 26.457902957625837, + "grad_norm": 0.6655163168907166, + "learning_rate": 0.0001, + "loss": 58.7625, + "step": 16430 + }, + { + "epoch": 26.474026301204212, + "grad_norm": 0.6393458247184753, + "learning_rate": 0.0001, + "loss": 58.88, + "step": 16440 + }, + { + "epoch": 26.490149644782587, + "grad_norm": 0.7284976840019226, + "learning_rate": 0.0001, + "loss": 59.734, + "step": 16450 + }, + { + "epoch": 26.506272988360962, + "grad_norm": 0.7217265963554382, + "learning_rate": 0.0001, + "loss": 58.9343, + "step": 16460 + }, + { + "epoch": 26.522396331939337, + "grad_norm": 0.6045732498168945, + "learning_rate": 0.0001, + "loss": 58.758, + "step": 16470 + }, + { + "epoch": 26.538519675517712, + "grad_norm": 0.6189675331115723, + "learning_rate": 0.0001, + "loss": 56.4975, + "step": 16480 + }, + { + "epoch": 26.554643019096083, + "grad_norm": 0.5564262270927429, + "learning_rate": 0.0001, + "loss": 58.6003, + "step": 16490 + }, + { + "epoch": 26.57076636267446, + "grad_norm": 0.727336049079895, + "learning_rate": 0.0001, + "loss": 59.097, + "step": 16500 + }, + { + "epoch": 26.586889706252833, + "grad_norm": 0.5768782496452332, + "learning_rate": 0.0001, + "loss": 59.4801, + "step": 16510 + }, + { + "epoch": 26.60301304983121, + "grad_norm": 0.5879726409912109, + "learning_rate": 0.0001, + "loss": 58.8839, + "step": 16520 + }, + { + "epoch": 26.619136393409583, + "grad_norm": 0.6342545747756958, + "learning_rate": 0.0001, + "loss": 58.4209, + "step": 16530 + }, + { + "epoch": 26.63525973698796, + "grad_norm": 0.7538003325462341, + "learning_rate": 0.0001, + "loss": 58.973, + "step": 16540 + }, + { + "epoch": 26.651383080566333, + "grad_norm": 0.7319458723068237, + "learning_rate": 0.0001, + "loss": 59.5449, + "step": 16550 + }, + { + "epoch": 26.667506424144708, + "grad_norm": 0.6014299392700195, + "learning_rate": 0.0001, + "loss": 59.2934, + "step": 16560 + }, + { + "epoch": 26.683629767723083, + "grad_norm": 0.6779450178146362, + "learning_rate": 0.0001, + "loss": 59.5138, + "step": 16570 + }, + { + "epoch": 26.699753111301455, + "grad_norm": 0.6749789118766785, + "learning_rate": 0.0001, + "loss": 58.2765, + "step": 16580 + }, + { + "epoch": 26.71587645487983, + "grad_norm": 0.6327691674232483, + "learning_rate": 0.0001, + "loss": 59.766, + "step": 16590 + }, + { + "epoch": 26.731999798458205, + "grad_norm": 0.5834855437278748, + "learning_rate": 0.0001, + "loss": 57.2493, + "step": 16600 + }, + { + "epoch": 26.74812314203658, + "grad_norm": 0.6618366837501526, + "learning_rate": 0.0001, + "loss": 57.2663, + "step": 16610 + }, + { + "epoch": 26.764246485614954, + "grad_norm": 0.7070173621177673, + "learning_rate": 0.0001, + "loss": 61.1754, + "step": 16620 + }, + { + "epoch": 26.78036982919333, + "grad_norm": 0.651278555393219, + "learning_rate": 0.0001, + "loss": 59.033, + "step": 16630 + }, + { + "epoch": 26.796493172771704, + "grad_norm": 0.634915828704834, + "learning_rate": 0.0001, + "loss": 58.2336, + "step": 16640 + }, + { + "epoch": 26.81261651635008, + "grad_norm": 0.8514180779457092, + "learning_rate": 0.0001, + "loss": 57.1678, + "step": 16650 + }, + { + "epoch": 26.828739859928454, + "grad_norm": 0.6718772053718567, + "learning_rate": 0.0001, + "loss": 58.5139, + "step": 16660 + }, + { + "epoch": 26.844863203506826, + "grad_norm": 0.6963548064231873, + "learning_rate": 0.0001, + "loss": 58.8449, + "step": 16670 + }, + { + "epoch": 26.8609865470852, + "grad_norm": 0.6028134822845459, + "learning_rate": 0.0001, + "loss": 58.4613, + "step": 16680 + }, + { + "epoch": 26.877109890663576, + "grad_norm": 0.7414282560348511, + "learning_rate": 0.0001, + "loss": 58.4078, + "step": 16690 + }, + { + "epoch": 26.89323323424195, + "grad_norm": 0.6019772291183472, + "learning_rate": 0.0001, + "loss": 58.086, + "step": 16700 + }, + { + "epoch": 26.909356577820326, + "grad_norm": 0.6267445683479309, + "learning_rate": 0.0001, + "loss": 58.5921, + "step": 16710 + }, + { + "epoch": 26.9254799213987, + "grad_norm": 0.7474961280822754, + "learning_rate": 0.0001, + "loss": 59.1194, + "step": 16720 + }, + { + "epoch": 26.941603264977076, + "grad_norm": 0.7952353358268738, + "learning_rate": 0.0001, + "loss": 58.4545, + "step": 16730 + }, + { + "epoch": 26.95772660855545, + "grad_norm": 0.6562651991844177, + "learning_rate": 0.0001, + "loss": 58.699, + "step": 16740 + }, + { + "epoch": 26.973849952133826, + "grad_norm": 0.6084649562835693, + "learning_rate": 0.0001, + "loss": 58.5552, + "step": 16750 + }, + { + "epoch": 26.989973295712197, + "grad_norm": 0.5753876566886902, + "learning_rate": 0.0001, + "loss": 58.5317, + "step": 16760 + }, + { + "epoch": 27.00483700307351, + "grad_norm": 0.7396492958068848, + "learning_rate": 0.0001, + "loss": 54.4917, + "step": 16770 + }, + { + "epoch": 27.020960346651886, + "grad_norm": 0.5759479999542236, + "learning_rate": 0.0001, + "loss": 60.5276, + "step": 16780 + }, + { + "epoch": 27.03708369023026, + "grad_norm": 0.6621933579444885, + "learning_rate": 0.0001, + "loss": 57.8109, + "step": 16790 + }, + { + "epoch": 27.053207033808636, + "grad_norm": 0.6219635605812073, + "learning_rate": 0.0001, + "loss": 59.1457, + "step": 16800 + }, + { + "epoch": 27.06933037738701, + "grad_norm": 0.6395589113235474, + "learning_rate": 0.0001, + "loss": 58.2069, + "step": 16810 + }, + { + "epoch": 27.085453720965386, + "grad_norm": 0.9114459753036499, + "learning_rate": 0.0001, + "loss": 58.7224, + "step": 16820 + }, + { + "epoch": 27.10157706454376, + "grad_norm": 0.6759451031684875, + "learning_rate": 0.0001, + "loss": 59.3057, + "step": 16830 + }, + { + "epoch": 27.117700408122133, + "grad_norm": 0.6997210383415222, + "learning_rate": 0.0001, + "loss": 59.7155, + "step": 16840 + }, + { + "epoch": 27.133823751700508, + "grad_norm": 1.4591188430786133, + "learning_rate": 0.0001, + "loss": 59.7251, + "step": 16850 + }, + { + "epoch": 27.149947095278883, + "grad_norm": 0.661625325679779, + "learning_rate": 0.0001, + "loss": 59.065, + "step": 16860 + }, + { + "epoch": 27.166070438857258, + "grad_norm": 0.7471212148666382, + "learning_rate": 0.0001, + "loss": 60.2483, + "step": 16870 + }, + { + "epoch": 27.182193782435633, + "grad_norm": 0.7325915098190308, + "learning_rate": 0.0001, + "loss": 59.5878, + "step": 16880 + }, + { + "epoch": 27.198317126014008, + "grad_norm": 0.6631053686141968, + "learning_rate": 0.0001, + "loss": 60.045, + "step": 16890 + }, + { + "epoch": 27.214440469592382, + "grad_norm": 0.7794095277786255, + "learning_rate": 0.0001, + "loss": 58.6427, + "step": 16900 + }, + { + "epoch": 27.230563813170757, + "grad_norm": 0.683525025844574, + "learning_rate": 0.0001, + "loss": 58.2072, + "step": 16910 + }, + { + "epoch": 27.246687156749132, + "grad_norm": 0.7676731944084167, + "learning_rate": 0.0001, + "loss": 59.5006, + "step": 16920 + }, + { + "epoch": 27.262810500327504, + "grad_norm": 0.6451899409294128, + "learning_rate": 0.0001, + "loss": 59.5833, + "step": 16930 + }, + { + "epoch": 27.27893384390588, + "grad_norm": 0.8003790378570557, + "learning_rate": 0.0001, + "loss": 59.6564, + "step": 16940 + }, + { + "epoch": 27.295057187484254, + "grad_norm": 0.8254024386405945, + "learning_rate": 0.0001, + "loss": 59.8074, + "step": 16950 + }, + { + "epoch": 27.31118053106263, + "grad_norm": 0.8280795812606812, + "learning_rate": 0.0001, + "loss": 57.1944, + "step": 16960 + }, + { + "epoch": 27.327303874641004, + "grad_norm": 0.6558408141136169, + "learning_rate": 0.0001, + "loss": 55.8846, + "step": 16970 + }, + { + "epoch": 27.34342721821938, + "grad_norm": 0.8485943078994751, + "learning_rate": 0.0001, + "loss": 57.6715, + "step": 16980 + }, + { + "epoch": 27.359550561797754, + "grad_norm": 0.7805924415588379, + "learning_rate": 0.0001, + "loss": 61.0427, + "step": 16990 + }, + { + "epoch": 27.37567390537613, + "grad_norm": 0.7988429069519043, + "learning_rate": 0.0001, + "loss": 57.6632, + "step": 17000 + }, + { + "epoch": 27.391797248954504, + "grad_norm": 0.816113293170929, + "learning_rate": 0.0001, + "loss": 59.2601, + "step": 17010 + }, + { + "epoch": 27.407920592532875, + "grad_norm": 0.7985144257545471, + "learning_rate": 0.0001, + "loss": 60.3656, + "step": 17020 + }, + { + "epoch": 27.42404393611125, + "grad_norm": 0.7022702097892761, + "learning_rate": 0.0001, + "loss": 58.657, + "step": 17030 + }, + { + "epoch": 27.440167279689625, + "grad_norm": 0.8588566780090332, + "learning_rate": 0.0001, + "loss": 58.9401, + "step": 17040 + }, + { + "epoch": 27.456290623268, + "grad_norm": 0.6444416046142578, + "learning_rate": 0.0001, + "loss": 57.79, + "step": 17050 + }, + { + "epoch": 27.472413966846375, + "grad_norm": 0.7800275683403015, + "learning_rate": 0.0001, + "loss": 60.0955, + "step": 17060 + }, + { + "epoch": 27.48853731042475, + "grad_norm": 0.5514053702354431, + "learning_rate": 0.0001, + "loss": 59.2458, + "step": 17070 + }, + { + "epoch": 27.504660654003125, + "grad_norm": 0.6667745113372803, + "learning_rate": 0.0001, + "loss": 59.872, + "step": 17080 + }, + { + "epoch": 27.5207839975815, + "grad_norm": 0.6694201231002808, + "learning_rate": 0.0001, + "loss": 59.4699, + "step": 17090 + }, + { + "epoch": 27.536907341159875, + "grad_norm": 0.721091091632843, + "learning_rate": 0.0001, + "loss": 59.8662, + "step": 17100 + }, + { + "epoch": 27.553030684738246, + "grad_norm": 0.8182216882705688, + "learning_rate": 0.0001, + "loss": 57.3634, + "step": 17110 + }, + { + "epoch": 27.56915402831662, + "grad_norm": 1.0078271627426147, + "learning_rate": 0.0001, + "loss": 59.3628, + "step": 17120 + }, + { + "epoch": 27.585277371894996, + "grad_norm": 0.6158314347267151, + "learning_rate": 0.0001, + "loss": 59.123, + "step": 17130 + }, + { + "epoch": 27.60140071547337, + "grad_norm": 0.5983971357345581, + "learning_rate": 0.0001, + "loss": 58.6259, + "step": 17140 + }, + { + "epoch": 27.617524059051746, + "grad_norm": 0.4854629337787628, + "learning_rate": 0.0001, + "loss": 58.781, + "step": 17150 + }, + { + "epoch": 27.63364740263012, + "grad_norm": 0.582637369632721, + "learning_rate": 0.0001, + "loss": 59.6194, + "step": 17160 + }, + { + "epoch": 27.649770746208496, + "grad_norm": 0.7129100561141968, + "learning_rate": 0.0001, + "loss": 58.495, + "step": 17170 + }, + { + "epoch": 27.66589408978687, + "grad_norm": 0.8179757595062256, + "learning_rate": 0.0001, + "loss": 58.7483, + "step": 17180 + }, + { + "epoch": 27.682017433365242, + "grad_norm": 0.7066773772239685, + "learning_rate": 0.0001, + "loss": 58.2444, + "step": 17190 + }, + { + "epoch": 27.698140776943617, + "grad_norm": 0.725228488445282, + "learning_rate": 0.0001, + "loss": 58.0686, + "step": 17200 + }, + { + "epoch": 27.714264120521992, + "grad_norm": 0.630705714225769, + "learning_rate": 0.0001, + "loss": 56.873, + "step": 17210 + }, + { + "epoch": 27.730387464100367, + "grad_norm": 0.7248587608337402, + "learning_rate": 0.0001, + "loss": 59.6893, + "step": 17220 + }, + { + "epoch": 27.746510807678742, + "grad_norm": 0.715254545211792, + "learning_rate": 0.0001, + "loss": 57.7157, + "step": 17230 + }, + { + "epoch": 27.762634151257117, + "grad_norm": 0.6525955200195312, + "learning_rate": 0.0001, + "loss": 58.6077, + "step": 17240 + }, + { + "epoch": 27.778757494835492, + "grad_norm": 0.7938500642776489, + "learning_rate": 0.0001, + "loss": 60.325, + "step": 17250 + }, + { + "epoch": 27.794880838413867, + "grad_norm": 0.6821472644805908, + "learning_rate": 0.0001, + "loss": 56.5372, + "step": 17260 + }, + { + "epoch": 27.811004181992242, + "grad_norm": 0.5398596525192261, + "learning_rate": 0.0001, + "loss": 57.433, + "step": 17270 + }, + { + "epoch": 27.827127525570614, + "grad_norm": 0.6805903911590576, + "learning_rate": 0.0001, + "loss": 58.6501, + "step": 17280 + }, + { + "epoch": 27.84325086914899, + "grad_norm": 0.6547013521194458, + "learning_rate": 0.0001, + "loss": 57.9083, + "step": 17290 + }, + { + "epoch": 27.859374212727364, + "grad_norm": 0.6482307314872742, + "learning_rate": 0.0001, + "loss": 59.2978, + "step": 17300 + }, + { + "epoch": 27.87549755630574, + "grad_norm": 0.867108941078186, + "learning_rate": 0.0001, + "loss": 59.452, + "step": 17310 + }, + { + "epoch": 27.891620899884114, + "grad_norm": 0.5809998512268066, + "learning_rate": 0.0001, + "loss": 59.0863, + "step": 17320 + }, + { + "epoch": 27.90774424346249, + "grad_norm": 0.7422227263450623, + "learning_rate": 0.0001, + "loss": 58.7768, + "step": 17330 + }, + { + "epoch": 27.923867587040863, + "grad_norm": 0.6092364192008972, + "learning_rate": 0.0001, + "loss": 58.1772, + "step": 17340 + }, + { + "epoch": 27.93999093061924, + "grad_norm": 0.688312292098999, + "learning_rate": 0.0001, + "loss": 58.5475, + "step": 17350 + }, + { + "epoch": 27.956114274197613, + "grad_norm": 0.8303641080856323, + "learning_rate": 0.0001, + "loss": 59.8533, + "step": 17360 + }, + { + "epoch": 27.972237617775985, + "grad_norm": 0.903464138507843, + "learning_rate": 0.0001, + "loss": 58.5677, + "step": 17370 + }, + { + "epoch": 27.98836096135436, + "grad_norm": 0.7864969968795776, + "learning_rate": 0.0001, + "loss": 59.1479, + "step": 17380 + }, + { + "epoch": 28.003224668715674, + "grad_norm": 0.7318345904350281, + "learning_rate": 0.0001, + "loss": 56.2894, + "step": 17390 + }, + { + "epoch": 28.01934801229405, + "grad_norm": 0.6740039587020874, + "learning_rate": 0.0001, + "loss": 60.3117, + "step": 17400 + }, + { + "epoch": 28.035471355872424, + "grad_norm": 0.6491691470146179, + "learning_rate": 0.0001, + "loss": 59.2796, + "step": 17410 + }, + { + "epoch": 28.0515946994508, + "grad_norm": 0.8077054619789124, + "learning_rate": 0.0001, + "loss": 60.4482, + "step": 17420 + }, + { + "epoch": 28.067718043029174, + "grad_norm": 0.8239777088165283, + "learning_rate": 0.0001, + "loss": 58.5352, + "step": 17430 + }, + { + "epoch": 28.08384138660755, + "grad_norm": 0.6907490491867065, + "learning_rate": 0.0001, + "loss": 60.3665, + "step": 17440 + }, + { + "epoch": 28.09996473018592, + "grad_norm": 0.7226580381393433, + "learning_rate": 0.0001, + "loss": 58.3535, + "step": 17450 + }, + { + "epoch": 28.116088073764296, + "grad_norm": 0.5712003707885742, + "learning_rate": 0.0001, + "loss": 57.8514, + "step": 17460 + }, + { + "epoch": 28.13221141734267, + "grad_norm": 0.532891571521759, + "learning_rate": 0.0001, + "loss": 57.5121, + "step": 17470 + }, + { + "epoch": 28.148334760921045, + "grad_norm": 0.6007943153381348, + "learning_rate": 0.0001, + "loss": 59.1072, + "step": 17480 + }, + { + "epoch": 28.16445810449942, + "grad_norm": 0.9194851517677307, + "learning_rate": 0.0001, + "loss": 59.8352, + "step": 17490 + }, + { + "epoch": 28.180581448077795, + "grad_norm": 0.5962793827056885, + "learning_rate": 0.0001, + "loss": 58.8552, + "step": 17500 + }, + { + "epoch": 28.19670479165617, + "grad_norm": 0.6472097635269165, + "learning_rate": 0.0001, + "loss": 60.072, + "step": 17510 + }, + { + "epoch": 28.212828135234545, + "grad_norm": 0.7954792976379395, + "learning_rate": 0.0001, + "loss": 57.754, + "step": 17520 + }, + { + "epoch": 28.22895147881292, + "grad_norm": 0.5923368334770203, + "learning_rate": 0.0001, + "loss": 56.9673, + "step": 17530 + }, + { + "epoch": 28.24507482239129, + "grad_norm": 1.0150458812713623, + "learning_rate": 0.0001, + "loss": 57.1309, + "step": 17540 + }, + { + "epoch": 28.261198165969667, + "grad_norm": 0.597544252872467, + "learning_rate": 0.0001, + "loss": 58.6526, + "step": 17550 + }, + { + "epoch": 28.27732150954804, + "grad_norm": 0.8652740716934204, + "learning_rate": 0.0001, + "loss": 58.7396, + "step": 17560 + }, + { + "epoch": 28.293444853126417, + "grad_norm": 0.8436110615730286, + "learning_rate": 0.0001, + "loss": 59.0186, + "step": 17570 + }, + { + "epoch": 28.30956819670479, + "grad_norm": 0.6380386352539062, + "learning_rate": 0.0001, + "loss": 58.3723, + "step": 17580 + }, + { + "epoch": 28.325691540283167, + "grad_norm": 0.6668462157249451, + "learning_rate": 0.0001, + "loss": 58.0738, + "step": 17590 + }, + { + "epoch": 28.34181488386154, + "grad_norm": 0.7173913717269897, + "learning_rate": 0.0001, + "loss": 58.1463, + "step": 17600 + }, + { + "epoch": 28.357938227439917, + "grad_norm": 0.6073333621025085, + "learning_rate": 0.0001, + "loss": 58.0279, + "step": 17610 + }, + { + "epoch": 28.37406157101829, + "grad_norm": 0.6092989444732666, + "learning_rate": 0.0001, + "loss": 56.1147, + "step": 17620 + }, + { + "epoch": 28.390184914596663, + "grad_norm": 0.7214272618293762, + "learning_rate": 0.0001, + "loss": 58.1375, + "step": 17630 + }, + { + "epoch": 28.406308258175038, + "grad_norm": 0.5026358366012573, + "learning_rate": 0.0001, + "loss": 59.2814, + "step": 17640 + }, + { + "epoch": 28.422431601753413, + "grad_norm": 0.6362181901931763, + "learning_rate": 0.0001, + "loss": 60.6815, + "step": 17650 + }, + { + "epoch": 28.438554945331788, + "grad_norm": 0.7994716167449951, + "learning_rate": 0.0001, + "loss": 58.4757, + "step": 17660 + }, + { + "epoch": 28.454678288910163, + "grad_norm": 0.6012625694274902, + "learning_rate": 0.0001, + "loss": 58.1216, + "step": 17670 + }, + { + "epoch": 28.470801632488538, + "grad_norm": 0.5765920877456665, + "learning_rate": 0.0001, + "loss": 60.1921, + "step": 17680 + }, + { + "epoch": 28.486924976066913, + "grad_norm": 0.6323729753494263, + "learning_rate": 0.0001, + "loss": 59.5405, + "step": 17690 + }, + { + "epoch": 28.503048319645288, + "grad_norm": 0.6641665697097778, + "learning_rate": 0.0001, + "loss": 59.3311, + "step": 17700 + }, + { + "epoch": 28.519171663223663, + "grad_norm": 0.7021269202232361, + "learning_rate": 0.0001, + "loss": 58.3317, + "step": 17710 + }, + { + "epoch": 28.535295006802034, + "grad_norm": 0.6319525241851807, + "learning_rate": 0.0001, + "loss": 57.4676, + "step": 17720 + }, + { + "epoch": 28.55141835038041, + "grad_norm": 0.6366249322891235, + "learning_rate": 0.0001, + "loss": 60.761, + "step": 17730 + }, + { + "epoch": 28.567541693958784, + "grad_norm": 1.487281322479248, + "learning_rate": 0.0001, + "loss": 58.2667, + "step": 17740 + }, + { + "epoch": 28.58366503753716, + "grad_norm": 0.6286079287528992, + "learning_rate": 0.0001, + "loss": 56.5199, + "step": 17750 + }, + { + "epoch": 28.599788381115534, + "grad_norm": 0.6458479762077332, + "learning_rate": 0.0001, + "loss": 59.0782, + "step": 17760 + }, + { + "epoch": 28.61591172469391, + "grad_norm": 0.6947740316390991, + "learning_rate": 0.0001, + "loss": 61.1563, + "step": 17770 + }, + { + "epoch": 28.632035068272284, + "grad_norm": 1.3151084184646606, + "learning_rate": 0.0001, + "loss": 58.7798, + "step": 17780 + }, + { + "epoch": 28.64815841185066, + "grad_norm": 0.5333471894264221, + "learning_rate": 0.0001, + "loss": 58.1223, + "step": 17790 + }, + { + "epoch": 28.66428175542903, + "grad_norm": 0.6268185973167419, + "learning_rate": 0.0001, + "loss": 58.9371, + "step": 17800 + }, + { + "epoch": 28.680405099007405, + "grad_norm": 0.5982078313827515, + "learning_rate": 0.0001, + "loss": 57.9883, + "step": 17810 + }, + { + "epoch": 28.69652844258578, + "grad_norm": 0.5744349360466003, + "learning_rate": 0.0001, + "loss": 59.4091, + "step": 17820 + }, + { + "epoch": 28.712651786164155, + "grad_norm": 0.7235150337219238, + "learning_rate": 0.0001, + "loss": 58.1419, + "step": 17830 + }, + { + "epoch": 28.72877512974253, + "grad_norm": 0.5630679726600647, + "learning_rate": 0.0001, + "loss": 59.547, + "step": 17840 + }, + { + "epoch": 28.744898473320905, + "grad_norm": 0.854364812374115, + "learning_rate": 0.0001, + "loss": 58.413, + "step": 17850 + }, + { + "epoch": 28.76102181689928, + "grad_norm": 0.6486361622810364, + "learning_rate": 0.0001, + "loss": 58.0385, + "step": 17860 + }, + { + "epoch": 28.777145160477655, + "grad_norm": 0.5967833995819092, + "learning_rate": 0.0001, + "loss": 57.4478, + "step": 17870 + }, + { + "epoch": 28.79326850405603, + "grad_norm": 0.6199812889099121, + "learning_rate": 0.0001, + "loss": 56.8104, + "step": 17880 + }, + { + "epoch": 28.8093918476344, + "grad_norm": 0.6966105699539185, + "learning_rate": 0.0001, + "loss": 59.4416, + "step": 17890 + }, + { + "epoch": 28.825515191212777, + "grad_norm": 0.7352570295333862, + "learning_rate": 0.0001, + "loss": 58.9475, + "step": 17900 + }, + { + "epoch": 28.84163853479115, + "grad_norm": 0.7693241238594055, + "learning_rate": 0.0001, + "loss": 57.4324, + "step": 17910 + }, + { + "epoch": 28.857761878369526, + "grad_norm": 1.0005452632904053, + "learning_rate": 0.0001, + "loss": 59.0267, + "step": 17920 + }, + { + "epoch": 28.8738852219479, + "grad_norm": 0.6526029109954834, + "learning_rate": 0.0001, + "loss": 57.7836, + "step": 17930 + }, + { + "epoch": 28.890008565526276, + "grad_norm": 0.7833093404769897, + "learning_rate": 0.0001, + "loss": 59.3319, + "step": 17940 + }, + { + "epoch": 28.90613190910465, + "grad_norm": 0.6504502892494202, + "learning_rate": 0.0001, + "loss": 57.5452, + "step": 17950 + }, + { + "epoch": 28.922255252683026, + "grad_norm": 0.5760630369186401, + "learning_rate": 0.0001, + "loss": 56.2115, + "step": 17960 + }, + { + "epoch": 28.9383785962614, + "grad_norm": 0.7640287280082703, + "learning_rate": 0.0001, + "loss": 59.051, + "step": 17970 + }, + { + "epoch": 28.954501939839773, + "grad_norm": 0.6596423983573914, + "learning_rate": 0.0001, + "loss": 59.4821, + "step": 17980 + }, + { + "epoch": 28.970625283418148, + "grad_norm": 2.1421217918395996, + "learning_rate": 0.0001, + "loss": 58.2994, + "step": 17990 + }, + { + "epoch": 28.986748626996523, + "grad_norm": 0.6420243382453918, + "learning_rate": 0.0001, + "loss": 55.9762, + "step": 18000 + }, + { + "epoch": 29.001612334357837, + "grad_norm": 0.6530152559280396, + "learning_rate": 0.0001, + "loss": 52.9146, + "step": 18010 + }, + { + "epoch": 29.017735677936212, + "grad_norm": 0.6559716463088989, + "learning_rate": 0.0001, + "loss": 57.5697, + "step": 18020 + }, + { + "epoch": 29.033859021514587, + "grad_norm": 0.7522405982017517, + "learning_rate": 0.0001, + "loss": 58.5981, + "step": 18030 + }, + { + "epoch": 29.049982365092962, + "grad_norm": 0.7637670636177063, + "learning_rate": 0.0001, + "loss": 58.4835, + "step": 18040 + }, + { + "epoch": 29.066105708671337, + "grad_norm": 0.6451973915100098, + "learning_rate": 0.0001, + "loss": 58.5291, + "step": 18050 + }, + { + "epoch": 29.082229052249712, + "grad_norm": 0.6325218081474304, + "learning_rate": 0.0001, + "loss": 57.3294, + "step": 18060 + }, + { + "epoch": 29.098352395828083, + "grad_norm": 0.7488126754760742, + "learning_rate": 0.0001, + "loss": 59.6257, + "step": 18070 + }, + { + "epoch": 29.11447573940646, + "grad_norm": 0.6828444004058838, + "learning_rate": 0.0001, + "loss": 58.4092, + "step": 18080 + }, + { + "epoch": 29.130599082984833, + "grad_norm": 0.6108933687210083, + "learning_rate": 0.0001, + "loss": 60.2422, + "step": 18090 + }, + { + "epoch": 29.14672242656321, + "grad_norm": 0.640813410282135, + "learning_rate": 0.0001, + "loss": 59.8748, + "step": 18100 + }, + { + "epoch": 29.162845770141583, + "grad_norm": 0.6216452121734619, + "learning_rate": 0.0001, + "loss": 57.5762, + "step": 18110 + }, + { + "epoch": 29.17896911371996, + "grad_norm": 0.5484854578971863, + "learning_rate": 0.0001, + "loss": 57.5171, + "step": 18120 + }, + { + "epoch": 29.195092457298333, + "grad_norm": 0.6442582011222839, + "learning_rate": 0.0001, + "loss": 60.7324, + "step": 18130 + }, + { + "epoch": 29.211215800876708, + "grad_norm": 0.6196489334106445, + "learning_rate": 0.0001, + "loss": 58.6092, + "step": 18140 + }, + { + "epoch": 29.22733914445508, + "grad_norm": 0.6655876636505127, + "learning_rate": 0.0001, + "loss": 56.9686, + "step": 18150 + }, + { + "epoch": 29.243462488033455, + "grad_norm": 5.002511978149414, + "learning_rate": 0.0001, + "loss": 59.1202, + "step": 18160 + }, + { + "epoch": 29.25958583161183, + "grad_norm": 0.5576016902923584, + "learning_rate": 0.0001, + "loss": 57.3112, + "step": 18170 + }, + { + "epoch": 29.275709175190205, + "grad_norm": 0.6420336961746216, + "learning_rate": 0.0001, + "loss": 58.2884, + "step": 18180 + }, + { + "epoch": 29.29183251876858, + "grad_norm": 0.7113134860992432, + "learning_rate": 0.0001, + "loss": 58.9768, + "step": 18190 + }, + { + "epoch": 29.307955862346954, + "grad_norm": 0.7830891609191895, + "learning_rate": 0.0001, + "loss": 58.2999, + "step": 18200 + }, + { + "epoch": 29.32407920592533, + "grad_norm": 0.7687374353408813, + "learning_rate": 0.0001, + "loss": 60.3956, + "step": 18210 + }, + { + "epoch": 29.340202549503704, + "grad_norm": 0.596026599407196, + "learning_rate": 0.0001, + "loss": 56.5389, + "step": 18220 + }, + { + "epoch": 29.35632589308208, + "grad_norm": 0.6291328072547913, + "learning_rate": 0.0001, + "loss": 56.752, + "step": 18230 + }, + { + "epoch": 29.37244923666045, + "grad_norm": 0.8147439360618591, + "learning_rate": 0.0001, + "loss": 57.7872, + "step": 18240 + }, + { + "epoch": 29.388572580238826, + "grad_norm": 0.8024112582206726, + "learning_rate": 0.0001, + "loss": 58.2364, + "step": 18250 + }, + { + "epoch": 29.4046959238172, + "grad_norm": 0.6654099822044373, + "learning_rate": 0.0001, + "loss": 57.1549, + "step": 18260 + }, + { + "epoch": 29.420819267395576, + "grad_norm": 0.7781001329421997, + "learning_rate": 0.0001, + "loss": 57.8705, + "step": 18270 + }, + { + "epoch": 29.43694261097395, + "grad_norm": 0.699664294719696, + "learning_rate": 0.0001, + "loss": 59.2756, + "step": 18280 + }, + { + "epoch": 29.453065954552326, + "grad_norm": 0.6834261417388916, + "learning_rate": 0.0001, + "loss": 57.0605, + "step": 18290 + }, + { + "epoch": 29.4691892981307, + "grad_norm": 2.225274085998535, + "learning_rate": 0.0001, + "loss": 58.5602, + "step": 18300 + }, + { + "epoch": 29.485312641709076, + "grad_norm": 0.8640811443328857, + "learning_rate": 0.0001, + "loss": 57.6847, + "step": 18310 + }, + { + "epoch": 29.50143598528745, + "grad_norm": 0.7287184000015259, + "learning_rate": 0.0001, + "loss": 58.4448, + "step": 18320 + }, + { + "epoch": 29.517559328865822, + "grad_norm": 0.6942048668861389, + "learning_rate": 0.0001, + "loss": 58.3656, + "step": 18330 + }, + { + "epoch": 29.533682672444197, + "grad_norm": 0.8364513516426086, + "learning_rate": 0.0001, + "loss": 57.3624, + "step": 18340 + }, + { + "epoch": 29.549806016022572, + "grad_norm": 0.9943494200706482, + "learning_rate": 0.0001, + "loss": 58.2563, + "step": 18350 + }, + { + "epoch": 29.565929359600947, + "grad_norm": 0.6764407753944397, + "learning_rate": 0.0001, + "loss": 58.0796, + "step": 18360 + }, + { + "epoch": 29.582052703179322, + "grad_norm": 0.6659194827079773, + "learning_rate": 0.0001, + "loss": 60.181, + "step": 18370 + }, + { + "epoch": 29.598176046757697, + "grad_norm": 0.5525355339050293, + "learning_rate": 0.0001, + "loss": 57.7179, + "step": 18380 + }, + { + "epoch": 29.614299390336072, + "grad_norm": 0.6220526099205017, + "learning_rate": 0.0001, + "loss": 57.5135, + "step": 18390 + }, + { + "epoch": 29.630422733914447, + "grad_norm": 0.5199670195579529, + "learning_rate": 0.0001, + "loss": 59.3721, + "step": 18400 + }, + { + "epoch": 29.646546077492822, + "grad_norm": 0.6937117576599121, + "learning_rate": 0.0001, + "loss": 58.4896, + "step": 18410 + }, + { + "epoch": 29.662669421071193, + "grad_norm": 0.6366041302680969, + "learning_rate": 0.0001, + "loss": 59.1487, + "step": 18420 + }, + { + "epoch": 29.678792764649568, + "grad_norm": 0.520003080368042, + "learning_rate": 0.0001, + "loss": 58.8141, + "step": 18430 + }, + { + "epoch": 29.694916108227943, + "grad_norm": 1.3317536115646362, + "learning_rate": 0.0001, + "loss": 56.8534, + "step": 18440 + }, + { + "epoch": 29.711039451806318, + "grad_norm": 0.7395436763763428, + "learning_rate": 0.0001, + "loss": 59.7522, + "step": 18450 + }, + { + "epoch": 29.727162795384693, + "grad_norm": 0.5532745122909546, + "learning_rate": 0.0001, + "loss": 58.2614, + "step": 18460 + }, + { + "epoch": 29.743286138963068, + "grad_norm": 0.7754526138305664, + "learning_rate": 0.0001, + "loss": 59.1621, + "step": 18470 + }, + { + "epoch": 29.759409482541443, + "grad_norm": 0.6477323174476624, + "learning_rate": 0.0001, + "loss": 59.1955, + "step": 18480 + }, + { + "epoch": 29.775532826119818, + "grad_norm": 0.8694291710853577, + "learning_rate": 0.0001, + "loss": 57.7143, + "step": 18490 + }, + { + "epoch": 29.791656169698193, + "grad_norm": 0.6923109292984009, + "learning_rate": 0.0001, + "loss": 57.9599, + "step": 18500 + }, + { + "epoch": 29.807779513276564, + "grad_norm": 0.7835638523101807, + "learning_rate": 0.0001, + "loss": 59.1726, + "step": 18510 + }, + { + "epoch": 29.82390285685494, + "grad_norm": 0.7412428855895996, + "learning_rate": 0.0001, + "loss": 57.7021, + "step": 18520 + }, + { + "epoch": 29.840026200433314, + "grad_norm": 0.7160241007804871, + "learning_rate": 0.0001, + "loss": 57.755, + "step": 18530 + }, + { + "epoch": 29.85614954401169, + "grad_norm": 0.7844975590705872, + "learning_rate": 0.0001, + "loss": 58.6039, + "step": 18540 + }, + { + "epoch": 29.872272887590064, + "grad_norm": 0.8730555772781372, + "learning_rate": 0.0001, + "loss": 57.8274, + "step": 18550 + }, + { + "epoch": 29.88839623116844, + "grad_norm": 0.6070888042449951, + "learning_rate": 0.0001, + "loss": 57.646, + "step": 18560 + }, + { + "epoch": 29.904519574746814, + "grad_norm": 0.7387932538986206, + "learning_rate": 0.0001, + "loss": 58.7126, + "step": 18570 + }, + { + "epoch": 29.92064291832519, + "grad_norm": 0.6337044835090637, + "learning_rate": 0.0001, + "loss": 59.0599, + "step": 18580 + }, + { + "epoch": 29.93676626190356, + "grad_norm": 0.7069441676139832, + "learning_rate": 0.0001, + "loss": 58.3129, + "step": 18590 + }, + { + "epoch": 29.952889605481936, + "grad_norm": 0.6618002653121948, + "learning_rate": 0.0001, + "loss": 58.0288, + "step": 18600 + }, + { + "epoch": 29.96901294906031, + "grad_norm": 0.6438560485839844, + "learning_rate": 0.0001, + "loss": 58.4161, + "step": 18610 + }, + { + "epoch": 29.985136292638686, + "grad_norm": 0.7716667056083679, + "learning_rate": 0.0001, + "loss": 57.9302, + "step": 18620 + }, + { + "epoch": 30.0, + "grad_norm": 0.6670845746994019, + "learning_rate": 0.0001, + "loss": 53.8077, + "step": 18630 + }, + { + "epoch": 30.016123343578375, + "grad_norm": 0.6700365543365479, + "learning_rate": 0.0001, + "loss": 58.1248, + "step": 18640 + }, + { + "epoch": 30.03224668715675, + "grad_norm": 0.5925083756446838, + "learning_rate": 0.0001, + "loss": 58.0006, + "step": 18650 + }, + { + "epoch": 30.048370030735125, + "grad_norm": 0.5727967023849487, + "learning_rate": 0.0001, + "loss": 57.9431, + "step": 18660 + }, + { + "epoch": 30.0644933743135, + "grad_norm": 0.4864082634449005, + "learning_rate": 0.0001, + "loss": 56.5658, + "step": 18670 + }, + { + "epoch": 30.08061671789187, + "grad_norm": 0.8846231698989868, + "learning_rate": 0.0001, + "loss": 57.087, + "step": 18680 + }, + { + "epoch": 30.096740061470246, + "grad_norm": 0.6806173920631409, + "learning_rate": 0.0001, + "loss": 58.3673, + "step": 18690 + }, + { + "epoch": 30.11286340504862, + "grad_norm": 0.6638009548187256, + "learning_rate": 0.0001, + "loss": 59.5407, + "step": 18700 + }, + { + "epoch": 30.128986748626996, + "grad_norm": 0.7844518423080444, + "learning_rate": 0.0001, + "loss": 58.3762, + "step": 18710 + }, + { + "epoch": 30.14511009220537, + "grad_norm": 0.6602208018302917, + "learning_rate": 0.0001, + "loss": 58.3672, + "step": 18720 + }, + { + "epoch": 30.161233435783746, + "grad_norm": 0.5882258415222168, + "learning_rate": 0.0001, + "loss": 58.5334, + "step": 18730 + }, + { + "epoch": 30.17735677936212, + "grad_norm": 0.7399165034294128, + "learning_rate": 0.0001, + "loss": 59.8635, + "step": 18740 + }, + { + "epoch": 30.193480122940496, + "grad_norm": 0.6812124848365784, + "learning_rate": 0.0001, + "loss": 57.8219, + "step": 18750 + }, + { + "epoch": 30.20960346651887, + "grad_norm": 0.5468288660049438, + "learning_rate": 0.0001, + "loss": 57.7236, + "step": 18760 + }, + { + "epoch": 30.225726810097242, + "grad_norm": 0.6843936443328857, + "learning_rate": 0.0001, + "loss": 58.7141, + "step": 18770 + }, + { + "epoch": 30.241850153675617, + "grad_norm": 0.6243616342544556, + "learning_rate": 0.0001, + "loss": 59.791, + "step": 18780 + }, + { + "epoch": 30.257973497253992, + "grad_norm": 0.6359348893165588, + "learning_rate": 0.0001, + "loss": 57.3614, + "step": 18790 + }, + { + "epoch": 30.274096840832367, + "grad_norm": 1.4484305381774902, + "learning_rate": 0.0001, + "loss": 57.3209, + "step": 18800 + }, + { + "epoch": 30.290220184410742, + "grad_norm": 0.594723105430603, + "learning_rate": 0.0001, + "loss": 59.6517, + "step": 18810 + }, + { + "epoch": 30.306343527989117, + "grad_norm": 0.7364279627799988, + "learning_rate": 0.0001, + "loss": 60.1228, + "step": 18820 + }, + { + "epoch": 30.322466871567492, + "grad_norm": 0.7273280024528503, + "learning_rate": 0.0001, + "loss": 57.3952, + "step": 18830 + }, + { + "epoch": 30.338590215145867, + "grad_norm": 0.5930690169334412, + "learning_rate": 0.0001, + "loss": 57.9403, + "step": 18840 + }, + { + "epoch": 30.35471355872424, + "grad_norm": 0.6662129759788513, + "learning_rate": 0.0001, + "loss": 57.2412, + "step": 18850 + }, + { + "epoch": 30.370836902302614, + "grad_norm": 0.7068710923194885, + "learning_rate": 0.0001, + "loss": 56.4046, + "step": 18860 + }, + { + "epoch": 30.38696024588099, + "grad_norm": 0.9942213892936707, + "learning_rate": 0.0001, + "loss": 58.0625, + "step": 18870 + }, + { + "epoch": 30.403083589459364, + "grad_norm": 0.6646976470947266, + "learning_rate": 0.0001, + "loss": 58.0701, + "step": 18880 + }, + { + "epoch": 30.41920693303774, + "grad_norm": 0.7813248038291931, + "learning_rate": 0.0001, + "loss": 56.0009, + "step": 18890 + }, + { + "epoch": 30.435330276616114, + "grad_norm": 0.5491666197776794, + "learning_rate": 0.0001, + "loss": 57.3629, + "step": 18900 + }, + { + "epoch": 30.45145362019449, + "grad_norm": 0.6482571363449097, + "learning_rate": 0.0001, + "loss": 58.9482, + "step": 18910 + }, + { + "epoch": 30.467576963772864, + "grad_norm": 0.8382819890975952, + "learning_rate": 0.0001, + "loss": 57.3887, + "step": 18920 + }, + { + "epoch": 30.48370030735124, + "grad_norm": 0.7013330459594727, + "learning_rate": 0.0001, + "loss": 57.3789, + "step": 18930 + }, + { + "epoch": 30.49982365092961, + "grad_norm": 0.5680707693099976, + "learning_rate": 0.0001, + "loss": 57.2686, + "step": 18940 + }, + { + "epoch": 30.515946994507985, + "grad_norm": 0.5723867416381836, + "learning_rate": 0.0001, + "loss": 56.8546, + "step": 18950 + }, + { + "epoch": 30.53207033808636, + "grad_norm": 0.6836649775505066, + "learning_rate": 0.0001, + "loss": 58.8521, + "step": 18960 + }, + { + "epoch": 30.548193681664735, + "grad_norm": 0.6368657946586609, + "learning_rate": 0.0001, + "loss": 58.5553, + "step": 18970 + }, + { + "epoch": 30.56431702524311, + "grad_norm": 0.8456745743751526, + "learning_rate": 0.0001, + "loss": 58.0349, + "step": 18980 + }, + { + "epoch": 30.580440368821485, + "grad_norm": 0.7769978046417236, + "learning_rate": 0.0001, + "loss": 59.5172, + "step": 18990 + }, + { + "epoch": 30.59656371239986, + "grad_norm": 0.6169134974479675, + "learning_rate": 0.0001, + "loss": 57.8756, + "step": 19000 + }, + { + "epoch": 30.612687055978235, + "grad_norm": 0.6255633234977722, + "learning_rate": 0.0001, + "loss": 58.3724, + "step": 19010 + }, + { + "epoch": 30.62881039955661, + "grad_norm": 0.6262527704238892, + "learning_rate": 0.0001, + "loss": 59.6478, + "step": 19020 + }, + { + "epoch": 30.64493374313498, + "grad_norm": 0.9178198575973511, + "learning_rate": 0.0001, + "loss": 59.4106, + "step": 19030 + }, + { + "epoch": 30.661057086713356, + "grad_norm": 0.5433737635612488, + "learning_rate": 0.0001, + "loss": 57.9312, + "step": 19040 + }, + { + "epoch": 30.67718043029173, + "grad_norm": 0.6306186318397522, + "learning_rate": 0.0001, + "loss": 57.9233, + "step": 19050 + }, + { + "epoch": 30.693303773870106, + "grad_norm": 0.5454888939857483, + "learning_rate": 0.0001, + "loss": 58.1754, + "step": 19060 + }, + { + "epoch": 30.70942711744848, + "grad_norm": 0.53844153881073, + "learning_rate": 0.0001, + "loss": 59.1227, + "step": 19070 + }, + { + "epoch": 30.725550461026856, + "grad_norm": 0.7765403985977173, + "learning_rate": 0.0001, + "loss": 56.9315, + "step": 19080 + }, + { + "epoch": 30.74167380460523, + "grad_norm": 0.657633900642395, + "learning_rate": 0.0001, + "loss": 57.5241, + "step": 19090 + }, + { + "epoch": 30.757797148183606, + "grad_norm": 0.6395857334136963, + "learning_rate": 0.0001, + "loss": 58.8294, + "step": 19100 + }, + { + "epoch": 30.77392049176198, + "grad_norm": 0.679739236831665, + "learning_rate": 0.0001, + "loss": 58.0586, + "step": 19110 + }, + { + "epoch": 30.790043835340352, + "grad_norm": 0.5583140850067139, + "learning_rate": 0.0001, + "loss": 57.604, + "step": 19120 + }, + { + "epoch": 30.806167178918727, + "grad_norm": 1.0134044885635376, + "learning_rate": 0.0001, + "loss": 59.1611, + "step": 19130 + }, + { + "epoch": 30.822290522497102, + "grad_norm": 0.6906328797340393, + "learning_rate": 0.0001, + "loss": 57.9054, + "step": 19140 + }, + { + "epoch": 30.838413866075477, + "grad_norm": 0.6082453727722168, + "learning_rate": 0.0001, + "loss": 57.1707, + "step": 19150 + }, + { + "epoch": 30.854537209653852, + "grad_norm": 0.7322260737419128, + "learning_rate": 0.0001, + "loss": 57.4455, + "step": 19160 + }, + { + "epoch": 30.870660553232227, + "grad_norm": 1.7016369104385376, + "learning_rate": 0.0001, + "loss": 57.2955, + "step": 19170 + }, + { + "epoch": 30.886783896810602, + "grad_norm": 0.6822830438613892, + "learning_rate": 0.0001, + "loss": 57.1203, + "step": 19180 + }, + { + "epoch": 30.902907240388977, + "grad_norm": 0.6205664277076721, + "learning_rate": 0.0001, + "loss": 57.191, + "step": 19190 + }, + { + "epoch": 30.91903058396735, + "grad_norm": 0.6766132712364197, + "learning_rate": 0.0001, + "loss": 58.2869, + "step": 19200 + }, + { + "epoch": 30.935153927545723, + "grad_norm": 0.6686915755271912, + "learning_rate": 0.0001, + "loss": 57.9603, + "step": 19210 + }, + { + "epoch": 30.9512772711241, + "grad_norm": 0.691638171672821, + "learning_rate": 0.0001, + "loss": 58.5623, + "step": 19220 + }, + { + "epoch": 30.967400614702473, + "grad_norm": 0.5663546919822693, + "learning_rate": 0.0001, + "loss": 58.4948, + "step": 19230 + }, + { + "epoch": 30.98352395828085, + "grad_norm": 0.6469123363494873, + "learning_rate": 0.0001, + "loss": 59.2639, + "step": 19240 + }, + { + "epoch": 30.999647301859223, + "grad_norm": 0.8151462078094482, + "learning_rate": 0.0001, + "loss": 58.5958, + "step": 19250 + }, + { + "epoch": 31.014511009220538, + "grad_norm": 0.6884656548500061, + "learning_rate": 0.0001, + "loss": 52.5185, + "step": 19260 + }, + { + "epoch": 31.030634352798913, + "grad_norm": 0.7065051794052124, + "learning_rate": 0.0001, + "loss": 55.8415, + "step": 19270 + }, + { + "epoch": 31.046757696377288, + "grad_norm": 0.5404031872749329, + "learning_rate": 0.0001, + "loss": 58.5464, + "step": 19280 + }, + { + "epoch": 31.06288103995566, + "grad_norm": 0.7152573466300964, + "learning_rate": 0.0001, + "loss": 57.2573, + "step": 19290 + }, + { + "epoch": 31.079004383534034, + "grad_norm": 0.7122138142585754, + "learning_rate": 0.0001, + "loss": 59.6534, + "step": 19300 + }, + { + "epoch": 31.09512772711241, + "grad_norm": 0.633294403553009, + "learning_rate": 0.0001, + "loss": 56.2421, + "step": 19310 + }, + { + "epoch": 31.111251070690784, + "grad_norm": 0.5658999085426331, + "learning_rate": 0.0001, + "loss": 57.4885, + "step": 19320 + }, + { + "epoch": 31.12737441426916, + "grad_norm": 0.5926643013954163, + "learning_rate": 0.0001, + "loss": 58.7341, + "step": 19330 + }, + { + "epoch": 31.143497757847534, + "grad_norm": 0.6227424144744873, + "learning_rate": 0.0001, + "loss": 55.9883, + "step": 19340 + }, + { + "epoch": 31.15962110142591, + "grad_norm": 0.6378360986709595, + "learning_rate": 0.0001, + "loss": 57.8796, + "step": 19350 + }, + { + "epoch": 31.175744445004284, + "grad_norm": 0.7493644952774048, + "learning_rate": 0.0001, + "loss": 58.0501, + "step": 19360 + }, + { + "epoch": 31.19186778858266, + "grad_norm": 0.7071518898010254, + "learning_rate": 0.0001, + "loss": 57.9335, + "step": 19370 + }, + { + "epoch": 31.20799113216103, + "grad_norm": 0.6761748194694519, + "learning_rate": 0.0001, + "loss": 56.7835, + "step": 19380 + }, + { + "epoch": 31.224114475739405, + "grad_norm": 0.49391838908195496, + "learning_rate": 0.0001, + "loss": 57.4453, + "step": 19390 + }, + { + "epoch": 31.24023781931778, + "grad_norm": 0.574937641620636, + "learning_rate": 0.0001, + "loss": 60.3609, + "step": 19400 + }, + { + "epoch": 31.256361162896155, + "grad_norm": 0.6177113652229309, + "learning_rate": 0.0001, + "loss": 56.4432, + "step": 19410 + }, + { + "epoch": 31.27248450647453, + "grad_norm": 0.6551643013954163, + "learning_rate": 0.0001, + "loss": 57.0349, + "step": 19420 + }, + { + "epoch": 31.288607850052905, + "grad_norm": 0.5895516872406006, + "learning_rate": 0.0001, + "loss": 58.0312, + "step": 19430 + }, + { + "epoch": 31.30473119363128, + "grad_norm": 0.629224419593811, + "learning_rate": 0.0001, + "loss": 59.2731, + "step": 19440 + }, + { + "epoch": 31.320854537209655, + "grad_norm": 0.6345664262771606, + "learning_rate": 0.0001, + "loss": 61.5289, + "step": 19450 + }, + { + "epoch": 31.33697788078803, + "grad_norm": 0.6183397173881531, + "learning_rate": 0.0001, + "loss": 58.7228, + "step": 19460 + }, + { + "epoch": 31.3531012243664, + "grad_norm": 0.6631211638450623, + "learning_rate": 0.0001, + "loss": 58.3999, + "step": 19470 + }, + { + "epoch": 31.369224567944777, + "grad_norm": 0.698853611946106, + "learning_rate": 0.0001, + "loss": 59.6244, + "step": 19480 + }, + { + "epoch": 31.38534791152315, + "grad_norm": 0.6930462718009949, + "learning_rate": 0.0001, + "loss": 58.261, + "step": 19490 + }, + { + "epoch": 31.401471255101526, + "grad_norm": 0.6169912219047546, + "learning_rate": 0.0001, + "loss": 56.6868, + "step": 19500 + }, + { + "epoch": 31.4175945986799, + "grad_norm": 0.8206812739372253, + "learning_rate": 0.0001, + "loss": 57.3885, + "step": 19510 + }, + { + "epoch": 31.433717942258276, + "grad_norm": 0.7076541185379028, + "learning_rate": 0.0001, + "loss": 56.7746, + "step": 19520 + }, + { + "epoch": 31.44984128583665, + "grad_norm": 0.548710286617279, + "learning_rate": 0.0001, + "loss": 57.2149, + "step": 19530 + }, + { + "epoch": 31.465964629415026, + "grad_norm": 0.6735765337944031, + "learning_rate": 0.0001, + "loss": 58.8017, + "step": 19540 + }, + { + "epoch": 31.482087972993398, + "grad_norm": 0.6010057330131531, + "learning_rate": 0.0001, + "loss": 59.166, + "step": 19550 + }, + { + "epoch": 31.498211316571773, + "grad_norm": 0.5643196105957031, + "learning_rate": 0.0001, + "loss": 56.2109, + "step": 19560 + }, + { + "epoch": 31.514334660150148, + "grad_norm": 0.7801907062530518, + "learning_rate": 0.0001, + "loss": 58.0577, + "step": 19570 + }, + { + "epoch": 31.530458003728523, + "grad_norm": 0.630387544631958, + "learning_rate": 0.0001, + "loss": 56.1545, + "step": 19580 + }, + { + "epoch": 31.546581347306898, + "grad_norm": 0.6961885094642639, + "learning_rate": 0.0001, + "loss": 57.8717, + "step": 19590 + }, + { + "epoch": 31.562704690885273, + "grad_norm": 0.5340575575828552, + "learning_rate": 0.0001, + "loss": 58.2695, + "step": 19600 + }, + { + "epoch": 31.578828034463648, + "grad_norm": 0.657589316368103, + "learning_rate": 0.0001, + "loss": 59.3382, + "step": 19610 + }, + { + "epoch": 31.594951378042023, + "grad_norm": 0.6341890096664429, + "learning_rate": 0.0001, + "loss": 58.159, + "step": 19620 + }, + { + "epoch": 31.611074721620398, + "grad_norm": 0.7497854828834534, + "learning_rate": 0.0001, + "loss": 55.7069, + "step": 19630 + }, + { + "epoch": 31.62719806519877, + "grad_norm": 0.7069994211196899, + "learning_rate": 0.0001, + "loss": 57.2228, + "step": 19640 + }, + { + "epoch": 31.643321408777144, + "grad_norm": 0.7523748278617859, + "learning_rate": 0.0001, + "loss": 59.5134, + "step": 19650 + }, + { + "epoch": 31.65944475235552, + "grad_norm": 0.6233903169631958, + "learning_rate": 0.0001, + "loss": 58.5135, + "step": 19660 + }, + { + "epoch": 31.675568095933894, + "grad_norm": 0.6637253761291504, + "learning_rate": 0.0001, + "loss": 58.1763, + "step": 19670 + }, + { + "epoch": 31.69169143951227, + "grad_norm": 0.8216729760169983, + "learning_rate": 0.0001, + "loss": 60.0401, + "step": 19680 + }, + { + "epoch": 31.707814783090644, + "grad_norm": 0.534037709236145, + "learning_rate": 0.0001, + "loss": 57.4307, + "step": 19690 + }, + { + "epoch": 31.72393812666902, + "grad_norm": 0.5953571200370789, + "learning_rate": 0.0001, + "loss": 59.8216, + "step": 19700 + }, + { + "epoch": 31.740061470247394, + "grad_norm": 0.5853090286254883, + "learning_rate": 0.0001, + "loss": 58.2106, + "step": 19710 + }, + { + "epoch": 31.75618481382577, + "grad_norm": 0.7370662689208984, + "learning_rate": 0.0001, + "loss": 56.6751, + "step": 19720 + }, + { + "epoch": 31.77230815740414, + "grad_norm": 0.6396768689155579, + "learning_rate": 0.0001, + "loss": 55.3496, + "step": 19730 + }, + { + "epoch": 31.788431500982515, + "grad_norm": 0.7257981896400452, + "learning_rate": 0.0001, + "loss": 56.8753, + "step": 19740 + }, + { + "epoch": 31.80455484456089, + "grad_norm": 0.623684287071228, + "learning_rate": 0.0001, + "loss": 58.8956, + "step": 19750 + }, + { + "epoch": 31.820678188139265, + "grad_norm": 0.680181622505188, + "learning_rate": 0.0001, + "loss": 58.0901, + "step": 19760 + }, + { + "epoch": 31.83680153171764, + "grad_norm": 0.6862589120864868, + "learning_rate": 0.0001, + "loss": 55.7762, + "step": 19770 + }, + { + "epoch": 31.852924875296015, + "grad_norm": 0.5723304152488708, + "learning_rate": 0.0001, + "loss": 57.1686, + "step": 19780 + }, + { + "epoch": 31.86904821887439, + "grad_norm": 0.7448561787605286, + "learning_rate": 0.0001, + "loss": 57.7282, + "step": 19790 + }, + { + "epoch": 31.885171562452765, + "grad_norm": 0.6579545736312866, + "learning_rate": 0.0001, + "loss": 59.0573, + "step": 19800 + }, + { + "epoch": 31.90129490603114, + "grad_norm": 0.7359026670455933, + "learning_rate": 0.0001, + "loss": 56.4827, + "step": 19810 + }, + { + "epoch": 31.91741824960951, + "grad_norm": 0.5813704133033752, + "learning_rate": 0.0001, + "loss": 57.2898, + "step": 19820 + }, + { + "epoch": 31.933541593187886, + "grad_norm": 0.8108206987380981, + "learning_rate": 0.0001, + "loss": 59.701, + "step": 19830 + }, + { + "epoch": 31.94966493676626, + "grad_norm": 0.7966176271438599, + "learning_rate": 0.0001, + "loss": 58.4976, + "step": 19840 + }, + { + "epoch": 31.965788280344636, + "grad_norm": 0.7988247871398926, + "learning_rate": 0.0001, + "loss": 58.5399, + "step": 19850 + }, + { + "epoch": 31.98191162392301, + "grad_norm": 0.7580775022506714, + "learning_rate": 0.0001, + "loss": 59.3756, + "step": 19860 + }, + { + "epoch": 31.998034967501386, + "grad_norm": 0.6985690593719482, + "learning_rate": 0.0001, + "loss": 58.5244, + "step": 19870 + }, + { + "epoch": 32.0128986748627, + "grad_norm": 0.5899099111557007, + "learning_rate": 0.0001, + "loss": 51.9032, + "step": 19880 + }, + { + "epoch": 32.029022018441076, + "grad_norm": 0.5154966115951538, + "learning_rate": 0.0001, + "loss": 57.8494, + "step": 19890 + }, + { + "epoch": 32.04514536201945, + "grad_norm": 0.7414538860321045, + "learning_rate": 0.0001, + "loss": 60.1024, + "step": 19900 + }, + { + "epoch": 32.061268705597826, + "grad_norm": 0.6928898692131042, + "learning_rate": 0.0001, + "loss": 58.2986, + "step": 19910 + }, + { + "epoch": 32.0773920491762, + "grad_norm": 0.6019315719604492, + "learning_rate": 0.0001, + "loss": 58.7716, + "step": 19920 + }, + { + "epoch": 32.093515392754576, + "grad_norm": 0.5992019176483154, + "learning_rate": 0.0001, + "loss": 56.7196, + "step": 19930 + }, + { + "epoch": 32.10963873633295, + "grad_norm": 0.8731210231781006, + "learning_rate": 0.0001, + "loss": 58.7054, + "step": 19940 + }, + { + "epoch": 32.12576207991132, + "grad_norm": 0.6548051238059998, + "learning_rate": 0.0001, + "loss": 57.9742, + "step": 19950 + }, + { + "epoch": 32.1418854234897, + "grad_norm": 0.6520991325378418, + "learning_rate": 0.0001, + "loss": 58.1439, + "step": 19960 + }, + { + "epoch": 32.15800876706807, + "grad_norm": 0.7181893587112427, + "learning_rate": 0.0001, + "loss": 58.985, + "step": 19970 + }, + { + "epoch": 32.17413211064645, + "grad_norm": 0.7861577272415161, + "learning_rate": 0.0001, + "loss": 59.0184, + "step": 19980 + }, + { + "epoch": 32.19025545422482, + "grad_norm": 2.1008453369140625, + "learning_rate": 0.0001, + "loss": 57.6168, + "step": 19990 + }, + { + "epoch": 32.2063787978032, + "grad_norm": 0.7860429883003235, + "learning_rate": 0.0001, + "loss": 57.1095, + "step": 20000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 33, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.4460635761788165e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ad66941d8a4a859aa18e8017cb879260493845ad --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d36614926d6321b92164636079011a7236956d2d61a604c0ebb3100bc78860fa +size 6904 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-20001/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/latest new file mode 100644 index 0000000000000000000000000000000000000000..641c40ce8d5e05e04a0840d69bcc5fac045a3fbd --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/latest @@ -0,0 +1 @@ +global_step4000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cefbb0a5bc065ba27bb35e7de82bcd43015b3f86 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:153cb19c1c4d4615048123cc53826452b955c5ab523f443dc386933a5d1228ac +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..4172ddb00d43835bb3c9013b4f4ba2b414d1422c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d935e1a302c94bd0fd4be649c1f5f831b93109e4a4624a1ba29e993f03203d9 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..273a4a11eaa48d79d41ac721e04ecde0d51173e5 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bc565701968b8415df09405eb53409599e5963811749750d8145a02bf6d3e67 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..2e6c093b6f81df17125a8ae8d500a6fbbfff4126 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aff4820ead94ce7dffec9597ea6df788b93400da9a1f260e05b42236c00b903 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..6a0235d3de5dd5a9e9001731951010d5e13632ab --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c8adc4b49eca603a74f1add0c0c8ca899b05fcbf865a8439ab0fda45e6de7ce +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..da202be809acaefeef1fcd3c1ce30546914c04e5 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f4b47361d1a8b7dae47f1d9ba9c46bdfb213cdd1ee3365ab7cf267c4dc6c58f +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..4d8b7851c84369aef2a1324fc4c0f62599bde115 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5707aca801b27c5114f1ab0f0ae64faaf8faa201cae2300abc25de5de74fcfca +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..bb020032026fbb682320905e4711d6c10eb9487b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d4c8b0202a3f260408001bb0c379e5941dd4737e2903898fa290c8fc52f57a5 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..cb3650ec06bdfe67bfadc9b27f73fed6a37e7161 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5025fcfbea96910f51826f9721807cff5bf67459f9ed0e26e8b81f3b0b970a8a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7c33095320ec6a7f585e68e4e02ed022aa80652 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e88d5675c3e7af2f5bef04cb9622f8a630e6e25147b9f39eb466dd8ba603a69a +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..84719b92429369ed4bd3d744b0762331e12c5bf7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/trainer_state.json @@ -0,0 +1,2834 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.441779614047463, + "eval_steps": 2000, + "global_step": 4000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.016123343578374565, + "grad_norm": 48.11856460571289, + "learning_rate": 0.0001, + "loss": 299.1088, + "step": 10 + }, + { + "epoch": 0.03224668715674913, + "grad_norm": 9.07711410522461, + "learning_rate": 0.0001, + "loss": 234.0888, + "step": 20 + }, + { + "epoch": 0.0483700307351237, + "grad_norm": 3.5935330390930176, + "learning_rate": 0.0001, + "loss": 231.4619, + "step": 30 + }, + { + "epoch": 0.06449337431349826, + "grad_norm": 3.5784752368927, + "learning_rate": 0.0001, + "loss": 229.7082, + "step": 40 + }, + { + "epoch": 0.08061671789187283, + "grad_norm": 3.5757956504821777, + "learning_rate": 0.0001, + "loss": 230.1062, + "step": 50 + }, + { + "epoch": 0.0967400614702474, + "grad_norm": 3.8763978481292725, + "learning_rate": 0.0001, + "loss": 229.1914, + "step": 60 + }, + { + "epoch": 0.11286340504862195, + "grad_norm": 3.006988525390625, + "learning_rate": 0.0001, + "loss": 229.6482, + "step": 70 + }, + { + "epoch": 0.12898674862699652, + "grad_norm": 2.3527095317840576, + "learning_rate": 0.0001, + "loss": 228.1141, + "step": 80 + }, + { + "epoch": 0.14511009220537108, + "grad_norm": 5.353086471557617, + "learning_rate": 0.0001, + "loss": 228.1448, + "step": 90 + }, + { + "epoch": 0.16123343578374566, + "grad_norm": 2.251854419708252, + "learning_rate": 0.0001, + "loss": 228.3897, + "step": 100 + }, + { + "epoch": 0.17735677936212021, + "grad_norm": 1.5789915323257446, + "learning_rate": 0.0001, + "loss": 227.7662, + "step": 110 + }, + { + "epoch": 0.1934801229404948, + "grad_norm": 3.7646398544311523, + "learning_rate": 0.0001, + "loss": 223.6411, + "step": 120 + }, + { + "epoch": 0.20960346651886935, + "grad_norm": 3.444000720977783, + "learning_rate": 0.0001, + "loss": 221.2073, + "step": 130 + }, + { + "epoch": 0.2257268100972439, + "grad_norm": 2.1871185302734375, + "learning_rate": 0.0001, + "loss": 215.9944, + "step": 140 + }, + { + "epoch": 0.2418501536756185, + "grad_norm": 2.266244411468506, + "learning_rate": 0.0001, + "loss": 212.3066, + "step": 150 + }, + { + "epoch": 0.25797349725399304, + "grad_norm": 2.043832778930664, + "learning_rate": 0.0001, + "loss": 208.1343, + "step": 160 + }, + { + "epoch": 0.2740968408323676, + "grad_norm": 2.2641494274139404, + "learning_rate": 0.0001, + "loss": 203.1485, + "step": 170 + }, + { + "epoch": 0.29022018441074215, + "grad_norm": 2.9727790355682373, + "learning_rate": 0.0001, + "loss": 198.181, + "step": 180 + }, + { + "epoch": 0.30634352798911674, + "grad_norm": 2.751523494720459, + "learning_rate": 0.0001, + "loss": 194.8422, + "step": 190 + }, + { + "epoch": 0.3224668715674913, + "grad_norm": 1.9561562538146973, + "learning_rate": 0.0001, + "loss": 189.9401, + "step": 200 + }, + { + "epoch": 0.33859021514586585, + "grad_norm": 2.3264107704162598, + "learning_rate": 0.0001, + "loss": 186.1309, + "step": 210 + }, + { + "epoch": 0.35471355872424043, + "grad_norm": 5.212426662445068, + "learning_rate": 0.0001, + "loss": 182.2255, + "step": 220 + }, + { + "epoch": 0.370836902302615, + "grad_norm": 2.488032102584839, + "learning_rate": 0.0001, + "loss": 181.0103, + "step": 230 + }, + { + "epoch": 0.3869602458809896, + "grad_norm": 2.6173202991485596, + "learning_rate": 0.0001, + "loss": 177.6162, + "step": 240 + }, + { + "epoch": 0.4030835894593641, + "grad_norm": 2.6245291233062744, + "learning_rate": 0.0001, + "loss": 173.0431, + "step": 250 + }, + { + "epoch": 0.4192069330377387, + "grad_norm": 5.356006145477295, + "learning_rate": 0.0001, + "loss": 172.2371, + "step": 260 + }, + { + "epoch": 0.4353302766161133, + "grad_norm": 2.0453829765319824, + "learning_rate": 0.0001, + "loss": 169.7781, + "step": 270 + }, + { + "epoch": 0.4514536201944878, + "grad_norm": 1.7002843618392944, + "learning_rate": 0.0001, + "loss": 166.9564, + "step": 280 + }, + { + "epoch": 0.4675769637728624, + "grad_norm": 2.181410312652588, + "learning_rate": 0.0001, + "loss": 166.0094, + "step": 290 + }, + { + "epoch": 0.483700307351237, + "grad_norm": 2.2948265075683594, + "learning_rate": 0.0001, + "loss": 162.4388, + "step": 300 + }, + { + "epoch": 0.4998236509296115, + "grad_norm": 3.0272789001464844, + "learning_rate": 0.0001, + "loss": 160.3446, + "step": 310 + }, + { + "epoch": 0.5159469945079861, + "grad_norm": 1.948184847831726, + "learning_rate": 0.0001, + "loss": 158.488, + "step": 320 + }, + { + "epoch": 0.5320703380863606, + "grad_norm": 1.7981538772583008, + "learning_rate": 0.0001, + "loss": 156.2107, + "step": 330 + }, + { + "epoch": 0.5481936816647353, + "grad_norm": 2.049323797225952, + "learning_rate": 0.0001, + "loss": 155.5434, + "step": 340 + }, + { + "epoch": 0.5643170252431098, + "grad_norm": 1.9415442943572998, + "learning_rate": 0.0001, + "loss": 153.8682, + "step": 350 + }, + { + "epoch": 0.5804403688214843, + "grad_norm": 2.2168045043945312, + "learning_rate": 0.0001, + "loss": 150.4087, + "step": 360 + }, + { + "epoch": 0.596563712399859, + "grad_norm": 1.510750651359558, + "learning_rate": 0.0001, + "loss": 151.7655, + "step": 370 + }, + { + "epoch": 0.6126870559782335, + "grad_norm": 2.7469751834869385, + "learning_rate": 0.0001, + "loss": 147.4665, + "step": 380 + }, + { + "epoch": 0.628810399556608, + "grad_norm": 2.0436975955963135, + "learning_rate": 0.0001, + "loss": 147.4405, + "step": 390 + }, + { + "epoch": 0.6449337431349826, + "grad_norm": 1.8219181299209595, + "learning_rate": 0.0001, + "loss": 146.6332, + "step": 400 + }, + { + "epoch": 0.6610570867133572, + "grad_norm": 1.948431134223938, + "learning_rate": 0.0001, + "loss": 143.3298, + "step": 410 + }, + { + "epoch": 0.6771804302917317, + "grad_norm": 1.8887908458709717, + "learning_rate": 0.0001, + "loss": 140.5058, + "step": 420 + }, + { + "epoch": 0.6933037738701063, + "grad_norm": 1.8739320039749146, + "learning_rate": 0.0001, + "loss": 137.6334, + "step": 430 + }, + { + "epoch": 0.7094271174484809, + "grad_norm": 2.057689666748047, + "learning_rate": 0.0001, + "loss": 135.0343, + "step": 440 + }, + { + "epoch": 0.7255504610268555, + "grad_norm": 1.6926478147506714, + "learning_rate": 0.0001, + "loss": 132.8766, + "step": 450 + }, + { + "epoch": 0.74167380460523, + "grad_norm": 1.5039501190185547, + "learning_rate": 0.0001, + "loss": 131.6774, + "step": 460 + }, + { + "epoch": 0.7577971481836046, + "grad_norm": 1.6903547048568726, + "learning_rate": 0.0001, + "loss": 129.3116, + "step": 470 + }, + { + "epoch": 0.7739204917619792, + "grad_norm": 3.7062244415283203, + "learning_rate": 0.0001, + "loss": 127.9314, + "step": 480 + }, + { + "epoch": 0.7900438353403537, + "grad_norm": 2.0586934089660645, + "learning_rate": 0.0001, + "loss": 126.6398, + "step": 490 + }, + { + "epoch": 0.8061671789187282, + "grad_norm": 1.7598215341567993, + "learning_rate": 0.0001, + "loss": 125.1501, + "step": 500 + }, + { + "epoch": 0.8222905224971029, + "grad_norm": 1.5992189645767212, + "learning_rate": 0.0001, + "loss": 124.7039, + "step": 510 + }, + { + "epoch": 0.8384138660754774, + "grad_norm": 2.8735458850860596, + "learning_rate": 0.0001, + "loss": 122.1619, + "step": 520 + }, + { + "epoch": 0.8545372096538519, + "grad_norm": 1.867098093032837, + "learning_rate": 0.0001, + "loss": 122.4506, + "step": 530 + }, + { + "epoch": 0.8706605532322266, + "grad_norm": 2.2088418006896973, + "learning_rate": 0.0001, + "loss": 122.1047, + "step": 540 + }, + { + "epoch": 0.8867838968106011, + "grad_norm": 1.642657995223999, + "learning_rate": 0.0001, + "loss": 120.1141, + "step": 550 + }, + { + "epoch": 0.9029072403889756, + "grad_norm": 1.5567114353179932, + "learning_rate": 0.0001, + "loss": 118.532, + "step": 560 + }, + { + "epoch": 0.9190305839673503, + "grad_norm": 1.667843222618103, + "learning_rate": 0.0001, + "loss": 118.0734, + "step": 570 + }, + { + "epoch": 0.9351539275457248, + "grad_norm": 1.5304442644119263, + "learning_rate": 0.0001, + "loss": 114.4925, + "step": 580 + }, + { + "epoch": 0.9512772711240993, + "grad_norm": 1.6920255422592163, + "learning_rate": 0.0001, + "loss": 114.0729, + "step": 590 + }, + { + "epoch": 0.967400614702474, + "grad_norm": 1.6313318014144897, + "learning_rate": 0.0001, + "loss": 114.2879, + "step": 600 + }, + { + "epoch": 0.9835239582808485, + "grad_norm": 1.9033657312393188, + "learning_rate": 0.0001, + "loss": 110.5299, + "step": 610 + }, + { + "epoch": 0.999647301859223, + "grad_norm": 1.9010754823684692, + "learning_rate": 0.0001, + "loss": 109.1145, + "step": 620 + }, + { + "epoch": 1.0145110092205372, + "grad_norm": 3.5390989780426025, + "learning_rate": 0.0001, + "loss": 101.0258, + "step": 630 + }, + { + "epoch": 1.0306343527989117, + "grad_norm": 2.2925171852111816, + "learning_rate": 0.0001, + "loss": 107.3693, + "step": 640 + }, + { + "epoch": 1.0467576963772862, + "grad_norm": 2.329108238220215, + "learning_rate": 0.0001, + "loss": 104.3778, + "step": 650 + }, + { + "epoch": 1.0628810399556607, + "grad_norm": 1.7325661182403564, + "learning_rate": 0.0001, + "loss": 102.8409, + "step": 660 + }, + { + "epoch": 1.0790043835340353, + "grad_norm": 1.989586353302002, + "learning_rate": 0.0001, + "loss": 104.2577, + "step": 670 + }, + { + "epoch": 1.09512772711241, + "grad_norm": 1.900484323501587, + "learning_rate": 0.0001, + "loss": 102.3051, + "step": 680 + }, + { + "epoch": 1.1112510706907845, + "grad_norm": 2.015859842300415, + "learning_rate": 0.0001, + "loss": 100.893, + "step": 690 + }, + { + "epoch": 1.127374414269159, + "grad_norm": 1.7695508003234863, + "learning_rate": 0.0001, + "loss": 99.215, + "step": 700 + }, + { + "epoch": 1.1434977578475336, + "grad_norm": 2.025923490524292, + "learning_rate": 0.0001, + "loss": 98.8768, + "step": 710 + }, + { + "epoch": 1.1596211014259081, + "grad_norm": 2.946708917617798, + "learning_rate": 0.0001, + "loss": 96.5588, + "step": 720 + }, + { + "epoch": 1.1757444450042827, + "grad_norm": 2.4511258602142334, + "learning_rate": 0.0001, + "loss": 95.0545, + "step": 730 + }, + { + "epoch": 1.1918677885826574, + "grad_norm": 2.3094682693481445, + "learning_rate": 0.0001, + "loss": 97.004, + "step": 740 + }, + { + "epoch": 1.207991132161032, + "grad_norm": 2.1313347816467285, + "learning_rate": 0.0001, + "loss": 94.8819, + "step": 750 + }, + { + "epoch": 1.2241144757394065, + "grad_norm": 1.614182710647583, + "learning_rate": 0.0001, + "loss": 95.5339, + "step": 760 + }, + { + "epoch": 1.240237819317781, + "grad_norm": 2.029953956604004, + "learning_rate": 0.0001, + "loss": 93.7303, + "step": 770 + }, + { + "epoch": 1.2563611628961555, + "grad_norm": 1.7871663570404053, + "learning_rate": 0.0001, + "loss": 93.8354, + "step": 780 + }, + { + "epoch": 1.2724845064745303, + "grad_norm": 1.72455632686615, + "learning_rate": 0.0001, + "loss": 94.7348, + "step": 790 + }, + { + "epoch": 1.2886078500529048, + "grad_norm": 1.9776639938354492, + "learning_rate": 0.0001, + "loss": 93.5957, + "step": 800 + }, + { + "epoch": 1.3047311936312793, + "grad_norm": 1.7187533378601074, + "learning_rate": 0.0001, + "loss": 92.9583, + "step": 810 + }, + { + "epoch": 1.3208545372096538, + "grad_norm": 1.5949162244796753, + "learning_rate": 0.0001, + "loss": 93.3088, + "step": 820 + }, + { + "epoch": 1.3369778807880284, + "grad_norm": 1.4616206884384155, + "learning_rate": 0.0001, + "loss": 93.2125, + "step": 830 + }, + { + "epoch": 1.353101224366403, + "grad_norm": 2.832163095474243, + "learning_rate": 0.0001, + "loss": 90.3301, + "step": 840 + }, + { + "epoch": 1.3692245679447774, + "grad_norm": 2.174501419067383, + "learning_rate": 0.0001, + "loss": 91.6988, + "step": 850 + }, + { + "epoch": 1.3853479115231522, + "grad_norm": 1.2156339883804321, + "learning_rate": 0.0001, + "loss": 92.7419, + "step": 860 + }, + { + "epoch": 1.4014712551015267, + "grad_norm": 1.608261227607727, + "learning_rate": 0.0001, + "loss": 89.8255, + "step": 870 + }, + { + "epoch": 1.4175945986799012, + "grad_norm": 1.6671923398971558, + "learning_rate": 0.0001, + "loss": 90.9683, + "step": 880 + }, + { + "epoch": 1.4337179422582758, + "grad_norm": 1.2346433401107788, + "learning_rate": 0.0001, + "loss": 90.6092, + "step": 890 + }, + { + "epoch": 1.4498412858366505, + "grad_norm": 1.561085820198059, + "learning_rate": 0.0001, + "loss": 88.0842, + "step": 900 + }, + { + "epoch": 1.465964629415025, + "grad_norm": 1.5475201606750488, + "learning_rate": 0.0001, + "loss": 89.8786, + "step": 910 + }, + { + "epoch": 1.4820879729933996, + "grad_norm": 1.6044800281524658, + "learning_rate": 0.0001, + "loss": 88.0364, + "step": 920 + }, + { + "epoch": 1.498211316571774, + "grad_norm": 1.4081908464431763, + "learning_rate": 0.0001, + "loss": 88.8799, + "step": 930 + }, + { + "epoch": 1.5143346601501486, + "grad_norm": 2.341053009033203, + "learning_rate": 0.0001, + "loss": 88.2023, + "step": 940 + }, + { + "epoch": 1.5304580037285231, + "grad_norm": 1.3976799249649048, + "learning_rate": 0.0001, + "loss": 86.6493, + "step": 950 + }, + { + "epoch": 1.5465813473068977, + "grad_norm": 1.2485483884811401, + "learning_rate": 0.0001, + "loss": 88.9224, + "step": 960 + }, + { + "epoch": 1.5627046908852722, + "grad_norm": 1.6824235916137695, + "learning_rate": 0.0001, + "loss": 89.6484, + "step": 970 + }, + { + "epoch": 1.578828034463647, + "grad_norm": 1.64790940284729, + "learning_rate": 0.0001, + "loss": 87.2479, + "step": 980 + }, + { + "epoch": 1.5949513780420215, + "grad_norm": 1.2093443870544434, + "learning_rate": 0.0001, + "loss": 86.8241, + "step": 990 + }, + { + "epoch": 1.611074721620396, + "grad_norm": 1.5306812524795532, + "learning_rate": 0.0001, + "loss": 88.0006, + "step": 1000 + }, + { + "epoch": 1.6271980651987707, + "grad_norm": 1.2465763092041016, + "learning_rate": 0.0001, + "loss": 86.8143, + "step": 1010 + }, + { + "epoch": 1.6433214087771453, + "grad_norm": 1.5224069356918335, + "learning_rate": 0.0001, + "loss": 87.0102, + "step": 1020 + }, + { + "epoch": 1.6594447523555198, + "grad_norm": 1.312179684638977, + "learning_rate": 0.0001, + "loss": 88.7395, + "step": 1030 + }, + { + "epoch": 1.6755680959338943, + "grad_norm": 1.2343158721923828, + "learning_rate": 0.0001, + "loss": 87.2868, + "step": 1040 + }, + { + "epoch": 1.6916914395122689, + "grad_norm": 1.2778265476226807, + "learning_rate": 0.0001, + "loss": 87.043, + "step": 1050 + }, + { + "epoch": 1.7078147830906434, + "grad_norm": 1.2799627780914307, + "learning_rate": 0.0001, + "loss": 87.7843, + "step": 1060 + }, + { + "epoch": 1.723938126669018, + "grad_norm": 1.3688952922821045, + "learning_rate": 0.0001, + "loss": 85.8985, + "step": 1070 + }, + { + "epoch": 1.7400614702473924, + "grad_norm": 1.4317796230316162, + "learning_rate": 0.0001, + "loss": 85.5408, + "step": 1080 + }, + { + "epoch": 1.756184813825767, + "grad_norm": 1.164801001548767, + "learning_rate": 0.0001, + "loss": 85.8484, + "step": 1090 + }, + { + "epoch": 1.7723081574041417, + "grad_norm": 1.3929173946380615, + "learning_rate": 0.0001, + "loss": 86.9031, + "step": 1100 + }, + { + "epoch": 1.7884315009825162, + "grad_norm": 2.116119861602783, + "learning_rate": 0.0001, + "loss": 86.5343, + "step": 1110 + }, + { + "epoch": 1.8045548445608908, + "grad_norm": 1.4117276668548584, + "learning_rate": 0.0001, + "loss": 85.2678, + "step": 1120 + }, + { + "epoch": 1.8206781881392655, + "grad_norm": 1.3213504552841187, + "learning_rate": 0.0001, + "loss": 85.8935, + "step": 1130 + }, + { + "epoch": 1.83680153171764, + "grad_norm": 1.3589770793914795, + "learning_rate": 0.0001, + "loss": 85.032, + "step": 1140 + }, + { + "epoch": 1.8529248752960146, + "grad_norm": 1.3336007595062256, + "learning_rate": 0.0001, + "loss": 84.0993, + "step": 1150 + }, + { + "epoch": 1.869048218874389, + "grad_norm": 1.1081451177597046, + "learning_rate": 0.0001, + "loss": 84.2225, + "step": 1160 + }, + { + "epoch": 1.8851715624527636, + "grad_norm": 1.482038974761963, + "learning_rate": 0.0001, + "loss": 82.1439, + "step": 1170 + }, + { + "epoch": 1.9012949060311382, + "grad_norm": 1.1571478843688965, + "learning_rate": 0.0001, + "loss": 82.5045, + "step": 1180 + }, + { + "epoch": 1.9174182496095127, + "grad_norm": 1.2665468454360962, + "learning_rate": 0.0001, + "loss": 82.8761, + "step": 1190 + }, + { + "epoch": 1.9335415931878872, + "grad_norm": 1.0771509408950806, + "learning_rate": 0.0001, + "loss": 81.5162, + "step": 1200 + }, + { + "epoch": 1.949664936766262, + "grad_norm": 1.2992299795150757, + "learning_rate": 0.0001, + "loss": 82.4038, + "step": 1210 + }, + { + "epoch": 1.9657882803446365, + "grad_norm": 1.2946668863296509, + "learning_rate": 0.0001, + "loss": 83.4837, + "step": 1220 + }, + { + "epoch": 1.981911623923011, + "grad_norm": 1.3505686521530151, + "learning_rate": 0.0001, + "loss": 81.8993, + "step": 1230 + }, + { + "epoch": 1.9980349675013858, + "grad_norm": 1.1247715950012207, + "learning_rate": 0.0001, + "loss": 82.9006, + "step": 1240 + }, + { + "epoch": 2.0128986748627, + "grad_norm": 1.5652127265930176, + "learning_rate": 0.0001, + "loss": 75.8347, + "step": 1250 + }, + { + "epoch": 2.0290220184410743, + "grad_norm": 7.311964988708496, + "learning_rate": 0.0001, + "loss": 80.5435, + "step": 1260 + }, + { + "epoch": 2.045145362019449, + "grad_norm": 1.2229893207550049, + "learning_rate": 0.0001, + "loss": 83.3277, + "step": 1270 + }, + { + "epoch": 2.0612687055978234, + "grad_norm": 1.1387693881988525, + "learning_rate": 0.0001, + "loss": 81.3252, + "step": 1280 + }, + { + "epoch": 2.077392049176198, + "grad_norm": 1.307201623916626, + "learning_rate": 0.0001, + "loss": 82.3311, + "step": 1290 + }, + { + "epoch": 2.0935153927545724, + "grad_norm": 1.1502896547317505, + "learning_rate": 0.0001, + "loss": 81.1682, + "step": 1300 + }, + { + "epoch": 2.109638736332947, + "grad_norm": 0.9291583299636841, + "learning_rate": 0.0001, + "loss": 81.7017, + "step": 1310 + }, + { + "epoch": 2.1257620799113215, + "grad_norm": 1.0334666967391968, + "learning_rate": 0.0001, + "loss": 81.718, + "step": 1320 + }, + { + "epoch": 2.141885423489696, + "grad_norm": 1.1099056005477905, + "learning_rate": 0.0001, + "loss": 81.2232, + "step": 1330 + }, + { + "epoch": 2.1580087670680705, + "grad_norm": 1.289478063583374, + "learning_rate": 0.0001, + "loss": 81.8736, + "step": 1340 + }, + { + "epoch": 2.174132110646445, + "grad_norm": 1.1634794473648071, + "learning_rate": 0.0001, + "loss": 79.9434, + "step": 1350 + }, + { + "epoch": 2.19025545422482, + "grad_norm": 0.9563440680503845, + "learning_rate": 0.0001, + "loss": 80.628, + "step": 1360 + }, + { + "epoch": 2.2063787978031946, + "grad_norm": 1.1268844604492188, + "learning_rate": 0.0001, + "loss": 82.1389, + "step": 1370 + }, + { + "epoch": 2.222502141381569, + "grad_norm": 1.1050443649291992, + "learning_rate": 0.0001, + "loss": 78.9023, + "step": 1380 + }, + { + "epoch": 2.2386254849599436, + "grad_norm": 1.1936142444610596, + "learning_rate": 0.0001, + "loss": 81.2118, + "step": 1390 + }, + { + "epoch": 2.254748828538318, + "grad_norm": 1.7825465202331543, + "learning_rate": 0.0001, + "loss": 80.8804, + "step": 1400 + }, + { + "epoch": 2.2708721721166927, + "grad_norm": 2.437251329421997, + "learning_rate": 0.0001, + "loss": 79.3807, + "step": 1410 + }, + { + "epoch": 2.286995515695067, + "grad_norm": 1.0604854822158813, + "learning_rate": 0.0001, + "loss": 79.5881, + "step": 1420 + }, + { + "epoch": 2.3031188592734417, + "grad_norm": 1.2285529375076294, + "learning_rate": 0.0001, + "loss": 79.5517, + "step": 1430 + }, + { + "epoch": 2.3192422028518163, + "grad_norm": 1.1916660070419312, + "learning_rate": 0.0001, + "loss": 80.5887, + "step": 1440 + }, + { + "epoch": 2.335365546430191, + "grad_norm": 1.224304437637329, + "learning_rate": 0.0001, + "loss": 79.8109, + "step": 1450 + }, + { + "epoch": 2.3514888900085653, + "grad_norm": 0.9228448867797852, + "learning_rate": 0.0001, + "loss": 79.2787, + "step": 1460 + }, + { + "epoch": 2.3676122335869403, + "grad_norm": 1.2085226774215698, + "learning_rate": 0.0001, + "loss": 78.4983, + "step": 1470 + }, + { + "epoch": 2.383735577165315, + "grad_norm": 2.424083948135376, + "learning_rate": 0.0001, + "loss": 79.8657, + "step": 1480 + }, + { + "epoch": 2.3998589207436893, + "grad_norm": 1.0277849435806274, + "learning_rate": 0.0001, + "loss": 80.6915, + "step": 1490 + }, + { + "epoch": 2.415982264322064, + "grad_norm": 1.0226075649261475, + "learning_rate": 0.0001, + "loss": 78.9751, + "step": 1500 + }, + { + "epoch": 2.4321056079004384, + "grad_norm": 0.8860388398170471, + "learning_rate": 0.0001, + "loss": 78.3915, + "step": 1510 + }, + { + "epoch": 2.448228951478813, + "grad_norm": 1.265637755393982, + "learning_rate": 0.0001, + "loss": 78.8737, + "step": 1520 + }, + { + "epoch": 2.4643522950571874, + "grad_norm": 1.2518186569213867, + "learning_rate": 0.0001, + "loss": 76.5169, + "step": 1530 + }, + { + "epoch": 2.480475638635562, + "grad_norm": 1.074277400970459, + "learning_rate": 0.0001, + "loss": 77.9156, + "step": 1540 + }, + { + "epoch": 2.4965989822139365, + "grad_norm": 1.0666711330413818, + "learning_rate": 0.0001, + "loss": 78.4103, + "step": 1550 + }, + { + "epoch": 2.512722325792311, + "grad_norm": 0.9492248892784119, + "learning_rate": 0.0001, + "loss": 76.7393, + "step": 1560 + }, + { + "epoch": 2.5288456693706856, + "grad_norm": 1.127238154411316, + "learning_rate": 0.0001, + "loss": 79.729, + "step": 1570 + }, + { + "epoch": 2.5449690129490605, + "grad_norm": 1.1520992517471313, + "learning_rate": 0.0001, + "loss": 78.5618, + "step": 1580 + }, + { + "epoch": 2.5610923565274346, + "grad_norm": 0.8632035255432129, + "learning_rate": 0.0001, + "loss": 79.032, + "step": 1590 + }, + { + "epoch": 2.5772157001058096, + "grad_norm": 1.0506329536437988, + "learning_rate": 0.0001, + "loss": 77.6204, + "step": 1600 + }, + { + "epoch": 2.593339043684184, + "grad_norm": 1.019895315170288, + "learning_rate": 0.0001, + "loss": 79.2634, + "step": 1610 + }, + { + "epoch": 2.6094623872625586, + "grad_norm": 1.1685571670532227, + "learning_rate": 0.0001, + "loss": 78.8371, + "step": 1620 + }, + { + "epoch": 2.625585730840933, + "grad_norm": 1.147546648979187, + "learning_rate": 0.0001, + "loss": 78.2113, + "step": 1630 + }, + { + "epoch": 2.6417090744193077, + "grad_norm": 1.7336841821670532, + "learning_rate": 0.0001, + "loss": 78.7866, + "step": 1640 + }, + { + "epoch": 2.657832417997682, + "grad_norm": 1.2150614261627197, + "learning_rate": 0.0001, + "loss": 78.5621, + "step": 1650 + }, + { + "epoch": 2.6739557615760567, + "grad_norm": 1.2257503271102905, + "learning_rate": 0.0001, + "loss": 77.5835, + "step": 1660 + }, + { + "epoch": 2.6900791051544313, + "grad_norm": 1.0375311374664307, + "learning_rate": 0.0001, + "loss": 77.7509, + "step": 1670 + }, + { + "epoch": 2.706202448732806, + "grad_norm": 1.2551004886627197, + "learning_rate": 0.0001, + "loss": 77.8615, + "step": 1680 + }, + { + "epoch": 2.7223257923111808, + "grad_norm": 1.0464223623275757, + "learning_rate": 0.0001, + "loss": 77.2837, + "step": 1690 + }, + { + "epoch": 2.738449135889555, + "grad_norm": 1.0254806280136108, + "learning_rate": 0.0001, + "loss": 76.9616, + "step": 1700 + }, + { + "epoch": 2.75457247946793, + "grad_norm": 1.1665682792663574, + "learning_rate": 0.0001, + "loss": 77.9108, + "step": 1710 + }, + { + "epoch": 2.7706958230463044, + "grad_norm": 1.1349453926086426, + "learning_rate": 0.0001, + "loss": 78.7636, + "step": 1720 + }, + { + "epoch": 2.786819166624679, + "grad_norm": 1.4961057901382446, + "learning_rate": 0.0001, + "loss": 76.3862, + "step": 1730 + }, + { + "epoch": 2.8029425102030534, + "grad_norm": 1.10721755027771, + "learning_rate": 0.0001, + "loss": 77.4105, + "step": 1740 + }, + { + "epoch": 2.819065853781428, + "grad_norm": 0.9878576397895813, + "learning_rate": 0.0001, + "loss": 78.4855, + "step": 1750 + }, + { + "epoch": 2.8351891973598025, + "grad_norm": 1.1722687482833862, + "learning_rate": 0.0001, + "loss": 75.1589, + "step": 1760 + }, + { + "epoch": 2.851312540938177, + "grad_norm": 1.6624362468719482, + "learning_rate": 0.0001, + "loss": 77.1647, + "step": 1770 + }, + { + "epoch": 2.8674358845165515, + "grad_norm": 0.9542295932769775, + "learning_rate": 0.0001, + "loss": 77.1656, + "step": 1780 + }, + { + "epoch": 2.883559228094926, + "grad_norm": 2.328542947769165, + "learning_rate": 0.0001, + "loss": 76.4686, + "step": 1790 + }, + { + "epoch": 2.899682571673301, + "grad_norm": 1.2532029151916504, + "learning_rate": 0.0001, + "loss": 76.1727, + "step": 1800 + }, + { + "epoch": 2.915805915251675, + "grad_norm": 0.9604716897010803, + "learning_rate": 0.0001, + "loss": 74.3132, + "step": 1810 + }, + { + "epoch": 2.93192925883005, + "grad_norm": 0.9182595610618591, + "learning_rate": 0.0001, + "loss": 77.8689, + "step": 1820 + }, + { + "epoch": 2.9480526024084246, + "grad_norm": 0.9623042941093445, + "learning_rate": 0.0001, + "loss": 77.7531, + "step": 1830 + }, + { + "epoch": 2.964175945986799, + "grad_norm": 1.0500119924545288, + "learning_rate": 0.0001, + "loss": 77.6007, + "step": 1840 + }, + { + "epoch": 2.9802992895651736, + "grad_norm": 1.0874103307724, + "learning_rate": 0.0001, + "loss": 77.042, + "step": 1850 + }, + { + "epoch": 2.996422633143548, + "grad_norm": 0.9955475330352783, + "learning_rate": 0.0001, + "loss": 75.468, + "step": 1860 + }, + { + "epoch": 3.011286340504862, + "grad_norm": 1.4281760454177856, + "learning_rate": 0.0001, + "loss": 69.5972, + "step": 1870 + }, + { + "epoch": 3.0274096840832367, + "grad_norm": 1.0292123556137085, + "learning_rate": 0.0001, + "loss": 76.0186, + "step": 1880 + }, + { + "epoch": 3.0435330276616113, + "grad_norm": 0.9535816311836243, + "learning_rate": 0.0001, + "loss": 76.0979, + "step": 1890 + }, + { + "epoch": 3.059656371239986, + "grad_norm": 0.9099674820899963, + "learning_rate": 0.0001, + "loss": 75.9641, + "step": 1900 + }, + { + "epoch": 3.0757797148183603, + "grad_norm": 0.9568380117416382, + "learning_rate": 0.0001, + "loss": 75.0656, + "step": 1910 + }, + { + "epoch": 3.091903058396735, + "grad_norm": 0.9701060652732849, + "learning_rate": 0.0001, + "loss": 75.1422, + "step": 1920 + }, + { + "epoch": 3.1080264019751094, + "grad_norm": 1.3879098892211914, + "learning_rate": 0.0001, + "loss": 76.0731, + "step": 1930 + }, + { + "epoch": 3.1241497455534843, + "grad_norm": 0.9782084226608276, + "learning_rate": 0.0001, + "loss": 75.9666, + "step": 1940 + }, + { + "epoch": 3.140273089131859, + "grad_norm": 1.0856964588165283, + "learning_rate": 0.0001, + "loss": 75.0217, + "step": 1950 + }, + { + "epoch": 3.1563964327102334, + "grad_norm": 0.8506444692611694, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 1960 + }, + { + "epoch": 3.172519776288608, + "grad_norm": 1.044980525970459, + "learning_rate": 0.0001, + "loss": 75.3471, + "step": 1970 + }, + { + "epoch": 3.1886431198669825, + "grad_norm": 0.8432291746139526, + "learning_rate": 0.0001, + "loss": 75.3563, + "step": 1980 + }, + { + "epoch": 3.204766463445357, + "grad_norm": 0.9651834964752197, + "learning_rate": 0.0001, + "loss": 76.3991, + "step": 1990 + }, + { + "epoch": 3.2208898070237315, + "grad_norm": 0.957943320274353, + "learning_rate": 0.0001, + "loss": 77.2613, + "step": 2000 + }, + { + "epoch": 3.237013150602106, + "grad_norm": 0.9495745301246643, + "learning_rate": 0.0001, + "loss": 73.3801, + "step": 2010 + }, + { + "epoch": 3.2531364941804806, + "grad_norm": 1.182983160018921, + "learning_rate": 0.0001, + "loss": 74.5695, + "step": 2020 + }, + { + "epoch": 3.269259837758855, + "grad_norm": 0.828452467918396, + "learning_rate": 0.0001, + "loss": 75.4313, + "step": 2030 + }, + { + "epoch": 3.2853831813372296, + "grad_norm": 0.8340104818344116, + "learning_rate": 0.0001, + "loss": 75.1086, + "step": 2040 + }, + { + "epoch": 3.301506524915604, + "grad_norm": 1.1700762510299683, + "learning_rate": 0.0001, + "loss": 75.7496, + "step": 2050 + }, + { + "epoch": 3.317629868493979, + "grad_norm": 0.8709110617637634, + "learning_rate": 0.0001, + "loss": 73.8403, + "step": 2060 + }, + { + "epoch": 3.3337532120723536, + "grad_norm": 0.8928954601287842, + "learning_rate": 0.0001, + "loss": 75.6134, + "step": 2070 + }, + { + "epoch": 3.349876555650728, + "grad_norm": 0.9787353277206421, + "learning_rate": 0.0001, + "loss": 75.8116, + "step": 2080 + }, + { + "epoch": 3.3659998992291027, + "grad_norm": 0.9515123963356018, + "learning_rate": 0.0001, + "loss": 75.5024, + "step": 2090 + }, + { + "epoch": 3.3821232428074772, + "grad_norm": 0.9529262185096741, + "learning_rate": 0.0001, + "loss": 74.1553, + "step": 2100 + }, + { + "epoch": 3.3982465863858518, + "grad_norm": 0.8934710621833801, + "learning_rate": 0.0001, + "loss": 74.1501, + "step": 2110 + }, + { + "epoch": 3.4143699299642263, + "grad_norm": 1.089447021484375, + "learning_rate": 0.0001, + "loss": 74.9123, + "step": 2120 + }, + { + "epoch": 3.430493273542601, + "grad_norm": 1.2569622993469238, + "learning_rate": 0.0001, + "loss": 74.3962, + "step": 2130 + }, + { + "epoch": 3.4466166171209753, + "grad_norm": 1.020094394683838, + "learning_rate": 0.0001, + "loss": 74.6617, + "step": 2140 + }, + { + "epoch": 3.46273996069935, + "grad_norm": 1.1579667329788208, + "learning_rate": 0.0001, + "loss": 74.2745, + "step": 2150 + }, + { + "epoch": 3.4788633042777244, + "grad_norm": 1.448935627937317, + "learning_rate": 0.0001, + "loss": 71.5285, + "step": 2160 + }, + { + "epoch": 3.4949866478560994, + "grad_norm": 1.0030494928359985, + "learning_rate": 0.0001, + "loss": 74.6666, + "step": 2170 + }, + { + "epoch": 3.511109991434474, + "grad_norm": 0.9010888934135437, + "learning_rate": 0.0001, + "loss": 74.9355, + "step": 2180 + }, + { + "epoch": 3.5272333350128484, + "grad_norm": 1.1595124006271362, + "learning_rate": 0.0001, + "loss": 74.5049, + "step": 2190 + }, + { + "epoch": 3.543356678591223, + "grad_norm": 1.066405177116394, + "learning_rate": 0.0001, + "loss": 74.6336, + "step": 2200 + }, + { + "epoch": 3.5594800221695975, + "grad_norm": 1.2301596403121948, + "learning_rate": 0.0001, + "loss": 74.6664, + "step": 2210 + }, + { + "epoch": 3.575603365747972, + "grad_norm": 1.0073511600494385, + "learning_rate": 0.0001, + "loss": 73.1405, + "step": 2220 + }, + { + "epoch": 3.5917267093263465, + "grad_norm": 0.9913995862007141, + "learning_rate": 0.0001, + "loss": 75.7443, + "step": 2230 + }, + { + "epoch": 3.607850052904721, + "grad_norm": 0.9555648565292358, + "learning_rate": 0.0001, + "loss": 74.8036, + "step": 2240 + }, + { + "epoch": 3.6239733964830956, + "grad_norm": 0.9350649118423462, + "learning_rate": 0.0001, + "loss": 73.5783, + "step": 2250 + }, + { + "epoch": 3.64009674006147, + "grad_norm": 0.935840368270874, + "learning_rate": 0.0001, + "loss": 75.1694, + "step": 2260 + }, + { + "epoch": 3.6562200836398446, + "grad_norm": 0.9303803443908691, + "learning_rate": 0.0001, + "loss": 72.8282, + "step": 2270 + }, + { + "epoch": 3.6723434272182196, + "grad_norm": 0.937164843082428, + "learning_rate": 0.0001, + "loss": 70.3305, + "step": 2280 + }, + { + "epoch": 3.6884667707965937, + "grad_norm": 1.027329444885254, + "learning_rate": 0.0001, + "loss": 73.748, + "step": 2290 + }, + { + "epoch": 3.7045901143749687, + "grad_norm": 1.0915215015411377, + "learning_rate": 0.0001, + "loss": 72.2265, + "step": 2300 + }, + { + "epoch": 3.720713457953343, + "grad_norm": 1.1031261682510376, + "learning_rate": 0.0001, + "loss": 71.2795, + "step": 2310 + }, + { + "epoch": 3.7368368015317177, + "grad_norm": 1.1466140747070312, + "learning_rate": 0.0001, + "loss": 72.0116, + "step": 2320 + }, + { + "epoch": 3.7529601451100922, + "grad_norm": 1.0441662073135376, + "learning_rate": 0.0001, + "loss": 73.3735, + "step": 2330 + }, + { + "epoch": 3.7690834886884668, + "grad_norm": 0.8610432744026184, + "learning_rate": 0.0001, + "loss": 73.3433, + "step": 2340 + }, + { + "epoch": 3.7852068322668413, + "grad_norm": 0.885720431804657, + "learning_rate": 0.0001, + "loss": 73.575, + "step": 2350 + }, + { + "epoch": 3.801330175845216, + "grad_norm": 1.0023950338363647, + "learning_rate": 0.0001, + "loss": 73.4166, + "step": 2360 + }, + { + "epoch": 3.8174535194235903, + "grad_norm": 1.3820304870605469, + "learning_rate": 0.0001, + "loss": 72.9599, + "step": 2370 + }, + { + "epoch": 3.833576863001965, + "grad_norm": 0.9645710587501526, + "learning_rate": 0.0001, + "loss": 72.3922, + "step": 2380 + }, + { + "epoch": 3.84970020658034, + "grad_norm": 1.273369312286377, + "learning_rate": 0.0001, + "loss": 72.0158, + "step": 2390 + }, + { + "epoch": 3.865823550158714, + "grad_norm": 0.859863817691803, + "learning_rate": 0.0001, + "loss": 74.6642, + "step": 2400 + }, + { + "epoch": 3.881946893737089, + "grad_norm": 0.9396682381629944, + "learning_rate": 0.0001, + "loss": 71.9003, + "step": 2410 + }, + { + "epoch": 3.8980702373154634, + "grad_norm": 0.9976151585578918, + "learning_rate": 0.0001, + "loss": 73.0716, + "step": 2420 + }, + { + "epoch": 3.914193580893838, + "grad_norm": 1.0730767250061035, + "learning_rate": 0.0001, + "loss": 72.7847, + "step": 2430 + }, + { + "epoch": 3.9303169244722125, + "grad_norm": 1.002018690109253, + "learning_rate": 0.0001, + "loss": 71.7309, + "step": 2440 + }, + { + "epoch": 3.946440268050587, + "grad_norm": 0.9102753400802612, + "learning_rate": 0.0001, + "loss": 71.616, + "step": 2450 + }, + { + "epoch": 3.9625636116289615, + "grad_norm": 0.9827467799186707, + "learning_rate": 0.0001, + "loss": 72.22, + "step": 2460 + }, + { + "epoch": 3.978686955207336, + "grad_norm": 1.2781893014907837, + "learning_rate": 0.0001, + "loss": 73.2048, + "step": 2470 + }, + { + "epoch": 3.9948102987857106, + "grad_norm": 0.9042410850524902, + "learning_rate": 0.0001, + "loss": 72.9843, + "step": 2480 + }, + { + "epoch": 4.009674006147025, + "grad_norm": 0.8662117123603821, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 2490 + }, + { + "epoch": 4.0257973497254, + "grad_norm": 0.9448756575584412, + "learning_rate": 0.0001, + "loss": 73.36, + "step": 2500 + }, + { + "epoch": 4.041920693303774, + "grad_norm": 0.9054566621780396, + "learning_rate": 0.0001, + "loss": 73.94, + "step": 2510 + }, + { + "epoch": 4.058044036882149, + "grad_norm": 0.957398533821106, + "learning_rate": 0.0001, + "loss": 73.2019, + "step": 2520 + }, + { + "epoch": 4.074167380460523, + "grad_norm": 0.9444408416748047, + "learning_rate": 0.0001, + "loss": 70.6553, + "step": 2530 + }, + { + "epoch": 4.090290724038898, + "grad_norm": 0.8716063499450684, + "learning_rate": 0.0001, + "loss": 70.1048, + "step": 2540 + }, + { + "epoch": 4.106414067617272, + "grad_norm": 0.949249267578125, + "learning_rate": 0.0001, + "loss": 72.4453, + "step": 2550 + }, + { + "epoch": 4.122537411195647, + "grad_norm": 0.9843282699584961, + "learning_rate": 0.0001, + "loss": 72.1725, + "step": 2560 + }, + { + "epoch": 4.138660754774021, + "grad_norm": 0.8977480530738831, + "learning_rate": 0.0001, + "loss": 72.5272, + "step": 2570 + }, + { + "epoch": 4.154784098352396, + "grad_norm": 0.9957827925682068, + "learning_rate": 0.0001, + "loss": 72.289, + "step": 2580 + }, + { + "epoch": 4.170907441930771, + "grad_norm": 1.1358791589736938, + "learning_rate": 0.0001, + "loss": 72.1949, + "step": 2590 + }, + { + "epoch": 4.187030785509145, + "grad_norm": 0.9953081607818604, + "learning_rate": 0.0001, + "loss": 72.6004, + "step": 2600 + }, + { + "epoch": 4.20315412908752, + "grad_norm": 1.019061803817749, + "learning_rate": 0.0001, + "loss": 72.3772, + "step": 2610 + }, + { + "epoch": 4.219277472665894, + "grad_norm": 0.9062562584877014, + "learning_rate": 0.0001, + "loss": 72.0566, + "step": 2620 + }, + { + "epoch": 4.235400816244269, + "grad_norm": 1.0877611637115479, + "learning_rate": 0.0001, + "loss": 72.854, + "step": 2630 + }, + { + "epoch": 4.251524159822643, + "grad_norm": 1.2419648170471191, + "learning_rate": 0.0001, + "loss": 72.6498, + "step": 2640 + }, + { + "epoch": 4.267647503401018, + "grad_norm": 0.9507225751876831, + "learning_rate": 0.0001, + "loss": 68.9583, + "step": 2650 + }, + { + "epoch": 4.283770846979392, + "grad_norm": 1.121191382408142, + "learning_rate": 0.0001, + "loss": 72.8464, + "step": 2660 + }, + { + "epoch": 4.299894190557767, + "grad_norm": 1.0543361902236938, + "learning_rate": 0.0001, + "loss": 71.9882, + "step": 2670 + }, + { + "epoch": 4.316017534136141, + "grad_norm": 0.8622955083847046, + "learning_rate": 0.0001, + "loss": 71.7373, + "step": 2680 + }, + { + "epoch": 4.332140877714516, + "grad_norm": 0.911253809928894, + "learning_rate": 0.0001, + "loss": 71.1968, + "step": 2690 + }, + { + "epoch": 4.34826422129289, + "grad_norm": 4.405472755432129, + "learning_rate": 0.0001, + "loss": 70.4128, + "step": 2700 + }, + { + "epoch": 4.364387564871265, + "grad_norm": 0.9142985939979553, + "learning_rate": 0.0001, + "loss": 69.7548, + "step": 2710 + }, + { + "epoch": 4.38051090844964, + "grad_norm": 1.1022522449493408, + "learning_rate": 0.0001, + "loss": 71.8282, + "step": 2720 + }, + { + "epoch": 4.396634252028014, + "grad_norm": 1.018386721611023, + "learning_rate": 0.0001, + "loss": 71.0853, + "step": 2730 + }, + { + "epoch": 4.412757595606389, + "grad_norm": 1.0483211278915405, + "learning_rate": 0.0001, + "loss": 72.1478, + "step": 2740 + }, + { + "epoch": 4.428880939184763, + "grad_norm": 1.026861548423767, + "learning_rate": 0.0001, + "loss": 71.8947, + "step": 2750 + }, + { + "epoch": 4.445004282763138, + "grad_norm": 1.1858552694320679, + "learning_rate": 0.0001, + "loss": 71.7016, + "step": 2760 + }, + { + "epoch": 4.461127626341512, + "grad_norm": 1.1212221384048462, + "learning_rate": 0.0001, + "loss": 71.4454, + "step": 2770 + }, + { + "epoch": 4.477250969919887, + "grad_norm": 0.8062102794647217, + "learning_rate": 0.0001, + "loss": 73.2672, + "step": 2780 + }, + { + "epoch": 4.493374313498261, + "grad_norm": 1.104483962059021, + "learning_rate": 0.0001, + "loss": 70.548, + "step": 2790 + }, + { + "epoch": 4.509497657076636, + "grad_norm": 0.942057728767395, + "learning_rate": 0.0001, + "loss": 72.3584, + "step": 2800 + }, + { + "epoch": 4.525621000655011, + "grad_norm": 0.9028333425521851, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 2810 + }, + { + "epoch": 4.541744344233385, + "grad_norm": 0.9096795916557312, + "learning_rate": 0.0001, + "loss": 70.6596, + "step": 2820 + }, + { + "epoch": 4.55786768781176, + "grad_norm": 0.9038829207420349, + "learning_rate": 0.0001, + "loss": 72.6148, + "step": 2830 + }, + { + "epoch": 4.573991031390134, + "grad_norm": 1.8511613607406616, + "learning_rate": 0.0001, + "loss": 71.7205, + "step": 2840 + }, + { + "epoch": 4.590114374968509, + "grad_norm": 1.3492343425750732, + "learning_rate": 0.0001, + "loss": 72.1492, + "step": 2850 + }, + { + "epoch": 4.6062377185468835, + "grad_norm": 1.1469528675079346, + "learning_rate": 0.0001, + "loss": 70.5892, + "step": 2860 + }, + { + "epoch": 4.622361062125258, + "grad_norm": 1.1374379396438599, + "learning_rate": 0.0001, + "loss": 70.061, + "step": 2870 + }, + { + "epoch": 4.6384844057036325, + "grad_norm": 0.9597565531730652, + "learning_rate": 0.0001, + "loss": 71.0869, + "step": 2880 + }, + { + "epoch": 4.6546077492820075, + "grad_norm": 0.8356521129608154, + "learning_rate": 0.0001, + "loss": 72.1891, + "step": 2890 + }, + { + "epoch": 4.670731092860382, + "grad_norm": 0.8882647752761841, + "learning_rate": 0.0001, + "loss": 71.3845, + "step": 2900 + }, + { + "epoch": 4.6868544364387565, + "grad_norm": 1.1904003620147705, + "learning_rate": 0.0001, + "loss": 72.3086, + "step": 2910 + }, + { + "epoch": 4.702977780017131, + "grad_norm": 0.8010085821151733, + "learning_rate": 0.0001, + "loss": 71.3051, + "step": 2920 + }, + { + "epoch": 4.719101123595506, + "grad_norm": 0.7572135329246521, + "learning_rate": 0.0001, + "loss": 72.5605, + "step": 2930 + }, + { + "epoch": 4.735224467173881, + "grad_norm": 1.0866942405700684, + "learning_rate": 0.0001, + "loss": 70.0935, + "step": 2940 + }, + { + "epoch": 4.751347810752255, + "grad_norm": 0.9358721971511841, + "learning_rate": 0.0001, + "loss": 71.0391, + "step": 2950 + }, + { + "epoch": 4.76747115433063, + "grad_norm": 1.0276161432266235, + "learning_rate": 0.0001, + "loss": 69.8462, + "step": 2960 + }, + { + "epoch": 4.783594497909004, + "grad_norm": 0.9573063850402832, + "learning_rate": 0.0001, + "loss": 69.7089, + "step": 2970 + }, + { + "epoch": 4.799717841487379, + "grad_norm": 1.0638892650604248, + "learning_rate": 0.0001, + "loss": 71.8376, + "step": 2980 + }, + { + "epoch": 4.815841185065753, + "grad_norm": 0.9894681572914124, + "learning_rate": 0.0001, + "loss": 70.1764, + "step": 2990 + }, + { + "epoch": 4.831964528644128, + "grad_norm": 1.1265769004821777, + "learning_rate": 0.0001, + "loss": 71.7918, + "step": 3000 + }, + { + "epoch": 4.848087872222502, + "grad_norm": 2.2792513370513916, + "learning_rate": 0.0001, + "loss": 70.4341, + "step": 3010 + }, + { + "epoch": 4.864211215800877, + "grad_norm": 0.8469329476356506, + "learning_rate": 0.0001, + "loss": 72.1137, + "step": 3020 + }, + { + "epoch": 4.880334559379252, + "grad_norm": 0.9333013296127319, + "learning_rate": 0.0001, + "loss": 71.0789, + "step": 3030 + }, + { + "epoch": 4.896457902957626, + "grad_norm": 0.7888994216918945, + "learning_rate": 0.0001, + "loss": 70.8197, + "step": 3040 + }, + { + "epoch": 4.912581246536, + "grad_norm": 0.9194874167442322, + "learning_rate": 0.0001, + "loss": 72.071, + "step": 3050 + }, + { + "epoch": 4.928704590114375, + "grad_norm": 1.1252421140670776, + "learning_rate": 0.0001, + "loss": 71.6943, + "step": 3060 + }, + { + "epoch": 4.94482793369275, + "grad_norm": 1.1801071166992188, + "learning_rate": 0.0001, + "loss": 70.8937, + "step": 3070 + }, + { + "epoch": 4.960951277271124, + "grad_norm": 0.9162415266036987, + "learning_rate": 0.0001, + "loss": 70.024, + "step": 3080 + }, + { + "epoch": 4.977074620849499, + "grad_norm": 0.8833363652229309, + "learning_rate": 0.0001, + "loss": 71.3835, + "step": 3090 + }, + { + "epoch": 4.993197964427873, + "grad_norm": 0.9985883831977844, + "learning_rate": 0.0001, + "loss": 71.635, + "step": 3100 + }, + { + "epoch": 5.0080616717891875, + "grad_norm": 1.097298264503479, + "learning_rate": 0.0001, + "loss": 65.3542, + "step": 3110 + }, + { + "epoch": 5.024185015367562, + "grad_norm": 0.9703903198242188, + "learning_rate": 0.0001, + "loss": 69.8587, + "step": 3120 + }, + { + "epoch": 5.0403083589459365, + "grad_norm": 0.8894070386886597, + "learning_rate": 0.0001, + "loss": 71.2868, + "step": 3130 + }, + { + "epoch": 5.056431702524311, + "grad_norm": 0.6984127163887024, + "learning_rate": 0.0001, + "loss": 69.0425, + "step": 3140 + }, + { + "epoch": 5.072555046102686, + "grad_norm": 0.8607762455940247, + "learning_rate": 0.0001, + "loss": 69.5354, + "step": 3150 + }, + { + "epoch": 5.08867838968106, + "grad_norm": 1.1215914487838745, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 3160 + }, + { + "epoch": 5.104801733259435, + "grad_norm": 0.9114409685134888, + "learning_rate": 0.0001, + "loss": 68.4186, + "step": 3170 + }, + { + "epoch": 5.12092507683781, + "grad_norm": 1.010400652885437, + "learning_rate": 0.0001, + "loss": 68.3965, + "step": 3180 + }, + { + "epoch": 5.137048420416184, + "grad_norm": 0.8741958141326904, + "learning_rate": 0.0001, + "loss": 70.6728, + "step": 3190 + }, + { + "epoch": 5.153171763994559, + "grad_norm": 0.8760207891464233, + "learning_rate": 0.0001, + "loss": 72.0704, + "step": 3200 + }, + { + "epoch": 5.169295107572933, + "grad_norm": 1.328247308731079, + "learning_rate": 0.0001, + "loss": 69.9082, + "step": 3210 + }, + { + "epoch": 5.185418451151308, + "grad_norm": 0.8263765573501587, + "learning_rate": 0.0001, + "loss": 70.3539, + "step": 3220 + }, + { + "epoch": 5.201541794729682, + "grad_norm": 0.9358834028244019, + "learning_rate": 0.0001, + "loss": 70.4221, + "step": 3230 + }, + { + "epoch": 5.217665138308057, + "grad_norm": 1.0769531726837158, + "learning_rate": 0.0001, + "loss": 68.8167, + "step": 3240 + }, + { + "epoch": 5.233788481886431, + "grad_norm": 1.1438790559768677, + "learning_rate": 0.0001, + "loss": 68.9288, + "step": 3250 + }, + { + "epoch": 5.249911825464806, + "grad_norm": 0.9534218311309814, + "learning_rate": 0.0001, + "loss": 70.3277, + "step": 3260 + }, + { + "epoch": 5.26603516904318, + "grad_norm": 0.88532954454422, + "learning_rate": 0.0001, + "loss": 67.5781, + "step": 3270 + }, + { + "epoch": 5.282158512621555, + "grad_norm": 0.7986133694648743, + "learning_rate": 0.0001, + "loss": 69.759, + "step": 3280 + }, + { + "epoch": 5.29828185619993, + "grad_norm": 0.9100566506385803, + "learning_rate": 0.0001, + "loss": 68.6368, + "step": 3290 + }, + { + "epoch": 5.314405199778304, + "grad_norm": 0.8861745595932007, + "learning_rate": 0.0001, + "loss": 70.5756, + "step": 3300 + }, + { + "epoch": 5.330528543356679, + "grad_norm": 1.0729680061340332, + "learning_rate": 0.0001, + "loss": 70.3178, + "step": 3310 + }, + { + "epoch": 5.346651886935053, + "grad_norm": 1.0288280248641968, + "learning_rate": 0.0001, + "loss": 69.422, + "step": 3320 + }, + { + "epoch": 5.362775230513428, + "grad_norm": 0.8335140347480774, + "learning_rate": 0.0001, + "loss": 69.8281, + "step": 3330 + }, + { + "epoch": 5.378898574091802, + "grad_norm": 0.9789717197418213, + "learning_rate": 0.0001, + "loss": 71.1532, + "step": 3340 + }, + { + "epoch": 5.395021917670177, + "grad_norm": 0.9884677529335022, + "learning_rate": 0.0001, + "loss": 69.5673, + "step": 3350 + }, + { + "epoch": 5.411145261248551, + "grad_norm": 1.050154447555542, + "learning_rate": 0.0001, + "loss": 70.8015, + "step": 3360 + }, + { + "epoch": 5.427268604826926, + "grad_norm": 0.9645342230796814, + "learning_rate": 0.0001, + "loss": 69.1615, + "step": 3370 + }, + { + "epoch": 5.4433919484053, + "grad_norm": 0.8383159041404724, + "learning_rate": 0.0001, + "loss": 70.6299, + "step": 3380 + }, + { + "epoch": 5.459515291983675, + "grad_norm": 0.9250556826591492, + "learning_rate": 0.0001, + "loss": 69.231, + "step": 3390 + }, + { + "epoch": 5.475638635562049, + "grad_norm": 0.8108687400817871, + "learning_rate": 0.0001, + "loss": 70.5858, + "step": 3400 + }, + { + "epoch": 5.491761979140424, + "grad_norm": 0.8748818039894104, + "learning_rate": 0.0001, + "loss": 69.9001, + "step": 3410 + }, + { + "epoch": 5.507885322718799, + "grad_norm": 0.7411381006240845, + "learning_rate": 0.0001, + "loss": 68.3248, + "step": 3420 + }, + { + "epoch": 5.524008666297173, + "grad_norm": 1.0753326416015625, + "learning_rate": 0.0001, + "loss": 69.5831, + "step": 3430 + }, + { + "epoch": 5.540132009875548, + "grad_norm": 0.875254213809967, + "learning_rate": 0.0001, + "loss": 69.7516, + "step": 3440 + }, + { + "epoch": 5.556255353453922, + "grad_norm": 0.7637733817100525, + "learning_rate": 0.0001, + "loss": 68.0078, + "step": 3450 + }, + { + "epoch": 5.572378697032297, + "grad_norm": 0.968203604221344, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 3460 + }, + { + "epoch": 5.588502040610671, + "grad_norm": 0.8903141617774963, + "learning_rate": 0.0001, + "loss": 68.5835, + "step": 3470 + }, + { + "epoch": 5.604625384189046, + "grad_norm": 0.9536195397377014, + "learning_rate": 0.0001, + "loss": 71.0259, + "step": 3480 + }, + { + "epoch": 5.62074872776742, + "grad_norm": 0.8951714038848877, + "learning_rate": 0.0001, + "loss": 69.4422, + "step": 3490 + }, + { + "epoch": 5.636872071345795, + "grad_norm": 0.9639522433280945, + "learning_rate": 0.0001, + "loss": 66.5173, + "step": 3500 + }, + { + "epoch": 5.65299541492417, + "grad_norm": 1.0022730827331543, + "learning_rate": 0.0001, + "loss": 68.3443, + "step": 3510 + }, + { + "epoch": 5.669118758502544, + "grad_norm": 1.0654487609863281, + "learning_rate": 0.0001, + "loss": 69.0835, + "step": 3520 + }, + { + "epoch": 5.685242102080919, + "grad_norm": 0.9934629797935486, + "learning_rate": 0.0001, + "loss": 69.2278, + "step": 3530 + }, + { + "epoch": 5.7013654456592935, + "grad_norm": 0.8773566484451294, + "learning_rate": 0.0001, + "loss": 68.9339, + "step": 3540 + }, + { + "epoch": 5.7174887892376685, + "grad_norm": 0.7756998538970947, + "learning_rate": 0.0001, + "loss": 69.2683, + "step": 3550 + }, + { + "epoch": 5.7336121328160425, + "grad_norm": 0.8819359540939331, + "learning_rate": 0.0001, + "loss": 68.5071, + "step": 3560 + }, + { + "epoch": 5.7497354763944175, + "grad_norm": 1.1069825887680054, + "learning_rate": 0.0001, + "loss": 68.5236, + "step": 3570 + }, + { + "epoch": 5.765858819972792, + "grad_norm": 0.8006312847137451, + "learning_rate": 0.0001, + "loss": 68.7192, + "step": 3580 + }, + { + "epoch": 5.781982163551167, + "grad_norm": 1.0278719663619995, + "learning_rate": 0.0001, + "loss": 69.2545, + "step": 3590 + }, + { + "epoch": 5.798105507129541, + "grad_norm": 0.9205041527748108, + "learning_rate": 0.0001, + "loss": 69.7698, + "step": 3600 + }, + { + "epoch": 5.814228850707916, + "grad_norm": 1.0769038200378418, + "learning_rate": 0.0001, + "loss": 67.6568, + "step": 3610 + }, + { + "epoch": 5.83035219428629, + "grad_norm": 1.0565663576126099, + "learning_rate": 0.0001, + "loss": 68.9166, + "step": 3620 + }, + { + "epoch": 5.846475537864665, + "grad_norm": 0.9008663296699524, + "learning_rate": 0.0001, + "loss": 69.5837, + "step": 3630 + }, + { + "epoch": 5.86259888144304, + "grad_norm": 0.8369411826133728, + "learning_rate": 0.0001, + "loss": 68.5661, + "step": 3640 + }, + { + "epoch": 5.878722225021414, + "grad_norm": 1.0472379922866821, + "learning_rate": 0.0001, + "loss": 68.3122, + "step": 3650 + }, + { + "epoch": 5.894845568599789, + "grad_norm": 0.965277373790741, + "learning_rate": 0.0001, + "loss": 69.8062, + "step": 3660 + }, + { + "epoch": 5.910968912178163, + "grad_norm": 0.8936495780944824, + "learning_rate": 0.0001, + "loss": 67.1016, + "step": 3670 + }, + { + "epoch": 5.927092255756538, + "grad_norm": 0.8559162020683289, + "learning_rate": 0.0001, + "loss": 69.9986, + "step": 3680 + }, + { + "epoch": 5.943215599334912, + "grad_norm": 1.4815773963928223, + "learning_rate": 0.0001, + "loss": 66.7648, + "step": 3690 + }, + { + "epoch": 5.959338942913287, + "grad_norm": 0.9034469723701477, + "learning_rate": 0.0001, + "loss": 68.0767, + "step": 3700 + }, + { + "epoch": 5.975462286491661, + "grad_norm": 1.838799238204956, + "learning_rate": 0.0001, + "loss": 66.5848, + "step": 3710 + }, + { + "epoch": 5.991585630070036, + "grad_norm": 0.9274787902832031, + "learning_rate": 0.0001, + "loss": 70.0396, + "step": 3720 + }, + { + "epoch": 6.0064493374313495, + "grad_norm": 1.016974687576294, + "learning_rate": 0.0001, + "loss": 62.2268, + "step": 3730 + }, + { + "epoch": 6.022572681009724, + "grad_norm": 1.0522642135620117, + "learning_rate": 0.0001, + "loss": 69.3381, + "step": 3740 + }, + { + "epoch": 6.0386960245880985, + "grad_norm": 1.1157186031341553, + "learning_rate": 0.0001, + "loss": 68.8969, + "step": 3750 + }, + { + "epoch": 6.0548193681664735, + "grad_norm": 0.9913288950920105, + "learning_rate": 0.0001, + "loss": 67.3634, + "step": 3760 + }, + { + "epoch": 6.0709427117448485, + "grad_norm": 0.8823490142822266, + "learning_rate": 0.0001, + "loss": 70.978, + "step": 3770 + }, + { + "epoch": 6.0870660553232225, + "grad_norm": 0.853837788105011, + "learning_rate": 0.0001, + "loss": 68.9924, + "step": 3780 + }, + { + "epoch": 6.1031893989015975, + "grad_norm": 0.7937766909599304, + "learning_rate": 0.0001, + "loss": 68.1994, + "step": 3790 + }, + { + "epoch": 6.119312742479972, + "grad_norm": 1.7638057470321655, + "learning_rate": 0.0001, + "loss": 68.5078, + "step": 3800 + }, + { + "epoch": 6.135436086058347, + "grad_norm": 1.0542887449264526, + "learning_rate": 0.0001, + "loss": 68.9362, + "step": 3810 + }, + { + "epoch": 6.151559429636721, + "grad_norm": 1.06019926071167, + "learning_rate": 0.0001, + "loss": 67.2493, + "step": 3820 + }, + { + "epoch": 6.167682773215096, + "grad_norm": 0.9765111804008484, + "learning_rate": 0.0001, + "loss": 70.4797, + "step": 3830 + }, + { + "epoch": 6.18380611679347, + "grad_norm": 1.006372332572937, + "learning_rate": 0.0001, + "loss": 69.4233, + "step": 3840 + }, + { + "epoch": 6.199929460371845, + "grad_norm": 0.8274241089820862, + "learning_rate": 0.0001, + "loss": 67.9907, + "step": 3850 + }, + { + "epoch": 6.216052803950219, + "grad_norm": 0.9905765056610107, + "learning_rate": 0.0001, + "loss": 69.6988, + "step": 3860 + }, + { + "epoch": 6.232176147528594, + "grad_norm": 1.0065921545028687, + "learning_rate": 0.0001, + "loss": 68.7678, + "step": 3870 + }, + { + "epoch": 6.248299491106969, + "grad_norm": 0.8845255970954895, + "learning_rate": 0.0001, + "loss": 68.3039, + "step": 3880 + }, + { + "epoch": 6.264422834685343, + "grad_norm": 0.9030968546867371, + "learning_rate": 0.0001, + "loss": 68.671, + "step": 3890 + }, + { + "epoch": 6.280546178263718, + "grad_norm": 0.8650258183479309, + "learning_rate": 0.0001, + "loss": 68.5309, + "step": 3900 + }, + { + "epoch": 6.296669521842092, + "grad_norm": 0.8961272835731506, + "learning_rate": 0.0001, + "loss": 67.5611, + "step": 3910 + }, + { + "epoch": 6.312792865420467, + "grad_norm": 1.1702407598495483, + "learning_rate": 0.0001, + "loss": 68.2768, + "step": 3920 + }, + { + "epoch": 6.328916208998841, + "grad_norm": 0.8598962426185608, + "learning_rate": 0.0001, + "loss": 69.1961, + "step": 3930 + }, + { + "epoch": 6.345039552577216, + "grad_norm": 0.8714122176170349, + "learning_rate": 0.0001, + "loss": 66.8628, + "step": 3940 + }, + { + "epoch": 6.36116289615559, + "grad_norm": 0.7757895588874817, + "learning_rate": 0.0001, + "loss": 68.6908, + "step": 3950 + }, + { + "epoch": 6.377286239733965, + "grad_norm": 0.9189490675926208, + "learning_rate": 0.0001, + "loss": 67.8685, + "step": 3960 + }, + { + "epoch": 6.393409583312339, + "grad_norm": 0.8161299228668213, + "learning_rate": 0.0001, + "loss": 66.816, + "step": 3970 + }, + { + "epoch": 6.409532926890714, + "grad_norm": 0.8559791445732117, + "learning_rate": 0.0001, + "loss": 68.4302, + "step": 3980 + }, + { + "epoch": 6.425656270469089, + "grad_norm": 0.8087984919548035, + "learning_rate": 0.0001, + "loss": 66.2646, + "step": 3990 + }, + { + "epoch": 6.441779614047463, + "grad_norm": 0.7733110785484314, + "learning_rate": 0.0001, + "loss": 69.2769, + "step": 4000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 33, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.892264908623289e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ade09d5233289ba4bc5076af39b8652078582dee --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016a4e9b5d71a1947317d3801b00fe5d0604edbef1f981d7b6c92f219131717c +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-4000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/latest new file mode 100644 index 0000000000000000000000000000000000000000..3ccb5110452b4a1eef79a1c432a18944dc25a985 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/latest @@ -0,0 +1 @@ +global_step6000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5ecdd767ad4fad94feadc67c10c30e5065c63053 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83bbb852c00e4b8216c556b302f3aaeb8948ffb55fc774a74ceec6c9c2b969a7 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..50f3667378dacbbbe8da61e9cd1b88694bd0d668 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce375c96d87719ecfbc30290c4aaab48dca65cba8121e39affbb72e52d149097 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..8583c5250628479b2624bcc1ac4be5aa861ecdbe --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6535208494203cfbacad1c7721178ba3fe243241d04c815d7db73784566896bd +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..1c34e01538fe7e81e63a3283d9b782106b42cde7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d28cc24f5ab251b297543ea43eb2c1bf649555dcfb399c2c21b604ed7892a7c1 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..7c1402bb348ec064f555f7adf8d26877f59fe62c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41f1fbb90f6b40405e20e00871d1906d637e4e4a8eff915452f050b87fc607e8 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..048291e0b50a814a7acc97d58b3795c5b13a2d01 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e91cac3af112b43d0aa156193f3413eb43e430117b57563c89023667e7604b2 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..090e40d08efa7f44f86b9a0fd8eab1e74dc29fb7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23d619ac58b7dadcedc5b2ee0af155728c1f9c9e30afbd21da6a46115ccb6e98 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..de678ce80f5d9c61f4d7791bf6537d825cc9da39 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:249cc7530bd62a86e5cd5db61096b7e7059f0e9b2d615015d1e632106f6794ea +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..7d1b1ca17a292b7057b11989d727faa7388fbc2d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d76e48838b356ac207e25c1202f87771929ff5e7244d70df2f915775002827aa +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..a76cd8e8b9ba139f39ea2186e82134fa532ba2e8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e7682127f80cd25a98dfdc8b12ba353464983fd8dca3440f4fbf60a3e51c0a5 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0ba2bcc0cc8c9812244cb2954e88f7e59446e73b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/trainer_state.json @@ -0,0 +1,4234 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.662669421071195, + "eval_steps": 2000, + "global_step": 6000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.016123343578374565, + "grad_norm": 48.11856460571289, + "learning_rate": 0.0001, + "loss": 299.1088, + "step": 10 + }, + { + "epoch": 0.03224668715674913, + "grad_norm": 9.07711410522461, + "learning_rate": 0.0001, + "loss": 234.0888, + "step": 20 + }, + { + "epoch": 0.0483700307351237, + "grad_norm": 3.5935330390930176, + "learning_rate": 0.0001, + "loss": 231.4619, + "step": 30 + }, + { + "epoch": 0.06449337431349826, + "grad_norm": 3.5784752368927, + "learning_rate": 0.0001, + "loss": 229.7082, + "step": 40 + }, + { + "epoch": 0.08061671789187283, + "grad_norm": 3.5757956504821777, + "learning_rate": 0.0001, + "loss": 230.1062, + "step": 50 + }, + { + "epoch": 0.0967400614702474, + "grad_norm": 3.8763978481292725, + "learning_rate": 0.0001, + "loss": 229.1914, + "step": 60 + }, + { + "epoch": 0.11286340504862195, + "grad_norm": 3.006988525390625, + "learning_rate": 0.0001, + "loss": 229.6482, + "step": 70 + }, + { + "epoch": 0.12898674862699652, + "grad_norm": 2.3527095317840576, + "learning_rate": 0.0001, + "loss": 228.1141, + "step": 80 + }, + { + "epoch": 0.14511009220537108, + "grad_norm": 5.353086471557617, + "learning_rate": 0.0001, + "loss": 228.1448, + "step": 90 + }, + { + "epoch": 0.16123343578374566, + "grad_norm": 2.251854419708252, + "learning_rate": 0.0001, + "loss": 228.3897, + "step": 100 + }, + { + "epoch": 0.17735677936212021, + "grad_norm": 1.5789915323257446, + "learning_rate": 0.0001, + "loss": 227.7662, + "step": 110 + }, + { + "epoch": 0.1934801229404948, + "grad_norm": 3.7646398544311523, + "learning_rate": 0.0001, + "loss": 223.6411, + "step": 120 + }, + { + "epoch": 0.20960346651886935, + "grad_norm": 3.444000720977783, + "learning_rate": 0.0001, + "loss": 221.2073, + "step": 130 + }, + { + "epoch": 0.2257268100972439, + "grad_norm": 2.1871185302734375, + "learning_rate": 0.0001, + "loss": 215.9944, + "step": 140 + }, + { + "epoch": 0.2418501536756185, + "grad_norm": 2.266244411468506, + "learning_rate": 0.0001, + "loss": 212.3066, + "step": 150 + }, + { + "epoch": 0.25797349725399304, + "grad_norm": 2.043832778930664, + "learning_rate": 0.0001, + "loss": 208.1343, + "step": 160 + }, + { + "epoch": 0.2740968408323676, + "grad_norm": 2.2641494274139404, + "learning_rate": 0.0001, + "loss": 203.1485, + "step": 170 + }, + { + "epoch": 0.29022018441074215, + "grad_norm": 2.9727790355682373, + "learning_rate": 0.0001, + "loss": 198.181, + "step": 180 + }, + { + "epoch": 0.30634352798911674, + "grad_norm": 2.751523494720459, + "learning_rate": 0.0001, + "loss": 194.8422, + "step": 190 + }, + { + "epoch": 0.3224668715674913, + "grad_norm": 1.9561562538146973, + "learning_rate": 0.0001, + "loss": 189.9401, + "step": 200 + }, + { + "epoch": 0.33859021514586585, + "grad_norm": 2.3264107704162598, + "learning_rate": 0.0001, + "loss": 186.1309, + "step": 210 + }, + { + "epoch": 0.35471355872424043, + "grad_norm": 5.212426662445068, + "learning_rate": 0.0001, + "loss": 182.2255, + "step": 220 + }, + { + "epoch": 0.370836902302615, + "grad_norm": 2.488032102584839, + "learning_rate": 0.0001, + "loss": 181.0103, + "step": 230 + }, + { + "epoch": 0.3869602458809896, + "grad_norm": 2.6173202991485596, + "learning_rate": 0.0001, + "loss": 177.6162, + "step": 240 + }, + { + "epoch": 0.4030835894593641, + "grad_norm": 2.6245291233062744, + "learning_rate": 0.0001, + "loss": 173.0431, + "step": 250 + }, + { + "epoch": 0.4192069330377387, + "grad_norm": 5.356006145477295, + "learning_rate": 0.0001, + "loss": 172.2371, + "step": 260 + }, + { + "epoch": 0.4353302766161133, + "grad_norm": 2.0453829765319824, + "learning_rate": 0.0001, + "loss": 169.7781, + "step": 270 + }, + { + "epoch": 0.4514536201944878, + "grad_norm": 1.7002843618392944, + "learning_rate": 0.0001, + "loss": 166.9564, + "step": 280 + }, + { + "epoch": 0.4675769637728624, + "grad_norm": 2.181410312652588, + "learning_rate": 0.0001, + "loss": 166.0094, + "step": 290 + }, + { + "epoch": 0.483700307351237, + "grad_norm": 2.2948265075683594, + "learning_rate": 0.0001, + "loss": 162.4388, + "step": 300 + }, + { + "epoch": 0.4998236509296115, + "grad_norm": 3.0272789001464844, + "learning_rate": 0.0001, + "loss": 160.3446, + "step": 310 + }, + { + "epoch": 0.5159469945079861, + "grad_norm": 1.948184847831726, + "learning_rate": 0.0001, + "loss": 158.488, + "step": 320 + }, + { + "epoch": 0.5320703380863606, + "grad_norm": 1.7981538772583008, + "learning_rate": 0.0001, + "loss": 156.2107, + "step": 330 + }, + { + "epoch": 0.5481936816647353, + "grad_norm": 2.049323797225952, + "learning_rate": 0.0001, + "loss": 155.5434, + "step": 340 + }, + { + "epoch": 0.5643170252431098, + "grad_norm": 1.9415442943572998, + "learning_rate": 0.0001, + "loss": 153.8682, + "step": 350 + }, + { + "epoch": 0.5804403688214843, + "grad_norm": 2.2168045043945312, + "learning_rate": 0.0001, + "loss": 150.4087, + "step": 360 + }, + { + "epoch": 0.596563712399859, + "grad_norm": 1.510750651359558, + "learning_rate": 0.0001, + "loss": 151.7655, + "step": 370 + }, + { + "epoch": 0.6126870559782335, + "grad_norm": 2.7469751834869385, + "learning_rate": 0.0001, + "loss": 147.4665, + "step": 380 + }, + { + "epoch": 0.628810399556608, + "grad_norm": 2.0436975955963135, + "learning_rate": 0.0001, + "loss": 147.4405, + "step": 390 + }, + { + "epoch": 0.6449337431349826, + "grad_norm": 1.8219181299209595, + "learning_rate": 0.0001, + "loss": 146.6332, + "step": 400 + }, + { + "epoch": 0.6610570867133572, + "grad_norm": 1.948431134223938, + "learning_rate": 0.0001, + "loss": 143.3298, + "step": 410 + }, + { + "epoch": 0.6771804302917317, + "grad_norm": 1.8887908458709717, + "learning_rate": 0.0001, + "loss": 140.5058, + "step": 420 + }, + { + "epoch": 0.6933037738701063, + "grad_norm": 1.8739320039749146, + "learning_rate": 0.0001, + "loss": 137.6334, + "step": 430 + }, + { + "epoch": 0.7094271174484809, + "grad_norm": 2.057689666748047, + "learning_rate": 0.0001, + "loss": 135.0343, + "step": 440 + }, + { + "epoch": 0.7255504610268555, + "grad_norm": 1.6926478147506714, + "learning_rate": 0.0001, + "loss": 132.8766, + "step": 450 + }, + { + "epoch": 0.74167380460523, + "grad_norm": 1.5039501190185547, + "learning_rate": 0.0001, + "loss": 131.6774, + "step": 460 + }, + { + "epoch": 0.7577971481836046, + "grad_norm": 1.6903547048568726, + "learning_rate": 0.0001, + "loss": 129.3116, + "step": 470 + }, + { + "epoch": 0.7739204917619792, + "grad_norm": 3.7062244415283203, + "learning_rate": 0.0001, + "loss": 127.9314, + "step": 480 + }, + { + "epoch": 0.7900438353403537, + "grad_norm": 2.0586934089660645, + "learning_rate": 0.0001, + "loss": 126.6398, + "step": 490 + }, + { + "epoch": 0.8061671789187282, + "grad_norm": 1.7598215341567993, + "learning_rate": 0.0001, + "loss": 125.1501, + "step": 500 + }, + { + "epoch": 0.8222905224971029, + "grad_norm": 1.5992189645767212, + "learning_rate": 0.0001, + "loss": 124.7039, + "step": 510 + }, + { + "epoch": 0.8384138660754774, + "grad_norm": 2.8735458850860596, + "learning_rate": 0.0001, + "loss": 122.1619, + "step": 520 + }, + { + "epoch": 0.8545372096538519, + "grad_norm": 1.867098093032837, + "learning_rate": 0.0001, + "loss": 122.4506, + "step": 530 + }, + { + "epoch": 0.8706605532322266, + "grad_norm": 2.2088418006896973, + "learning_rate": 0.0001, + "loss": 122.1047, + "step": 540 + }, + { + "epoch": 0.8867838968106011, + "grad_norm": 1.642657995223999, + "learning_rate": 0.0001, + "loss": 120.1141, + "step": 550 + }, + { + "epoch": 0.9029072403889756, + "grad_norm": 1.5567114353179932, + "learning_rate": 0.0001, + "loss": 118.532, + "step": 560 + }, + { + "epoch": 0.9190305839673503, + "grad_norm": 1.667843222618103, + "learning_rate": 0.0001, + "loss": 118.0734, + "step": 570 + }, + { + "epoch": 0.9351539275457248, + "grad_norm": 1.5304442644119263, + "learning_rate": 0.0001, + "loss": 114.4925, + "step": 580 + }, + { + "epoch": 0.9512772711240993, + "grad_norm": 1.6920255422592163, + "learning_rate": 0.0001, + "loss": 114.0729, + "step": 590 + }, + { + "epoch": 0.967400614702474, + "grad_norm": 1.6313318014144897, + "learning_rate": 0.0001, + "loss": 114.2879, + "step": 600 + }, + { + "epoch": 0.9835239582808485, + "grad_norm": 1.9033657312393188, + "learning_rate": 0.0001, + "loss": 110.5299, + "step": 610 + }, + { + "epoch": 0.999647301859223, + "grad_norm": 1.9010754823684692, + "learning_rate": 0.0001, + "loss": 109.1145, + "step": 620 + }, + { + "epoch": 1.0145110092205372, + "grad_norm": 3.5390989780426025, + "learning_rate": 0.0001, + "loss": 101.0258, + "step": 630 + }, + { + "epoch": 1.0306343527989117, + "grad_norm": 2.2925171852111816, + "learning_rate": 0.0001, + "loss": 107.3693, + "step": 640 + }, + { + "epoch": 1.0467576963772862, + "grad_norm": 2.329108238220215, + "learning_rate": 0.0001, + "loss": 104.3778, + "step": 650 + }, + { + "epoch": 1.0628810399556607, + "grad_norm": 1.7325661182403564, + "learning_rate": 0.0001, + "loss": 102.8409, + "step": 660 + }, + { + "epoch": 1.0790043835340353, + "grad_norm": 1.989586353302002, + "learning_rate": 0.0001, + "loss": 104.2577, + "step": 670 + }, + { + "epoch": 1.09512772711241, + "grad_norm": 1.900484323501587, + "learning_rate": 0.0001, + "loss": 102.3051, + "step": 680 + }, + { + "epoch": 1.1112510706907845, + "grad_norm": 2.015859842300415, + "learning_rate": 0.0001, + "loss": 100.893, + "step": 690 + }, + { + "epoch": 1.127374414269159, + "grad_norm": 1.7695508003234863, + "learning_rate": 0.0001, + "loss": 99.215, + "step": 700 + }, + { + "epoch": 1.1434977578475336, + "grad_norm": 2.025923490524292, + "learning_rate": 0.0001, + "loss": 98.8768, + "step": 710 + }, + { + "epoch": 1.1596211014259081, + "grad_norm": 2.946708917617798, + "learning_rate": 0.0001, + "loss": 96.5588, + "step": 720 + }, + { + "epoch": 1.1757444450042827, + "grad_norm": 2.4511258602142334, + "learning_rate": 0.0001, + "loss": 95.0545, + "step": 730 + }, + { + "epoch": 1.1918677885826574, + "grad_norm": 2.3094682693481445, + "learning_rate": 0.0001, + "loss": 97.004, + "step": 740 + }, + { + "epoch": 1.207991132161032, + "grad_norm": 2.1313347816467285, + "learning_rate": 0.0001, + "loss": 94.8819, + "step": 750 + }, + { + "epoch": 1.2241144757394065, + "grad_norm": 1.614182710647583, + "learning_rate": 0.0001, + "loss": 95.5339, + "step": 760 + }, + { + "epoch": 1.240237819317781, + "grad_norm": 2.029953956604004, + "learning_rate": 0.0001, + "loss": 93.7303, + "step": 770 + }, + { + "epoch": 1.2563611628961555, + "grad_norm": 1.7871663570404053, + "learning_rate": 0.0001, + "loss": 93.8354, + "step": 780 + }, + { + "epoch": 1.2724845064745303, + "grad_norm": 1.72455632686615, + "learning_rate": 0.0001, + "loss": 94.7348, + "step": 790 + }, + { + "epoch": 1.2886078500529048, + "grad_norm": 1.9776639938354492, + "learning_rate": 0.0001, + "loss": 93.5957, + "step": 800 + }, + { + "epoch": 1.3047311936312793, + "grad_norm": 1.7187533378601074, + "learning_rate": 0.0001, + "loss": 92.9583, + "step": 810 + }, + { + "epoch": 1.3208545372096538, + "grad_norm": 1.5949162244796753, + "learning_rate": 0.0001, + "loss": 93.3088, + "step": 820 + }, + { + "epoch": 1.3369778807880284, + "grad_norm": 1.4616206884384155, + "learning_rate": 0.0001, + "loss": 93.2125, + "step": 830 + }, + { + "epoch": 1.353101224366403, + "grad_norm": 2.832163095474243, + "learning_rate": 0.0001, + "loss": 90.3301, + "step": 840 + }, + { + "epoch": 1.3692245679447774, + "grad_norm": 2.174501419067383, + "learning_rate": 0.0001, + "loss": 91.6988, + "step": 850 + }, + { + "epoch": 1.3853479115231522, + "grad_norm": 1.2156339883804321, + "learning_rate": 0.0001, + "loss": 92.7419, + "step": 860 + }, + { + "epoch": 1.4014712551015267, + "grad_norm": 1.608261227607727, + "learning_rate": 0.0001, + "loss": 89.8255, + "step": 870 + }, + { + "epoch": 1.4175945986799012, + "grad_norm": 1.6671923398971558, + "learning_rate": 0.0001, + "loss": 90.9683, + "step": 880 + }, + { + "epoch": 1.4337179422582758, + "grad_norm": 1.2346433401107788, + "learning_rate": 0.0001, + "loss": 90.6092, + "step": 890 + }, + { + "epoch": 1.4498412858366505, + "grad_norm": 1.561085820198059, + "learning_rate": 0.0001, + "loss": 88.0842, + "step": 900 + }, + { + "epoch": 1.465964629415025, + "grad_norm": 1.5475201606750488, + "learning_rate": 0.0001, + "loss": 89.8786, + "step": 910 + }, + { + "epoch": 1.4820879729933996, + "grad_norm": 1.6044800281524658, + "learning_rate": 0.0001, + "loss": 88.0364, + "step": 920 + }, + { + "epoch": 1.498211316571774, + "grad_norm": 1.4081908464431763, + "learning_rate": 0.0001, + "loss": 88.8799, + "step": 930 + }, + { + "epoch": 1.5143346601501486, + "grad_norm": 2.341053009033203, + "learning_rate": 0.0001, + "loss": 88.2023, + "step": 940 + }, + { + "epoch": 1.5304580037285231, + "grad_norm": 1.3976799249649048, + "learning_rate": 0.0001, + "loss": 86.6493, + "step": 950 + }, + { + "epoch": 1.5465813473068977, + "grad_norm": 1.2485483884811401, + "learning_rate": 0.0001, + "loss": 88.9224, + "step": 960 + }, + { + "epoch": 1.5627046908852722, + "grad_norm": 1.6824235916137695, + "learning_rate": 0.0001, + "loss": 89.6484, + "step": 970 + }, + { + "epoch": 1.578828034463647, + "grad_norm": 1.64790940284729, + "learning_rate": 0.0001, + "loss": 87.2479, + "step": 980 + }, + { + "epoch": 1.5949513780420215, + "grad_norm": 1.2093443870544434, + "learning_rate": 0.0001, + "loss": 86.8241, + "step": 990 + }, + { + "epoch": 1.611074721620396, + "grad_norm": 1.5306812524795532, + "learning_rate": 0.0001, + "loss": 88.0006, + "step": 1000 + }, + { + "epoch": 1.6271980651987707, + "grad_norm": 1.2465763092041016, + "learning_rate": 0.0001, + "loss": 86.8143, + "step": 1010 + }, + { + "epoch": 1.6433214087771453, + "grad_norm": 1.5224069356918335, + "learning_rate": 0.0001, + "loss": 87.0102, + "step": 1020 + }, + { + "epoch": 1.6594447523555198, + "grad_norm": 1.312179684638977, + "learning_rate": 0.0001, + "loss": 88.7395, + "step": 1030 + }, + { + "epoch": 1.6755680959338943, + "grad_norm": 1.2343158721923828, + "learning_rate": 0.0001, + "loss": 87.2868, + "step": 1040 + }, + { + "epoch": 1.6916914395122689, + "grad_norm": 1.2778265476226807, + "learning_rate": 0.0001, + "loss": 87.043, + "step": 1050 + }, + { + "epoch": 1.7078147830906434, + "grad_norm": 1.2799627780914307, + "learning_rate": 0.0001, + "loss": 87.7843, + "step": 1060 + }, + { + "epoch": 1.723938126669018, + "grad_norm": 1.3688952922821045, + "learning_rate": 0.0001, + "loss": 85.8985, + "step": 1070 + }, + { + "epoch": 1.7400614702473924, + "grad_norm": 1.4317796230316162, + "learning_rate": 0.0001, + "loss": 85.5408, + "step": 1080 + }, + { + "epoch": 1.756184813825767, + "grad_norm": 1.164801001548767, + "learning_rate": 0.0001, + "loss": 85.8484, + "step": 1090 + }, + { + "epoch": 1.7723081574041417, + "grad_norm": 1.3929173946380615, + "learning_rate": 0.0001, + "loss": 86.9031, + "step": 1100 + }, + { + "epoch": 1.7884315009825162, + "grad_norm": 2.116119861602783, + "learning_rate": 0.0001, + "loss": 86.5343, + "step": 1110 + }, + { + "epoch": 1.8045548445608908, + "grad_norm": 1.4117276668548584, + "learning_rate": 0.0001, + "loss": 85.2678, + "step": 1120 + }, + { + "epoch": 1.8206781881392655, + "grad_norm": 1.3213504552841187, + "learning_rate": 0.0001, + "loss": 85.8935, + "step": 1130 + }, + { + "epoch": 1.83680153171764, + "grad_norm": 1.3589770793914795, + "learning_rate": 0.0001, + "loss": 85.032, + "step": 1140 + }, + { + "epoch": 1.8529248752960146, + "grad_norm": 1.3336007595062256, + "learning_rate": 0.0001, + "loss": 84.0993, + "step": 1150 + }, + { + "epoch": 1.869048218874389, + "grad_norm": 1.1081451177597046, + "learning_rate": 0.0001, + "loss": 84.2225, + "step": 1160 + }, + { + "epoch": 1.8851715624527636, + "grad_norm": 1.482038974761963, + "learning_rate": 0.0001, + "loss": 82.1439, + "step": 1170 + }, + { + "epoch": 1.9012949060311382, + "grad_norm": 1.1571478843688965, + "learning_rate": 0.0001, + "loss": 82.5045, + "step": 1180 + }, + { + "epoch": 1.9174182496095127, + "grad_norm": 1.2665468454360962, + "learning_rate": 0.0001, + "loss": 82.8761, + "step": 1190 + }, + { + "epoch": 1.9335415931878872, + "grad_norm": 1.0771509408950806, + "learning_rate": 0.0001, + "loss": 81.5162, + "step": 1200 + }, + { + "epoch": 1.949664936766262, + "grad_norm": 1.2992299795150757, + "learning_rate": 0.0001, + "loss": 82.4038, + "step": 1210 + }, + { + "epoch": 1.9657882803446365, + "grad_norm": 1.2946668863296509, + "learning_rate": 0.0001, + "loss": 83.4837, + "step": 1220 + }, + { + "epoch": 1.981911623923011, + "grad_norm": 1.3505686521530151, + "learning_rate": 0.0001, + "loss": 81.8993, + "step": 1230 + }, + { + "epoch": 1.9980349675013858, + "grad_norm": 1.1247715950012207, + "learning_rate": 0.0001, + "loss": 82.9006, + "step": 1240 + }, + { + "epoch": 2.0128986748627, + "grad_norm": 1.5652127265930176, + "learning_rate": 0.0001, + "loss": 75.8347, + "step": 1250 + }, + { + "epoch": 2.0290220184410743, + "grad_norm": 7.311964988708496, + "learning_rate": 0.0001, + "loss": 80.5435, + "step": 1260 + }, + { + "epoch": 2.045145362019449, + "grad_norm": 1.2229893207550049, + "learning_rate": 0.0001, + "loss": 83.3277, + "step": 1270 + }, + { + "epoch": 2.0612687055978234, + "grad_norm": 1.1387693881988525, + "learning_rate": 0.0001, + "loss": 81.3252, + "step": 1280 + }, + { + "epoch": 2.077392049176198, + "grad_norm": 1.307201623916626, + "learning_rate": 0.0001, + "loss": 82.3311, + "step": 1290 + }, + { + "epoch": 2.0935153927545724, + "grad_norm": 1.1502896547317505, + "learning_rate": 0.0001, + "loss": 81.1682, + "step": 1300 + }, + { + "epoch": 2.109638736332947, + "grad_norm": 0.9291583299636841, + "learning_rate": 0.0001, + "loss": 81.7017, + "step": 1310 + }, + { + "epoch": 2.1257620799113215, + "grad_norm": 1.0334666967391968, + "learning_rate": 0.0001, + "loss": 81.718, + "step": 1320 + }, + { + "epoch": 2.141885423489696, + "grad_norm": 1.1099056005477905, + "learning_rate": 0.0001, + "loss": 81.2232, + "step": 1330 + }, + { + "epoch": 2.1580087670680705, + "grad_norm": 1.289478063583374, + "learning_rate": 0.0001, + "loss": 81.8736, + "step": 1340 + }, + { + "epoch": 2.174132110646445, + "grad_norm": 1.1634794473648071, + "learning_rate": 0.0001, + "loss": 79.9434, + "step": 1350 + }, + { + "epoch": 2.19025545422482, + "grad_norm": 0.9563440680503845, + "learning_rate": 0.0001, + "loss": 80.628, + "step": 1360 + }, + { + "epoch": 2.2063787978031946, + "grad_norm": 1.1268844604492188, + "learning_rate": 0.0001, + "loss": 82.1389, + "step": 1370 + }, + { + "epoch": 2.222502141381569, + "grad_norm": 1.1050443649291992, + "learning_rate": 0.0001, + "loss": 78.9023, + "step": 1380 + }, + { + "epoch": 2.2386254849599436, + "grad_norm": 1.1936142444610596, + "learning_rate": 0.0001, + "loss": 81.2118, + "step": 1390 + }, + { + "epoch": 2.254748828538318, + "grad_norm": 1.7825465202331543, + "learning_rate": 0.0001, + "loss": 80.8804, + "step": 1400 + }, + { + "epoch": 2.2708721721166927, + "grad_norm": 2.437251329421997, + "learning_rate": 0.0001, + "loss": 79.3807, + "step": 1410 + }, + { + "epoch": 2.286995515695067, + "grad_norm": 1.0604854822158813, + "learning_rate": 0.0001, + "loss": 79.5881, + "step": 1420 + }, + { + "epoch": 2.3031188592734417, + "grad_norm": 1.2285529375076294, + "learning_rate": 0.0001, + "loss": 79.5517, + "step": 1430 + }, + { + "epoch": 2.3192422028518163, + "grad_norm": 1.1916660070419312, + "learning_rate": 0.0001, + "loss": 80.5887, + "step": 1440 + }, + { + "epoch": 2.335365546430191, + "grad_norm": 1.224304437637329, + "learning_rate": 0.0001, + "loss": 79.8109, + "step": 1450 + }, + { + "epoch": 2.3514888900085653, + "grad_norm": 0.9228448867797852, + "learning_rate": 0.0001, + "loss": 79.2787, + "step": 1460 + }, + { + "epoch": 2.3676122335869403, + "grad_norm": 1.2085226774215698, + "learning_rate": 0.0001, + "loss": 78.4983, + "step": 1470 + }, + { + "epoch": 2.383735577165315, + "grad_norm": 2.424083948135376, + "learning_rate": 0.0001, + "loss": 79.8657, + "step": 1480 + }, + { + "epoch": 2.3998589207436893, + "grad_norm": 1.0277849435806274, + "learning_rate": 0.0001, + "loss": 80.6915, + "step": 1490 + }, + { + "epoch": 2.415982264322064, + "grad_norm": 1.0226075649261475, + "learning_rate": 0.0001, + "loss": 78.9751, + "step": 1500 + }, + { + "epoch": 2.4321056079004384, + "grad_norm": 0.8860388398170471, + "learning_rate": 0.0001, + "loss": 78.3915, + "step": 1510 + }, + { + "epoch": 2.448228951478813, + "grad_norm": 1.265637755393982, + "learning_rate": 0.0001, + "loss": 78.8737, + "step": 1520 + }, + { + "epoch": 2.4643522950571874, + "grad_norm": 1.2518186569213867, + "learning_rate": 0.0001, + "loss": 76.5169, + "step": 1530 + }, + { + "epoch": 2.480475638635562, + "grad_norm": 1.074277400970459, + "learning_rate": 0.0001, + "loss": 77.9156, + "step": 1540 + }, + { + "epoch": 2.4965989822139365, + "grad_norm": 1.0666711330413818, + "learning_rate": 0.0001, + "loss": 78.4103, + "step": 1550 + }, + { + "epoch": 2.512722325792311, + "grad_norm": 0.9492248892784119, + "learning_rate": 0.0001, + "loss": 76.7393, + "step": 1560 + }, + { + "epoch": 2.5288456693706856, + "grad_norm": 1.127238154411316, + "learning_rate": 0.0001, + "loss": 79.729, + "step": 1570 + }, + { + "epoch": 2.5449690129490605, + "grad_norm": 1.1520992517471313, + "learning_rate": 0.0001, + "loss": 78.5618, + "step": 1580 + }, + { + "epoch": 2.5610923565274346, + "grad_norm": 0.8632035255432129, + "learning_rate": 0.0001, + "loss": 79.032, + "step": 1590 + }, + { + "epoch": 2.5772157001058096, + "grad_norm": 1.0506329536437988, + "learning_rate": 0.0001, + "loss": 77.6204, + "step": 1600 + }, + { + "epoch": 2.593339043684184, + "grad_norm": 1.019895315170288, + "learning_rate": 0.0001, + "loss": 79.2634, + "step": 1610 + }, + { + "epoch": 2.6094623872625586, + "grad_norm": 1.1685571670532227, + "learning_rate": 0.0001, + "loss": 78.8371, + "step": 1620 + }, + { + "epoch": 2.625585730840933, + "grad_norm": 1.147546648979187, + "learning_rate": 0.0001, + "loss": 78.2113, + "step": 1630 + }, + { + "epoch": 2.6417090744193077, + "grad_norm": 1.7336841821670532, + "learning_rate": 0.0001, + "loss": 78.7866, + "step": 1640 + }, + { + "epoch": 2.657832417997682, + "grad_norm": 1.2150614261627197, + "learning_rate": 0.0001, + "loss": 78.5621, + "step": 1650 + }, + { + "epoch": 2.6739557615760567, + "grad_norm": 1.2257503271102905, + "learning_rate": 0.0001, + "loss": 77.5835, + "step": 1660 + }, + { + "epoch": 2.6900791051544313, + "grad_norm": 1.0375311374664307, + "learning_rate": 0.0001, + "loss": 77.7509, + "step": 1670 + }, + { + "epoch": 2.706202448732806, + "grad_norm": 1.2551004886627197, + "learning_rate": 0.0001, + "loss": 77.8615, + "step": 1680 + }, + { + "epoch": 2.7223257923111808, + "grad_norm": 1.0464223623275757, + "learning_rate": 0.0001, + "loss": 77.2837, + "step": 1690 + }, + { + "epoch": 2.738449135889555, + "grad_norm": 1.0254806280136108, + "learning_rate": 0.0001, + "loss": 76.9616, + "step": 1700 + }, + { + "epoch": 2.75457247946793, + "grad_norm": 1.1665682792663574, + "learning_rate": 0.0001, + "loss": 77.9108, + "step": 1710 + }, + { + "epoch": 2.7706958230463044, + "grad_norm": 1.1349453926086426, + "learning_rate": 0.0001, + "loss": 78.7636, + "step": 1720 + }, + { + "epoch": 2.786819166624679, + "grad_norm": 1.4961057901382446, + "learning_rate": 0.0001, + "loss": 76.3862, + "step": 1730 + }, + { + "epoch": 2.8029425102030534, + "grad_norm": 1.10721755027771, + "learning_rate": 0.0001, + "loss": 77.4105, + "step": 1740 + }, + { + "epoch": 2.819065853781428, + "grad_norm": 0.9878576397895813, + "learning_rate": 0.0001, + "loss": 78.4855, + "step": 1750 + }, + { + "epoch": 2.8351891973598025, + "grad_norm": 1.1722687482833862, + "learning_rate": 0.0001, + "loss": 75.1589, + "step": 1760 + }, + { + "epoch": 2.851312540938177, + "grad_norm": 1.6624362468719482, + "learning_rate": 0.0001, + "loss": 77.1647, + "step": 1770 + }, + { + "epoch": 2.8674358845165515, + "grad_norm": 0.9542295932769775, + "learning_rate": 0.0001, + "loss": 77.1656, + "step": 1780 + }, + { + "epoch": 2.883559228094926, + "grad_norm": 2.328542947769165, + "learning_rate": 0.0001, + "loss": 76.4686, + "step": 1790 + }, + { + "epoch": 2.899682571673301, + "grad_norm": 1.2532029151916504, + "learning_rate": 0.0001, + "loss": 76.1727, + "step": 1800 + }, + { + "epoch": 2.915805915251675, + "grad_norm": 0.9604716897010803, + "learning_rate": 0.0001, + "loss": 74.3132, + "step": 1810 + }, + { + "epoch": 2.93192925883005, + "grad_norm": 0.9182595610618591, + "learning_rate": 0.0001, + "loss": 77.8689, + "step": 1820 + }, + { + "epoch": 2.9480526024084246, + "grad_norm": 0.9623042941093445, + "learning_rate": 0.0001, + "loss": 77.7531, + "step": 1830 + }, + { + "epoch": 2.964175945986799, + "grad_norm": 1.0500119924545288, + "learning_rate": 0.0001, + "loss": 77.6007, + "step": 1840 + }, + { + "epoch": 2.9802992895651736, + "grad_norm": 1.0874103307724, + "learning_rate": 0.0001, + "loss": 77.042, + "step": 1850 + }, + { + "epoch": 2.996422633143548, + "grad_norm": 0.9955475330352783, + "learning_rate": 0.0001, + "loss": 75.468, + "step": 1860 + }, + { + "epoch": 3.011286340504862, + "grad_norm": 1.4281760454177856, + "learning_rate": 0.0001, + "loss": 69.5972, + "step": 1870 + }, + { + "epoch": 3.0274096840832367, + "grad_norm": 1.0292123556137085, + "learning_rate": 0.0001, + "loss": 76.0186, + "step": 1880 + }, + { + "epoch": 3.0435330276616113, + "grad_norm": 0.9535816311836243, + "learning_rate": 0.0001, + "loss": 76.0979, + "step": 1890 + }, + { + "epoch": 3.059656371239986, + "grad_norm": 0.9099674820899963, + "learning_rate": 0.0001, + "loss": 75.9641, + "step": 1900 + }, + { + "epoch": 3.0757797148183603, + "grad_norm": 0.9568380117416382, + "learning_rate": 0.0001, + "loss": 75.0656, + "step": 1910 + }, + { + "epoch": 3.091903058396735, + "grad_norm": 0.9701060652732849, + "learning_rate": 0.0001, + "loss": 75.1422, + "step": 1920 + }, + { + "epoch": 3.1080264019751094, + "grad_norm": 1.3879098892211914, + "learning_rate": 0.0001, + "loss": 76.0731, + "step": 1930 + }, + { + "epoch": 3.1241497455534843, + "grad_norm": 0.9782084226608276, + "learning_rate": 0.0001, + "loss": 75.9666, + "step": 1940 + }, + { + "epoch": 3.140273089131859, + "grad_norm": 1.0856964588165283, + "learning_rate": 0.0001, + "loss": 75.0217, + "step": 1950 + }, + { + "epoch": 3.1563964327102334, + "grad_norm": 0.8506444692611694, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 1960 + }, + { + "epoch": 3.172519776288608, + "grad_norm": 1.044980525970459, + "learning_rate": 0.0001, + "loss": 75.3471, + "step": 1970 + }, + { + "epoch": 3.1886431198669825, + "grad_norm": 0.8432291746139526, + "learning_rate": 0.0001, + "loss": 75.3563, + "step": 1980 + }, + { + "epoch": 3.204766463445357, + "grad_norm": 0.9651834964752197, + "learning_rate": 0.0001, + "loss": 76.3991, + "step": 1990 + }, + { + "epoch": 3.2208898070237315, + "grad_norm": 0.957943320274353, + "learning_rate": 0.0001, + "loss": 77.2613, + "step": 2000 + }, + { + "epoch": 3.237013150602106, + "grad_norm": 0.9495745301246643, + "learning_rate": 0.0001, + "loss": 73.3801, + "step": 2010 + }, + { + "epoch": 3.2531364941804806, + "grad_norm": 1.182983160018921, + "learning_rate": 0.0001, + "loss": 74.5695, + "step": 2020 + }, + { + "epoch": 3.269259837758855, + "grad_norm": 0.828452467918396, + "learning_rate": 0.0001, + "loss": 75.4313, + "step": 2030 + }, + { + "epoch": 3.2853831813372296, + "grad_norm": 0.8340104818344116, + "learning_rate": 0.0001, + "loss": 75.1086, + "step": 2040 + }, + { + "epoch": 3.301506524915604, + "grad_norm": 1.1700762510299683, + "learning_rate": 0.0001, + "loss": 75.7496, + "step": 2050 + }, + { + "epoch": 3.317629868493979, + "grad_norm": 0.8709110617637634, + "learning_rate": 0.0001, + "loss": 73.8403, + "step": 2060 + }, + { + "epoch": 3.3337532120723536, + "grad_norm": 0.8928954601287842, + "learning_rate": 0.0001, + "loss": 75.6134, + "step": 2070 + }, + { + "epoch": 3.349876555650728, + "grad_norm": 0.9787353277206421, + "learning_rate": 0.0001, + "loss": 75.8116, + "step": 2080 + }, + { + "epoch": 3.3659998992291027, + "grad_norm": 0.9515123963356018, + "learning_rate": 0.0001, + "loss": 75.5024, + "step": 2090 + }, + { + "epoch": 3.3821232428074772, + "grad_norm": 0.9529262185096741, + "learning_rate": 0.0001, + "loss": 74.1553, + "step": 2100 + }, + { + "epoch": 3.3982465863858518, + "grad_norm": 0.8934710621833801, + "learning_rate": 0.0001, + "loss": 74.1501, + "step": 2110 + }, + { + "epoch": 3.4143699299642263, + "grad_norm": 1.089447021484375, + "learning_rate": 0.0001, + "loss": 74.9123, + "step": 2120 + }, + { + "epoch": 3.430493273542601, + "grad_norm": 1.2569622993469238, + "learning_rate": 0.0001, + "loss": 74.3962, + "step": 2130 + }, + { + "epoch": 3.4466166171209753, + "grad_norm": 1.020094394683838, + "learning_rate": 0.0001, + "loss": 74.6617, + "step": 2140 + }, + { + "epoch": 3.46273996069935, + "grad_norm": 1.1579667329788208, + "learning_rate": 0.0001, + "loss": 74.2745, + "step": 2150 + }, + { + "epoch": 3.4788633042777244, + "grad_norm": 1.448935627937317, + "learning_rate": 0.0001, + "loss": 71.5285, + "step": 2160 + }, + { + "epoch": 3.4949866478560994, + "grad_norm": 1.0030494928359985, + "learning_rate": 0.0001, + "loss": 74.6666, + "step": 2170 + }, + { + "epoch": 3.511109991434474, + "grad_norm": 0.9010888934135437, + "learning_rate": 0.0001, + "loss": 74.9355, + "step": 2180 + }, + { + "epoch": 3.5272333350128484, + "grad_norm": 1.1595124006271362, + "learning_rate": 0.0001, + "loss": 74.5049, + "step": 2190 + }, + { + "epoch": 3.543356678591223, + "grad_norm": 1.066405177116394, + "learning_rate": 0.0001, + "loss": 74.6336, + "step": 2200 + }, + { + "epoch": 3.5594800221695975, + "grad_norm": 1.2301596403121948, + "learning_rate": 0.0001, + "loss": 74.6664, + "step": 2210 + }, + { + "epoch": 3.575603365747972, + "grad_norm": 1.0073511600494385, + "learning_rate": 0.0001, + "loss": 73.1405, + "step": 2220 + }, + { + "epoch": 3.5917267093263465, + "grad_norm": 0.9913995862007141, + "learning_rate": 0.0001, + "loss": 75.7443, + "step": 2230 + }, + { + "epoch": 3.607850052904721, + "grad_norm": 0.9555648565292358, + "learning_rate": 0.0001, + "loss": 74.8036, + "step": 2240 + }, + { + "epoch": 3.6239733964830956, + "grad_norm": 0.9350649118423462, + "learning_rate": 0.0001, + "loss": 73.5783, + "step": 2250 + }, + { + "epoch": 3.64009674006147, + "grad_norm": 0.935840368270874, + "learning_rate": 0.0001, + "loss": 75.1694, + "step": 2260 + }, + { + "epoch": 3.6562200836398446, + "grad_norm": 0.9303803443908691, + "learning_rate": 0.0001, + "loss": 72.8282, + "step": 2270 + }, + { + "epoch": 3.6723434272182196, + "grad_norm": 0.937164843082428, + "learning_rate": 0.0001, + "loss": 70.3305, + "step": 2280 + }, + { + "epoch": 3.6884667707965937, + "grad_norm": 1.027329444885254, + "learning_rate": 0.0001, + "loss": 73.748, + "step": 2290 + }, + { + "epoch": 3.7045901143749687, + "grad_norm": 1.0915215015411377, + "learning_rate": 0.0001, + "loss": 72.2265, + "step": 2300 + }, + { + "epoch": 3.720713457953343, + "grad_norm": 1.1031261682510376, + "learning_rate": 0.0001, + "loss": 71.2795, + "step": 2310 + }, + { + "epoch": 3.7368368015317177, + "grad_norm": 1.1466140747070312, + "learning_rate": 0.0001, + "loss": 72.0116, + "step": 2320 + }, + { + "epoch": 3.7529601451100922, + "grad_norm": 1.0441662073135376, + "learning_rate": 0.0001, + "loss": 73.3735, + "step": 2330 + }, + { + "epoch": 3.7690834886884668, + "grad_norm": 0.8610432744026184, + "learning_rate": 0.0001, + "loss": 73.3433, + "step": 2340 + }, + { + "epoch": 3.7852068322668413, + "grad_norm": 0.885720431804657, + "learning_rate": 0.0001, + "loss": 73.575, + "step": 2350 + }, + { + "epoch": 3.801330175845216, + "grad_norm": 1.0023950338363647, + "learning_rate": 0.0001, + "loss": 73.4166, + "step": 2360 + }, + { + "epoch": 3.8174535194235903, + "grad_norm": 1.3820304870605469, + "learning_rate": 0.0001, + "loss": 72.9599, + "step": 2370 + }, + { + "epoch": 3.833576863001965, + "grad_norm": 0.9645710587501526, + "learning_rate": 0.0001, + "loss": 72.3922, + "step": 2380 + }, + { + "epoch": 3.84970020658034, + "grad_norm": 1.273369312286377, + "learning_rate": 0.0001, + "loss": 72.0158, + "step": 2390 + }, + { + "epoch": 3.865823550158714, + "grad_norm": 0.859863817691803, + "learning_rate": 0.0001, + "loss": 74.6642, + "step": 2400 + }, + { + "epoch": 3.881946893737089, + "grad_norm": 0.9396682381629944, + "learning_rate": 0.0001, + "loss": 71.9003, + "step": 2410 + }, + { + "epoch": 3.8980702373154634, + "grad_norm": 0.9976151585578918, + "learning_rate": 0.0001, + "loss": 73.0716, + "step": 2420 + }, + { + "epoch": 3.914193580893838, + "grad_norm": 1.0730767250061035, + "learning_rate": 0.0001, + "loss": 72.7847, + "step": 2430 + }, + { + "epoch": 3.9303169244722125, + "grad_norm": 1.002018690109253, + "learning_rate": 0.0001, + "loss": 71.7309, + "step": 2440 + }, + { + "epoch": 3.946440268050587, + "grad_norm": 0.9102753400802612, + "learning_rate": 0.0001, + "loss": 71.616, + "step": 2450 + }, + { + "epoch": 3.9625636116289615, + "grad_norm": 0.9827467799186707, + "learning_rate": 0.0001, + "loss": 72.22, + "step": 2460 + }, + { + "epoch": 3.978686955207336, + "grad_norm": 1.2781893014907837, + "learning_rate": 0.0001, + "loss": 73.2048, + "step": 2470 + }, + { + "epoch": 3.9948102987857106, + "grad_norm": 0.9042410850524902, + "learning_rate": 0.0001, + "loss": 72.9843, + "step": 2480 + }, + { + "epoch": 4.009674006147025, + "grad_norm": 0.8662117123603821, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 2490 + }, + { + "epoch": 4.0257973497254, + "grad_norm": 0.9448756575584412, + "learning_rate": 0.0001, + "loss": 73.36, + "step": 2500 + }, + { + "epoch": 4.041920693303774, + "grad_norm": 0.9054566621780396, + "learning_rate": 0.0001, + "loss": 73.94, + "step": 2510 + }, + { + "epoch": 4.058044036882149, + "grad_norm": 0.957398533821106, + "learning_rate": 0.0001, + "loss": 73.2019, + "step": 2520 + }, + { + "epoch": 4.074167380460523, + "grad_norm": 0.9444408416748047, + "learning_rate": 0.0001, + "loss": 70.6553, + "step": 2530 + }, + { + "epoch": 4.090290724038898, + "grad_norm": 0.8716063499450684, + "learning_rate": 0.0001, + "loss": 70.1048, + "step": 2540 + }, + { + "epoch": 4.106414067617272, + "grad_norm": 0.949249267578125, + "learning_rate": 0.0001, + "loss": 72.4453, + "step": 2550 + }, + { + "epoch": 4.122537411195647, + "grad_norm": 0.9843282699584961, + "learning_rate": 0.0001, + "loss": 72.1725, + "step": 2560 + }, + { + "epoch": 4.138660754774021, + "grad_norm": 0.8977480530738831, + "learning_rate": 0.0001, + "loss": 72.5272, + "step": 2570 + }, + { + "epoch": 4.154784098352396, + "grad_norm": 0.9957827925682068, + "learning_rate": 0.0001, + "loss": 72.289, + "step": 2580 + }, + { + "epoch": 4.170907441930771, + "grad_norm": 1.1358791589736938, + "learning_rate": 0.0001, + "loss": 72.1949, + "step": 2590 + }, + { + "epoch": 4.187030785509145, + "grad_norm": 0.9953081607818604, + "learning_rate": 0.0001, + "loss": 72.6004, + "step": 2600 + }, + { + "epoch": 4.20315412908752, + "grad_norm": 1.019061803817749, + "learning_rate": 0.0001, + "loss": 72.3772, + "step": 2610 + }, + { + "epoch": 4.219277472665894, + "grad_norm": 0.9062562584877014, + "learning_rate": 0.0001, + "loss": 72.0566, + "step": 2620 + }, + { + "epoch": 4.235400816244269, + "grad_norm": 1.0877611637115479, + "learning_rate": 0.0001, + "loss": 72.854, + "step": 2630 + }, + { + "epoch": 4.251524159822643, + "grad_norm": 1.2419648170471191, + "learning_rate": 0.0001, + "loss": 72.6498, + "step": 2640 + }, + { + "epoch": 4.267647503401018, + "grad_norm": 0.9507225751876831, + "learning_rate": 0.0001, + "loss": 68.9583, + "step": 2650 + }, + { + "epoch": 4.283770846979392, + "grad_norm": 1.121191382408142, + "learning_rate": 0.0001, + "loss": 72.8464, + "step": 2660 + }, + { + "epoch": 4.299894190557767, + "grad_norm": 1.0543361902236938, + "learning_rate": 0.0001, + "loss": 71.9882, + "step": 2670 + }, + { + "epoch": 4.316017534136141, + "grad_norm": 0.8622955083847046, + "learning_rate": 0.0001, + "loss": 71.7373, + "step": 2680 + }, + { + "epoch": 4.332140877714516, + "grad_norm": 0.911253809928894, + "learning_rate": 0.0001, + "loss": 71.1968, + "step": 2690 + }, + { + "epoch": 4.34826422129289, + "grad_norm": 4.405472755432129, + "learning_rate": 0.0001, + "loss": 70.4128, + "step": 2700 + }, + { + "epoch": 4.364387564871265, + "grad_norm": 0.9142985939979553, + "learning_rate": 0.0001, + "loss": 69.7548, + "step": 2710 + }, + { + "epoch": 4.38051090844964, + "grad_norm": 1.1022522449493408, + "learning_rate": 0.0001, + "loss": 71.8282, + "step": 2720 + }, + { + "epoch": 4.396634252028014, + "grad_norm": 1.018386721611023, + "learning_rate": 0.0001, + "loss": 71.0853, + "step": 2730 + }, + { + "epoch": 4.412757595606389, + "grad_norm": 1.0483211278915405, + "learning_rate": 0.0001, + "loss": 72.1478, + "step": 2740 + }, + { + "epoch": 4.428880939184763, + "grad_norm": 1.026861548423767, + "learning_rate": 0.0001, + "loss": 71.8947, + "step": 2750 + }, + { + "epoch": 4.445004282763138, + "grad_norm": 1.1858552694320679, + "learning_rate": 0.0001, + "loss": 71.7016, + "step": 2760 + }, + { + "epoch": 4.461127626341512, + "grad_norm": 1.1212221384048462, + "learning_rate": 0.0001, + "loss": 71.4454, + "step": 2770 + }, + { + "epoch": 4.477250969919887, + "grad_norm": 0.8062102794647217, + "learning_rate": 0.0001, + "loss": 73.2672, + "step": 2780 + }, + { + "epoch": 4.493374313498261, + "grad_norm": 1.104483962059021, + "learning_rate": 0.0001, + "loss": 70.548, + "step": 2790 + }, + { + "epoch": 4.509497657076636, + "grad_norm": 0.942057728767395, + "learning_rate": 0.0001, + "loss": 72.3584, + "step": 2800 + }, + { + "epoch": 4.525621000655011, + "grad_norm": 0.9028333425521851, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 2810 + }, + { + "epoch": 4.541744344233385, + "grad_norm": 0.9096795916557312, + "learning_rate": 0.0001, + "loss": 70.6596, + "step": 2820 + }, + { + "epoch": 4.55786768781176, + "grad_norm": 0.9038829207420349, + "learning_rate": 0.0001, + "loss": 72.6148, + "step": 2830 + }, + { + "epoch": 4.573991031390134, + "grad_norm": 1.8511613607406616, + "learning_rate": 0.0001, + "loss": 71.7205, + "step": 2840 + }, + { + "epoch": 4.590114374968509, + "grad_norm": 1.3492343425750732, + "learning_rate": 0.0001, + "loss": 72.1492, + "step": 2850 + }, + { + "epoch": 4.6062377185468835, + "grad_norm": 1.1469528675079346, + "learning_rate": 0.0001, + "loss": 70.5892, + "step": 2860 + }, + { + "epoch": 4.622361062125258, + "grad_norm": 1.1374379396438599, + "learning_rate": 0.0001, + "loss": 70.061, + "step": 2870 + }, + { + "epoch": 4.6384844057036325, + "grad_norm": 0.9597565531730652, + "learning_rate": 0.0001, + "loss": 71.0869, + "step": 2880 + }, + { + "epoch": 4.6546077492820075, + "grad_norm": 0.8356521129608154, + "learning_rate": 0.0001, + "loss": 72.1891, + "step": 2890 + }, + { + "epoch": 4.670731092860382, + "grad_norm": 0.8882647752761841, + "learning_rate": 0.0001, + "loss": 71.3845, + "step": 2900 + }, + { + "epoch": 4.6868544364387565, + "grad_norm": 1.1904003620147705, + "learning_rate": 0.0001, + "loss": 72.3086, + "step": 2910 + }, + { + "epoch": 4.702977780017131, + "grad_norm": 0.8010085821151733, + "learning_rate": 0.0001, + "loss": 71.3051, + "step": 2920 + }, + { + "epoch": 4.719101123595506, + "grad_norm": 0.7572135329246521, + "learning_rate": 0.0001, + "loss": 72.5605, + "step": 2930 + }, + { + "epoch": 4.735224467173881, + "grad_norm": 1.0866942405700684, + "learning_rate": 0.0001, + "loss": 70.0935, + "step": 2940 + }, + { + "epoch": 4.751347810752255, + "grad_norm": 0.9358721971511841, + "learning_rate": 0.0001, + "loss": 71.0391, + "step": 2950 + }, + { + "epoch": 4.76747115433063, + "grad_norm": 1.0276161432266235, + "learning_rate": 0.0001, + "loss": 69.8462, + "step": 2960 + }, + { + "epoch": 4.783594497909004, + "grad_norm": 0.9573063850402832, + "learning_rate": 0.0001, + "loss": 69.7089, + "step": 2970 + }, + { + "epoch": 4.799717841487379, + "grad_norm": 1.0638892650604248, + "learning_rate": 0.0001, + "loss": 71.8376, + "step": 2980 + }, + { + "epoch": 4.815841185065753, + "grad_norm": 0.9894681572914124, + "learning_rate": 0.0001, + "loss": 70.1764, + "step": 2990 + }, + { + "epoch": 4.831964528644128, + "grad_norm": 1.1265769004821777, + "learning_rate": 0.0001, + "loss": 71.7918, + "step": 3000 + }, + { + "epoch": 4.848087872222502, + "grad_norm": 2.2792513370513916, + "learning_rate": 0.0001, + "loss": 70.4341, + "step": 3010 + }, + { + "epoch": 4.864211215800877, + "grad_norm": 0.8469329476356506, + "learning_rate": 0.0001, + "loss": 72.1137, + "step": 3020 + }, + { + "epoch": 4.880334559379252, + "grad_norm": 0.9333013296127319, + "learning_rate": 0.0001, + "loss": 71.0789, + "step": 3030 + }, + { + "epoch": 4.896457902957626, + "grad_norm": 0.7888994216918945, + "learning_rate": 0.0001, + "loss": 70.8197, + "step": 3040 + }, + { + "epoch": 4.912581246536, + "grad_norm": 0.9194874167442322, + "learning_rate": 0.0001, + "loss": 72.071, + "step": 3050 + }, + { + "epoch": 4.928704590114375, + "grad_norm": 1.1252421140670776, + "learning_rate": 0.0001, + "loss": 71.6943, + "step": 3060 + }, + { + "epoch": 4.94482793369275, + "grad_norm": 1.1801071166992188, + "learning_rate": 0.0001, + "loss": 70.8937, + "step": 3070 + }, + { + "epoch": 4.960951277271124, + "grad_norm": 0.9162415266036987, + "learning_rate": 0.0001, + "loss": 70.024, + "step": 3080 + }, + { + "epoch": 4.977074620849499, + "grad_norm": 0.8833363652229309, + "learning_rate": 0.0001, + "loss": 71.3835, + "step": 3090 + }, + { + "epoch": 4.993197964427873, + "grad_norm": 0.9985883831977844, + "learning_rate": 0.0001, + "loss": 71.635, + "step": 3100 + }, + { + "epoch": 5.0080616717891875, + "grad_norm": 1.097298264503479, + "learning_rate": 0.0001, + "loss": 65.3542, + "step": 3110 + }, + { + "epoch": 5.024185015367562, + "grad_norm": 0.9703903198242188, + "learning_rate": 0.0001, + "loss": 69.8587, + "step": 3120 + }, + { + "epoch": 5.0403083589459365, + "grad_norm": 0.8894070386886597, + "learning_rate": 0.0001, + "loss": 71.2868, + "step": 3130 + }, + { + "epoch": 5.056431702524311, + "grad_norm": 0.6984127163887024, + "learning_rate": 0.0001, + "loss": 69.0425, + "step": 3140 + }, + { + "epoch": 5.072555046102686, + "grad_norm": 0.8607762455940247, + "learning_rate": 0.0001, + "loss": 69.5354, + "step": 3150 + }, + { + "epoch": 5.08867838968106, + "grad_norm": 1.1215914487838745, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 3160 + }, + { + "epoch": 5.104801733259435, + "grad_norm": 0.9114409685134888, + "learning_rate": 0.0001, + "loss": 68.4186, + "step": 3170 + }, + { + "epoch": 5.12092507683781, + "grad_norm": 1.010400652885437, + "learning_rate": 0.0001, + "loss": 68.3965, + "step": 3180 + }, + { + "epoch": 5.137048420416184, + "grad_norm": 0.8741958141326904, + "learning_rate": 0.0001, + "loss": 70.6728, + "step": 3190 + }, + { + "epoch": 5.153171763994559, + "grad_norm": 0.8760207891464233, + "learning_rate": 0.0001, + "loss": 72.0704, + "step": 3200 + }, + { + "epoch": 5.169295107572933, + "grad_norm": 1.328247308731079, + "learning_rate": 0.0001, + "loss": 69.9082, + "step": 3210 + }, + { + "epoch": 5.185418451151308, + "grad_norm": 0.8263765573501587, + "learning_rate": 0.0001, + "loss": 70.3539, + "step": 3220 + }, + { + "epoch": 5.201541794729682, + "grad_norm": 0.9358834028244019, + "learning_rate": 0.0001, + "loss": 70.4221, + "step": 3230 + }, + { + "epoch": 5.217665138308057, + "grad_norm": 1.0769531726837158, + "learning_rate": 0.0001, + "loss": 68.8167, + "step": 3240 + }, + { + "epoch": 5.233788481886431, + "grad_norm": 1.1438790559768677, + "learning_rate": 0.0001, + "loss": 68.9288, + "step": 3250 + }, + { + "epoch": 5.249911825464806, + "grad_norm": 0.9534218311309814, + "learning_rate": 0.0001, + "loss": 70.3277, + "step": 3260 + }, + { + "epoch": 5.26603516904318, + "grad_norm": 0.88532954454422, + "learning_rate": 0.0001, + "loss": 67.5781, + "step": 3270 + }, + { + "epoch": 5.282158512621555, + "grad_norm": 0.7986133694648743, + "learning_rate": 0.0001, + "loss": 69.759, + "step": 3280 + }, + { + "epoch": 5.29828185619993, + "grad_norm": 0.9100566506385803, + "learning_rate": 0.0001, + "loss": 68.6368, + "step": 3290 + }, + { + "epoch": 5.314405199778304, + "grad_norm": 0.8861745595932007, + "learning_rate": 0.0001, + "loss": 70.5756, + "step": 3300 + }, + { + "epoch": 5.330528543356679, + "grad_norm": 1.0729680061340332, + "learning_rate": 0.0001, + "loss": 70.3178, + "step": 3310 + }, + { + "epoch": 5.346651886935053, + "grad_norm": 1.0288280248641968, + "learning_rate": 0.0001, + "loss": 69.422, + "step": 3320 + }, + { + "epoch": 5.362775230513428, + "grad_norm": 0.8335140347480774, + "learning_rate": 0.0001, + "loss": 69.8281, + "step": 3330 + }, + { + "epoch": 5.378898574091802, + "grad_norm": 0.9789717197418213, + "learning_rate": 0.0001, + "loss": 71.1532, + "step": 3340 + }, + { + "epoch": 5.395021917670177, + "grad_norm": 0.9884677529335022, + "learning_rate": 0.0001, + "loss": 69.5673, + "step": 3350 + }, + { + "epoch": 5.411145261248551, + "grad_norm": 1.050154447555542, + "learning_rate": 0.0001, + "loss": 70.8015, + "step": 3360 + }, + { + "epoch": 5.427268604826926, + "grad_norm": 0.9645342230796814, + "learning_rate": 0.0001, + "loss": 69.1615, + "step": 3370 + }, + { + "epoch": 5.4433919484053, + "grad_norm": 0.8383159041404724, + "learning_rate": 0.0001, + "loss": 70.6299, + "step": 3380 + }, + { + "epoch": 5.459515291983675, + "grad_norm": 0.9250556826591492, + "learning_rate": 0.0001, + "loss": 69.231, + "step": 3390 + }, + { + "epoch": 5.475638635562049, + "grad_norm": 0.8108687400817871, + "learning_rate": 0.0001, + "loss": 70.5858, + "step": 3400 + }, + { + "epoch": 5.491761979140424, + "grad_norm": 0.8748818039894104, + "learning_rate": 0.0001, + "loss": 69.9001, + "step": 3410 + }, + { + "epoch": 5.507885322718799, + "grad_norm": 0.7411381006240845, + "learning_rate": 0.0001, + "loss": 68.3248, + "step": 3420 + }, + { + "epoch": 5.524008666297173, + "grad_norm": 1.0753326416015625, + "learning_rate": 0.0001, + "loss": 69.5831, + "step": 3430 + }, + { + "epoch": 5.540132009875548, + "grad_norm": 0.875254213809967, + "learning_rate": 0.0001, + "loss": 69.7516, + "step": 3440 + }, + { + "epoch": 5.556255353453922, + "grad_norm": 0.7637733817100525, + "learning_rate": 0.0001, + "loss": 68.0078, + "step": 3450 + }, + { + "epoch": 5.572378697032297, + "grad_norm": 0.968203604221344, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 3460 + }, + { + "epoch": 5.588502040610671, + "grad_norm": 0.8903141617774963, + "learning_rate": 0.0001, + "loss": 68.5835, + "step": 3470 + }, + { + "epoch": 5.604625384189046, + "grad_norm": 0.9536195397377014, + "learning_rate": 0.0001, + "loss": 71.0259, + "step": 3480 + }, + { + "epoch": 5.62074872776742, + "grad_norm": 0.8951714038848877, + "learning_rate": 0.0001, + "loss": 69.4422, + "step": 3490 + }, + { + "epoch": 5.636872071345795, + "grad_norm": 0.9639522433280945, + "learning_rate": 0.0001, + "loss": 66.5173, + "step": 3500 + }, + { + "epoch": 5.65299541492417, + "grad_norm": 1.0022730827331543, + "learning_rate": 0.0001, + "loss": 68.3443, + "step": 3510 + }, + { + "epoch": 5.669118758502544, + "grad_norm": 1.0654487609863281, + "learning_rate": 0.0001, + "loss": 69.0835, + "step": 3520 + }, + { + "epoch": 5.685242102080919, + "grad_norm": 0.9934629797935486, + "learning_rate": 0.0001, + "loss": 69.2278, + "step": 3530 + }, + { + "epoch": 5.7013654456592935, + "grad_norm": 0.8773566484451294, + "learning_rate": 0.0001, + "loss": 68.9339, + "step": 3540 + }, + { + "epoch": 5.7174887892376685, + "grad_norm": 0.7756998538970947, + "learning_rate": 0.0001, + "loss": 69.2683, + "step": 3550 + }, + { + "epoch": 5.7336121328160425, + "grad_norm": 0.8819359540939331, + "learning_rate": 0.0001, + "loss": 68.5071, + "step": 3560 + }, + { + "epoch": 5.7497354763944175, + "grad_norm": 1.1069825887680054, + "learning_rate": 0.0001, + "loss": 68.5236, + "step": 3570 + }, + { + "epoch": 5.765858819972792, + "grad_norm": 0.8006312847137451, + "learning_rate": 0.0001, + "loss": 68.7192, + "step": 3580 + }, + { + "epoch": 5.781982163551167, + "grad_norm": 1.0278719663619995, + "learning_rate": 0.0001, + "loss": 69.2545, + "step": 3590 + }, + { + "epoch": 5.798105507129541, + "grad_norm": 0.9205041527748108, + "learning_rate": 0.0001, + "loss": 69.7698, + "step": 3600 + }, + { + "epoch": 5.814228850707916, + "grad_norm": 1.0769038200378418, + "learning_rate": 0.0001, + "loss": 67.6568, + "step": 3610 + }, + { + "epoch": 5.83035219428629, + "grad_norm": 1.0565663576126099, + "learning_rate": 0.0001, + "loss": 68.9166, + "step": 3620 + }, + { + "epoch": 5.846475537864665, + "grad_norm": 0.9008663296699524, + "learning_rate": 0.0001, + "loss": 69.5837, + "step": 3630 + }, + { + "epoch": 5.86259888144304, + "grad_norm": 0.8369411826133728, + "learning_rate": 0.0001, + "loss": 68.5661, + "step": 3640 + }, + { + "epoch": 5.878722225021414, + "grad_norm": 1.0472379922866821, + "learning_rate": 0.0001, + "loss": 68.3122, + "step": 3650 + }, + { + "epoch": 5.894845568599789, + "grad_norm": 0.965277373790741, + "learning_rate": 0.0001, + "loss": 69.8062, + "step": 3660 + }, + { + "epoch": 5.910968912178163, + "grad_norm": 0.8936495780944824, + "learning_rate": 0.0001, + "loss": 67.1016, + "step": 3670 + }, + { + "epoch": 5.927092255756538, + "grad_norm": 0.8559162020683289, + "learning_rate": 0.0001, + "loss": 69.9986, + "step": 3680 + }, + { + "epoch": 5.943215599334912, + "grad_norm": 1.4815773963928223, + "learning_rate": 0.0001, + "loss": 66.7648, + "step": 3690 + }, + { + "epoch": 5.959338942913287, + "grad_norm": 0.9034469723701477, + "learning_rate": 0.0001, + "loss": 68.0767, + "step": 3700 + }, + { + "epoch": 5.975462286491661, + "grad_norm": 1.838799238204956, + "learning_rate": 0.0001, + "loss": 66.5848, + "step": 3710 + }, + { + "epoch": 5.991585630070036, + "grad_norm": 0.9274787902832031, + "learning_rate": 0.0001, + "loss": 70.0396, + "step": 3720 + }, + { + "epoch": 6.0064493374313495, + "grad_norm": 1.016974687576294, + "learning_rate": 0.0001, + "loss": 62.2268, + "step": 3730 + }, + { + "epoch": 6.022572681009724, + "grad_norm": 1.0522642135620117, + "learning_rate": 0.0001, + "loss": 69.3381, + "step": 3740 + }, + { + "epoch": 6.0386960245880985, + "grad_norm": 1.1157186031341553, + "learning_rate": 0.0001, + "loss": 68.8969, + "step": 3750 + }, + { + "epoch": 6.0548193681664735, + "grad_norm": 0.9913288950920105, + "learning_rate": 0.0001, + "loss": 67.3634, + "step": 3760 + }, + { + "epoch": 6.0709427117448485, + "grad_norm": 0.8823490142822266, + "learning_rate": 0.0001, + "loss": 70.978, + "step": 3770 + }, + { + "epoch": 6.0870660553232225, + "grad_norm": 0.853837788105011, + "learning_rate": 0.0001, + "loss": 68.9924, + "step": 3780 + }, + { + "epoch": 6.1031893989015975, + "grad_norm": 0.7937766909599304, + "learning_rate": 0.0001, + "loss": 68.1994, + "step": 3790 + }, + { + "epoch": 6.119312742479972, + "grad_norm": 1.7638057470321655, + "learning_rate": 0.0001, + "loss": 68.5078, + "step": 3800 + }, + { + "epoch": 6.135436086058347, + "grad_norm": 1.0542887449264526, + "learning_rate": 0.0001, + "loss": 68.9362, + "step": 3810 + }, + { + "epoch": 6.151559429636721, + "grad_norm": 1.06019926071167, + "learning_rate": 0.0001, + "loss": 67.2493, + "step": 3820 + }, + { + "epoch": 6.167682773215096, + "grad_norm": 0.9765111804008484, + "learning_rate": 0.0001, + "loss": 70.4797, + "step": 3830 + }, + { + "epoch": 6.18380611679347, + "grad_norm": 1.006372332572937, + "learning_rate": 0.0001, + "loss": 69.4233, + "step": 3840 + }, + { + "epoch": 6.199929460371845, + "grad_norm": 0.8274241089820862, + "learning_rate": 0.0001, + "loss": 67.9907, + "step": 3850 + }, + { + "epoch": 6.216052803950219, + "grad_norm": 0.9905765056610107, + "learning_rate": 0.0001, + "loss": 69.6988, + "step": 3860 + }, + { + "epoch": 6.232176147528594, + "grad_norm": 1.0065921545028687, + "learning_rate": 0.0001, + "loss": 68.7678, + "step": 3870 + }, + { + "epoch": 6.248299491106969, + "grad_norm": 0.8845255970954895, + "learning_rate": 0.0001, + "loss": 68.3039, + "step": 3880 + }, + { + "epoch": 6.264422834685343, + "grad_norm": 0.9030968546867371, + "learning_rate": 0.0001, + "loss": 68.671, + "step": 3890 + }, + { + "epoch": 6.280546178263718, + "grad_norm": 0.8650258183479309, + "learning_rate": 0.0001, + "loss": 68.5309, + "step": 3900 + }, + { + "epoch": 6.296669521842092, + "grad_norm": 0.8961272835731506, + "learning_rate": 0.0001, + "loss": 67.5611, + "step": 3910 + }, + { + "epoch": 6.312792865420467, + "grad_norm": 1.1702407598495483, + "learning_rate": 0.0001, + "loss": 68.2768, + "step": 3920 + }, + { + "epoch": 6.328916208998841, + "grad_norm": 0.8598962426185608, + "learning_rate": 0.0001, + "loss": 69.1961, + "step": 3930 + }, + { + "epoch": 6.345039552577216, + "grad_norm": 0.8714122176170349, + "learning_rate": 0.0001, + "loss": 66.8628, + "step": 3940 + }, + { + "epoch": 6.36116289615559, + "grad_norm": 0.7757895588874817, + "learning_rate": 0.0001, + "loss": 68.6908, + "step": 3950 + }, + { + "epoch": 6.377286239733965, + "grad_norm": 0.9189490675926208, + "learning_rate": 0.0001, + "loss": 67.8685, + "step": 3960 + }, + { + "epoch": 6.393409583312339, + "grad_norm": 0.8161299228668213, + "learning_rate": 0.0001, + "loss": 66.816, + "step": 3970 + }, + { + "epoch": 6.409532926890714, + "grad_norm": 0.8559791445732117, + "learning_rate": 0.0001, + "loss": 68.4302, + "step": 3980 + }, + { + "epoch": 6.425656270469089, + "grad_norm": 0.8087984919548035, + "learning_rate": 0.0001, + "loss": 66.2646, + "step": 3990 + }, + { + "epoch": 6.441779614047463, + "grad_norm": 0.7733110785484314, + "learning_rate": 0.0001, + "loss": 69.2769, + "step": 4000 + }, + { + "epoch": 6.457902957625838, + "grad_norm": 0.9335984587669373, + "learning_rate": 0.0001, + "loss": 67.1199, + "step": 4010 + }, + { + "epoch": 6.474026301204212, + "grad_norm": 0.9546418786048889, + "learning_rate": 0.0001, + "loss": 68.8706, + "step": 4020 + }, + { + "epoch": 6.490149644782587, + "grad_norm": 0.9670593738555908, + "learning_rate": 0.0001, + "loss": 68.2329, + "step": 4030 + }, + { + "epoch": 6.506272988360961, + "grad_norm": 0.8428588509559631, + "learning_rate": 0.0001, + "loss": 68.0105, + "step": 4040 + }, + { + "epoch": 6.522396331939336, + "grad_norm": 0.9755622744560242, + "learning_rate": 0.0001, + "loss": 68.656, + "step": 4050 + }, + { + "epoch": 6.53851967551771, + "grad_norm": 0.8972346186637878, + "learning_rate": 0.0001, + "loss": 69.3665, + "step": 4060 + }, + { + "epoch": 6.554643019096085, + "grad_norm": 0.9716668128967285, + "learning_rate": 0.0001, + "loss": 68.1053, + "step": 4070 + }, + { + "epoch": 6.570766362674459, + "grad_norm": 0.8147910833358765, + "learning_rate": 0.0001, + "loss": 67.9778, + "step": 4080 + }, + { + "epoch": 6.586889706252834, + "grad_norm": 0.8304234147071838, + "learning_rate": 0.0001, + "loss": 67.7069, + "step": 4090 + }, + { + "epoch": 6.603013049831208, + "grad_norm": 0.9356856346130371, + "learning_rate": 0.0001, + "loss": 68.1942, + "step": 4100 + }, + { + "epoch": 6.619136393409583, + "grad_norm": 1.017134189605713, + "learning_rate": 0.0001, + "loss": 67.4211, + "step": 4110 + }, + { + "epoch": 6.635259736987958, + "grad_norm": 1.142558217048645, + "learning_rate": 0.0001, + "loss": 68.4413, + "step": 4120 + }, + { + "epoch": 6.651383080566332, + "grad_norm": 0.9071153998374939, + "learning_rate": 0.0001, + "loss": 69.3666, + "step": 4130 + }, + { + "epoch": 6.667506424144707, + "grad_norm": 0.9620397686958313, + "learning_rate": 0.0001, + "loss": 68.208, + "step": 4140 + }, + { + "epoch": 6.683629767723081, + "grad_norm": 0.8282128572463989, + "learning_rate": 0.0001, + "loss": 67.1702, + "step": 4150 + }, + { + "epoch": 6.699753111301456, + "grad_norm": 0.8838199973106384, + "learning_rate": 0.0001, + "loss": 69.0074, + "step": 4160 + }, + { + "epoch": 6.71587645487983, + "grad_norm": 0.8864356279373169, + "learning_rate": 0.0001, + "loss": 67.8052, + "step": 4170 + }, + { + "epoch": 6.731999798458205, + "grad_norm": 0.9373385906219482, + "learning_rate": 0.0001, + "loss": 67.446, + "step": 4180 + }, + { + "epoch": 6.7481231420365795, + "grad_norm": 0.8308956027030945, + "learning_rate": 0.0001, + "loss": 68.716, + "step": 4190 + }, + { + "epoch": 6.7642464856149545, + "grad_norm": 0.8031372427940369, + "learning_rate": 0.0001, + "loss": 68.0482, + "step": 4200 + }, + { + "epoch": 6.780369829193329, + "grad_norm": 0.9258586764335632, + "learning_rate": 0.0001, + "loss": 68.5607, + "step": 4210 + }, + { + "epoch": 6.7964931727717035, + "grad_norm": 0.8147327899932861, + "learning_rate": 0.0001, + "loss": 67.7265, + "step": 4220 + }, + { + "epoch": 6.8126165163500785, + "grad_norm": 0.8717535734176636, + "learning_rate": 0.0001, + "loss": 67.8097, + "step": 4230 + }, + { + "epoch": 6.828739859928453, + "grad_norm": 0.9876209497451782, + "learning_rate": 0.0001, + "loss": 67.9068, + "step": 4240 + }, + { + "epoch": 6.8448632035068275, + "grad_norm": 0.718376874923706, + "learning_rate": 0.0001, + "loss": 68.1762, + "step": 4250 + }, + { + "epoch": 6.860986547085202, + "grad_norm": 0.8264320492744446, + "learning_rate": 0.0001, + "loss": 67.8391, + "step": 4260 + }, + { + "epoch": 6.877109890663577, + "grad_norm": 0.7478179335594177, + "learning_rate": 0.0001, + "loss": 67.6635, + "step": 4270 + }, + { + "epoch": 6.893233234241951, + "grad_norm": 0.8394957184791565, + "learning_rate": 0.0001, + "loss": 68.6073, + "step": 4280 + }, + { + "epoch": 6.909356577820326, + "grad_norm": 0.8534071445465088, + "learning_rate": 0.0001, + "loss": 69.6206, + "step": 4290 + }, + { + "epoch": 6.9254799213987, + "grad_norm": 0.9137582778930664, + "learning_rate": 0.0001, + "loss": 67.4362, + "step": 4300 + }, + { + "epoch": 6.941603264977075, + "grad_norm": 0.8869971036911011, + "learning_rate": 0.0001, + "loss": 67.6811, + "step": 4310 + }, + { + "epoch": 6.957726608555449, + "grad_norm": 0.889025866985321, + "learning_rate": 0.0001, + "loss": 65.3354, + "step": 4320 + }, + { + "epoch": 6.973849952133824, + "grad_norm": 0.7968549728393555, + "learning_rate": 0.0001, + "loss": 66.192, + "step": 4330 + }, + { + "epoch": 6.989973295712199, + "grad_norm": 1.0077433586120605, + "learning_rate": 0.0001, + "loss": 67.5216, + "step": 4340 + }, + { + "epoch": 7.004837003073512, + "grad_norm": 0.7900649309158325, + "learning_rate": 0.0001, + "loss": 62.2421, + "step": 4350 + }, + { + "epoch": 7.020960346651887, + "grad_norm": 1.0160410404205322, + "learning_rate": 0.0001, + "loss": 68.0978, + "step": 4360 + }, + { + "epoch": 7.037083690230261, + "grad_norm": 0.841387927532196, + "learning_rate": 0.0001, + "loss": 68.1008, + "step": 4370 + }, + { + "epoch": 7.053207033808636, + "grad_norm": 0.87884122133255, + "learning_rate": 0.0001, + "loss": 66.4667, + "step": 4380 + }, + { + "epoch": 7.06933037738701, + "grad_norm": 0.8580564856529236, + "learning_rate": 0.0001, + "loss": 67.2002, + "step": 4390 + }, + { + "epoch": 7.085453720965385, + "grad_norm": 0.7560063600540161, + "learning_rate": 0.0001, + "loss": 67.6743, + "step": 4400 + }, + { + "epoch": 7.1015770645437595, + "grad_norm": 0.855478048324585, + "learning_rate": 0.0001, + "loss": 67.822, + "step": 4410 + }, + { + "epoch": 7.1177004081221344, + "grad_norm": 0.7994911670684814, + "learning_rate": 0.0001, + "loss": 66.0094, + "step": 4420 + }, + { + "epoch": 7.1338237517005085, + "grad_norm": 0.8728616833686829, + "learning_rate": 0.0001, + "loss": 66.405, + "step": 4430 + }, + { + "epoch": 7.1499470952788835, + "grad_norm": 0.8434081673622131, + "learning_rate": 0.0001, + "loss": 67.0535, + "step": 4440 + }, + { + "epoch": 7.166070438857258, + "grad_norm": 0.8543094396591187, + "learning_rate": 0.0001, + "loss": 66.2417, + "step": 4450 + }, + { + "epoch": 7.182193782435633, + "grad_norm": 0.7209319472312927, + "learning_rate": 0.0001, + "loss": 67.3958, + "step": 4460 + }, + { + "epoch": 7.1983171260140075, + "grad_norm": 0.6735154390335083, + "learning_rate": 0.0001, + "loss": 67.4086, + "step": 4470 + }, + { + "epoch": 7.214440469592382, + "grad_norm": 0.9126104116439819, + "learning_rate": 0.0001, + "loss": 68.4191, + "step": 4480 + }, + { + "epoch": 7.230563813170757, + "grad_norm": 0.9334300756454468, + "learning_rate": 0.0001, + "loss": 68.3247, + "step": 4490 + }, + { + "epoch": 7.246687156749131, + "grad_norm": 1.5956038236618042, + "learning_rate": 0.0001, + "loss": 68.2779, + "step": 4500 + }, + { + "epoch": 7.262810500327506, + "grad_norm": 1.5012775659561157, + "learning_rate": 0.0001, + "loss": 68.3533, + "step": 4510 + }, + { + "epoch": 7.27893384390588, + "grad_norm": 1.1376632452011108, + "learning_rate": 0.0001, + "loss": 65.3627, + "step": 4520 + }, + { + "epoch": 7.295057187484255, + "grad_norm": 0.8597291111946106, + "learning_rate": 0.0001, + "loss": 67.8653, + "step": 4530 + }, + { + "epoch": 7.311180531062629, + "grad_norm": 0.7645048499107361, + "learning_rate": 0.0001, + "loss": 65.4592, + "step": 4540 + }, + { + "epoch": 7.327303874641004, + "grad_norm": 0.7717318534851074, + "learning_rate": 0.0001, + "loss": 66.902, + "step": 4550 + }, + { + "epoch": 7.343427218219378, + "grad_norm": 0.8884950876235962, + "learning_rate": 0.0001, + "loss": 67.1858, + "step": 4560 + }, + { + "epoch": 7.359550561797753, + "grad_norm": 0.7826898694038391, + "learning_rate": 0.0001, + "loss": 68.4673, + "step": 4570 + }, + { + "epoch": 7.375673905376128, + "grad_norm": 0.7484182119369507, + "learning_rate": 0.0001, + "loss": 67.4859, + "step": 4580 + }, + { + "epoch": 7.391797248954502, + "grad_norm": 0.918194591999054, + "learning_rate": 0.0001, + "loss": 67.7393, + "step": 4590 + }, + { + "epoch": 7.407920592532877, + "grad_norm": 1.0696896314620972, + "learning_rate": 0.0001, + "loss": 67.0101, + "step": 4600 + }, + { + "epoch": 7.424043936111251, + "grad_norm": 0.8190692067146301, + "learning_rate": 0.0001, + "loss": 68.5358, + "step": 4610 + }, + { + "epoch": 7.440167279689626, + "grad_norm": 0.9063783884048462, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 4620 + }, + { + "epoch": 7.456290623268, + "grad_norm": 0.7588100433349609, + "learning_rate": 0.0001, + "loss": 68.0066, + "step": 4630 + }, + { + "epoch": 7.472413966846375, + "grad_norm": 0.897281289100647, + "learning_rate": 0.0001, + "loss": 67.2499, + "step": 4640 + }, + { + "epoch": 7.488537310424749, + "grad_norm": 0.9505569338798523, + "learning_rate": 0.0001, + "loss": 67.5452, + "step": 4650 + }, + { + "epoch": 7.504660654003124, + "grad_norm": 0.8409159183502197, + "learning_rate": 0.0001, + "loss": 66.6293, + "step": 4660 + }, + { + "epoch": 7.520783997581498, + "grad_norm": 0.9878574013710022, + "learning_rate": 0.0001, + "loss": 65.3379, + "step": 4670 + }, + { + "epoch": 7.536907341159873, + "grad_norm": 1.0410685539245605, + "learning_rate": 0.0001, + "loss": 65.8028, + "step": 4680 + }, + { + "epoch": 7.553030684738248, + "grad_norm": 0.8417725563049316, + "learning_rate": 0.0001, + "loss": 65.4283, + "step": 4690 + }, + { + "epoch": 7.569154028316622, + "grad_norm": 0.7660863399505615, + "learning_rate": 0.0001, + "loss": 66.6765, + "step": 4700 + }, + { + "epoch": 7.585277371894997, + "grad_norm": 0.8074233531951904, + "learning_rate": 0.0001, + "loss": 67.1082, + "step": 4710 + }, + { + "epoch": 7.601400715473371, + "grad_norm": 0.7653326988220215, + "learning_rate": 0.0001, + "loss": 66.028, + "step": 4720 + }, + { + "epoch": 7.617524059051746, + "grad_norm": 1.1764041185379028, + "learning_rate": 0.0001, + "loss": 65.2587, + "step": 4730 + }, + { + "epoch": 7.63364740263012, + "grad_norm": 1.7604950666427612, + "learning_rate": 0.0001, + "loss": 66.0543, + "step": 4740 + }, + { + "epoch": 7.649770746208495, + "grad_norm": 0.941438615322113, + "learning_rate": 0.0001, + "loss": 65.6954, + "step": 4750 + }, + { + "epoch": 7.665894089786869, + "grad_norm": 0.740069568157196, + "learning_rate": 0.0001, + "loss": 67.9701, + "step": 4760 + }, + { + "epoch": 7.682017433365244, + "grad_norm": 0.9867332577705383, + "learning_rate": 0.0001, + "loss": 66.1445, + "step": 4770 + }, + { + "epoch": 7.698140776943618, + "grad_norm": 0.8156166672706604, + "learning_rate": 0.0001, + "loss": 67.2647, + "step": 4780 + }, + { + "epoch": 7.714264120521993, + "grad_norm": 0.7689579129219055, + "learning_rate": 0.0001, + "loss": 67.0961, + "step": 4790 + }, + { + "epoch": 7.730387464100367, + "grad_norm": 0.8159700036048889, + "learning_rate": 0.0001, + "loss": 66.4974, + "step": 4800 + }, + { + "epoch": 7.746510807678742, + "grad_norm": 0.8554222583770752, + "learning_rate": 0.0001, + "loss": 64.5057, + "step": 4810 + }, + { + "epoch": 7.762634151257117, + "grad_norm": 0.856629490852356, + "learning_rate": 0.0001, + "loss": 67.7819, + "step": 4820 + }, + { + "epoch": 7.778757494835491, + "grad_norm": 1.157179832458496, + "learning_rate": 0.0001, + "loss": 66.0122, + "step": 4830 + }, + { + "epoch": 7.794880838413866, + "grad_norm": 0.9597711563110352, + "learning_rate": 0.0001, + "loss": 66.8248, + "step": 4840 + }, + { + "epoch": 7.8110041819922404, + "grad_norm": 0.9175744652748108, + "learning_rate": 0.0001, + "loss": 65.7998, + "step": 4850 + }, + { + "epoch": 7.827127525570615, + "grad_norm": 0.8467298150062561, + "learning_rate": 0.0001, + "loss": 67.5363, + "step": 4860 + }, + { + "epoch": 7.8432508691489895, + "grad_norm": 0.863922655582428, + "learning_rate": 0.0001, + "loss": 67.7219, + "step": 4870 + }, + { + "epoch": 7.8593742127273645, + "grad_norm": 0.9896290302276611, + "learning_rate": 0.0001, + "loss": 66.7098, + "step": 4880 + }, + { + "epoch": 7.8754975563057386, + "grad_norm": 0.8828520774841309, + "learning_rate": 0.0001, + "loss": 66.0192, + "step": 4890 + }, + { + "epoch": 7.8916208998841135, + "grad_norm": 0.9093656539916992, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 4900 + }, + { + "epoch": 7.9077442434624885, + "grad_norm": 0.8665091395378113, + "learning_rate": 0.0001, + "loss": 66.5851, + "step": 4910 + }, + { + "epoch": 7.923867587040863, + "grad_norm": 0.8027398586273193, + "learning_rate": 0.0001, + "loss": 64.8607, + "step": 4920 + }, + { + "epoch": 7.9399909306192376, + "grad_norm": 0.7808436751365662, + "learning_rate": 0.0001, + "loss": 65.052, + "step": 4930 + }, + { + "epoch": 7.956114274197612, + "grad_norm": 0.7246070504188538, + "learning_rate": 0.0001, + "loss": 64.789, + "step": 4940 + }, + { + "epoch": 7.972237617775987, + "grad_norm": 1.0405027866363525, + "learning_rate": 0.0001, + "loss": 66.7735, + "step": 4950 + }, + { + "epoch": 7.988360961354361, + "grad_norm": 0.812921404838562, + "learning_rate": 0.0001, + "loss": 64.1286, + "step": 4960 + }, + { + "epoch": 8.003224668715674, + "grad_norm": 0.884215772151947, + "learning_rate": 0.0001, + "loss": 61.0812, + "step": 4970 + }, + { + "epoch": 8.01934801229405, + "grad_norm": 0.8519641757011414, + "learning_rate": 0.0001, + "loss": 66.2784, + "step": 4980 + }, + { + "epoch": 8.035471355872424, + "grad_norm": 0.8331199884414673, + "learning_rate": 0.0001, + "loss": 67.1405, + "step": 4990 + }, + { + "epoch": 8.0515946994508, + "grad_norm": 0.8568746447563171, + "learning_rate": 0.0001, + "loss": 66.7835, + "step": 5000 + }, + { + "epoch": 8.067718043029172, + "grad_norm": 1.170482873916626, + "learning_rate": 0.0001, + "loss": 64.2269, + "step": 5010 + }, + { + "epoch": 8.083841386607547, + "grad_norm": 0.9171113967895508, + "learning_rate": 0.0001, + "loss": 66.5567, + "step": 5020 + }, + { + "epoch": 8.099964730185922, + "grad_norm": 0.8988714814186096, + "learning_rate": 0.0001, + "loss": 66.0471, + "step": 5030 + }, + { + "epoch": 8.116088073764297, + "grad_norm": 0.7651671171188354, + "learning_rate": 0.0001, + "loss": 65.7185, + "step": 5040 + }, + { + "epoch": 8.132211417342672, + "grad_norm": 0.6890843510627747, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 5050 + }, + { + "epoch": 8.148334760921045, + "grad_norm": 0.7683818340301514, + "learning_rate": 0.0001, + "loss": 65.895, + "step": 5060 + }, + { + "epoch": 8.16445810449942, + "grad_norm": 0.9398573637008667, + "learning_rate": 0.0001, + "loss": 64.674, + "step": 5070 + }, + { + "epoch": 8.180581448077795, + "grad_norm": 0.7708531022071838, + "learning_rate": 0.0001, + "loss": 65.0637, + "step": 5080 + }, + { + "epoch": 8.19670479165617, + "grad_norm": 0.8840665221214294, + "learning_rate": 0.0001, + "loss": 65.6449, + "step": 5090 + }, + { + "epoch": 8.212828135234544, + "grad_norm": 0.8390524983406067, + "learning_rate": 0.0001, + "loss": 65.4738, + "step": 5100 + }, + { + "epoch": 8.228951478812919, + "grad_norm": 1.7559128999710083, + "learning_rate": 0.0001, + "loss": 68.911, + "step": 5110 + }, + { + "epoch": 8.245074822391294, + "grad_norm": 0.8293817043304443, + "learning_rate": 0.0001, + "loss": 65.2601, + "step": 5120 + }, + { + "epoch": 8.261198165969668, + "grad_norm": 0.8947123289108276, + "learning_rate": 0.0001, + "loss": 65.8611, + "step": 5130 + }, + { + "epoch": 8.277321509548042, + "grad_norm": 0.8526618480682373, + "learning_rate": 0.0001, + "loss": 65.559, + "step": 5140 + }, + { + "epoch": 8.293444853126417, + "grad_norm": 1.7423408031463623, + "learning_rate": 0.0001, + "loss": 64.5007, + "step": 5150 + }, + { + "epoch": 8.309568196704792, + "grad_norm": 1.0512502193450928, + "learning_rate": 0.0001, + "loss": 66.2593, + "step": 5160 + }, + { + "epoch": 8.325691540283167, + "grad_norm": 0.8129869103431702, + "learning_rate": 0.0001, + "loss": 64.7989, + "step": 5170 + }, + { + "epoch": 8.341814883861542, + "grad_norm": 0.852199912071228, + "learning_rate": 0.0001, + "loss": 65.9627, + "step": 5180 + }, + { + "epoch": 8.357938227439915, + "grad_norm": 0.8932474851608276, + "learning_rate": 0.0001, + "loss": 64.1035, + "step": 5190 + }, + { + "epoch": 8.37406157101829, + "grad_norm": 1.2631597518920898, + "learning_rate": 0.0001, + "loss": 65.3406, + "step": 5200 + }, + { + "epoch": 8.390184914596665, + "grad_norm": 0.9062619209289551, + "learning_rate": 0.0001, + "loss": 65.1841, + "step": 5210 + }, + { + "epoch": 8.40630825817504, + "grad_norm": 0.9153940081596375, + "learning_rate": 0.0001, + "loss": 66.3852, + "step": 5220 + }, + { + "epoch": 8.422431601753413, + "grad_norm": 0.8056164979934692, + "learning_rate": 0.0001, + "loss": 66.511, + "step": 5230 + }, + { + "epoch": 8.438554945331788, + "grad_norm": 0.7506036162376404, + "learning_rate": 0.0001, + "loss": 65.275, + "step": 5240 + }, + { + "epoch": 8.454678288910163, + "grad_norm": 0.7994081377983093, + "learning_rate": 0.0001, + "loss": 64.9946, + "step": 5250 + }, + { + "epoch": 8.470801632488538, + "grad_norm": 0.9887873530387878, + "learning_rate": 0.0001, + "loss": 67.2057, + "step": 5260 + }, + { + "epoch": 8.486924976066913, + "grad_norm": 0.8495044112205505, + "learning_rate": 0.0001, + "loss": 66.2481, + "step": 5270 + }, + { + "epoch": 8.503048319645286, + "grad_norm": 0.7353820204734802, + "learning_rate": 0.0001, + "loss": 65.6163, + "step": 5280 + }, + { + "epoch": 8.519171663223661, + "grad_norm": 0.795216977596283, + "learning_rate": 0.0001, + "loss": 64.808, + "step": 5290 + }, + { + "epoch": 8.535295006802036, + "grad_norm": 0.8121851682662964, + "learning_rate": 0.0001, + "loss": 65.9755, + "step": 5300 + }, + { + "epoch": 8.55141835038041, + "grad_norm": 0.8164243698120117, + "learning_rate": 0.0001, + "loss": 66.0583, + "step": 5310 + }, + { + "epoch": 8.567541693958784, + "grad_norm": 0.9893137216567993, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 5320 + }, + { + "epoch": 8.583665037537159, + "grad_norm": 1.3616434335708618, + "learning_rate": 0.0001, + "loss": 64.1444, + "step": 5330 + }, + { + "epoch": 8.599788381115534, + "grad_norm": 0.7002740502357483, + "learning_rate": 0.0001, + "loss": 65.0906, + "step": 5340 + }, + { + "epoch": 8.615911724693909, + "grad_norm": 0.838424563407898, + "learning_rate": 0.0001, + "loss": 65.4578, + "step": 5350 + }, + { + "epoch": 8.632035068272282, + "grad_norm": 0.988902747631073, + "learning_rate": 0.0001, + "loss": 64.3405, + "step": 5360 + }, + { + "epoch": 8.648158411850657, + "grad_norm": 0.7941252589225769, + "learning_rate": 0.0001, + "loss": 67.0146, + "step": 5370 + }, + { + "epoch": 8.664281755429032, + "grad_norm": 0.7727184295654297, + "learning_rate": 0.0001, + "loss": 64.8011, + "step": 5380 + }, + { + "epoch": 8.680405099007407, + "grad_norm": 0.9514367580413818, + "learning_rate": 0.0001, + "loss": 65.8165, + "step": 5390 + }, + { + "epoch": 8.69652844258578, + "grad_norm": 0.870544970035553, + "learning_rate": 0.0001, + "loss": 65.803, + "step": 5400 + }, + { + "epoch": 8.712651786164155, + "grad_norm": 0.7348082065582275, + "learning_rate": 0.0001, + "loss": 65.8409, + "step": 5410 + }, + { + "epoch": 8.72877512974253, + "grad_norm": 0.7543814778327942, + "learning_rate": 0.0001, + "loss": 65.5958, + "step": 5420 + }, + { + "epoch": 8.744898473320905, + "grad_norm": 0.7520368099212646, + "learning_rate": 0.0001, + "loss": 65.3323, + "step": 5430 + }, + { + "epoch": 8.76102181689928, + "grad_norm": 0.7610031366348267, + "learning_rate": 0.0001, + "loss": 65.4386, + "step": 5440 + }, + { + "epoch": 8.777145160477653, + "grad_norm": 0.8854930400848389, + "learning_rate": 0.0001, + "loss": 66.181, + "step": 5450 + }, + { + "epoch": 8.793268504056028, + "grad_norm": 0.7659342885017395, + "learning_rate": 0.0001, + "loss": 65.2586, + "step": 5460 + }, + { + "epoch": 8.809391847634403, + "grad_norm": 0.8592572212219238, + "learning_rate": 0.0001, + "loss": 66.3306, + "step": 5470 + }, + { + "epoch": 8.825515191212778, + "grad_norm": 0.8169172406196594, + "learning_rate": 0.0001, + "loss": 65.8283, + "step": 5480 + }, + { + "epoch": 8.841638534791151, + "grad_norm": 0.7380614876747131, + "learning_rate": 0.0001, + "loss": 65.8711, + "step": 5490 + }, + { + "epoch": 8.857761878369526, + "grad_norm": 0.9993331432342529, + "learning_rate": 0.0001, + "loss": 64.9378, + "step": 5500 + }, + { + "epoch": 8.873885221947901, + "grad_norm": 0.6252862811088562, + "learning_rate": 0.0001, + "loss": 64.1655, + "step": 5510 + }, + { + "epoch": 8.890008565526276, + "grad_norm": 0.8441852331161499, + "learning_rate": 0.0001, + "loss": 64.9795, + "step": 5520 + }, + { + "epoch": 8.906131909104651, + "grad_norm": 0.7075499892234802, + "learning_rate": 0.0001, + "loss": 65.5843, + "step": 5530 + }, + { + "epoch": 8.922255252683025, + "grad_norm": 0.7374144196510315, + "learning_rate": 0.0001, + "loss": 67.5064, + "step": 5540 + }, + { + "epoch": 8.9383785962614, + "grad_norm": 0.8073632121086121, + "learning_rate": 0.0001, + "loss": 67.0182, + "step": 5550 + }, + { + "epoch": 8.954501939839774, + "grad_norm": 0.7576195001602173, + "learning_rate": 0.0001, + "loss": 65.9139, + "step": 5560 + }, + { + "epoch": 8.97062528341815, + "grad_norm": 1.6479626893997192, + "learning_rate": 0.0001, + "loss": 66.3517, + "step": 5570 + }, + { + "epoch": 8.986748626996523, + "grad_norm": 0.846513569355011, + "learning_rate": 0.0001, + "loss": 64.2908, + "step": 5580 + }, + { + "epoch": 9.001612334357837, + "grad_norm": 0.9551037549972534, + "learning_rate": 0.0001, + "loss": 61.7312, + "step": 5590 + }, + { + "epoch": 9.017735677936212, + "grad_norm": 0.8250304460525513, + "learning_rate": 0.0001, + "loss": 66.3351, + "step": 5600 + }, + { + "epoch": 9.033859021514587, + "grad_norm": 0.7932960391044617, + "learning_rate": 0.0001, + "loss": 66.7753, + "step": 5610 + }, + { + "epoch": 9.04998236509296, + "grad_norm": 0.9839881658554077, + "learning_rate": 0.0001, + "loss": 64.5146, + "step": 5620 + }, + { + "epoch": 9.066105708671335, + "grad_norm": 0.7337850332260132, + "learning_rate": 0.0001, + "loss": 65.0544, + "step": 5630 + }, + { + "epoch": 9.08222905224971, + "grad_norm": 0.7496634721755981, + "learning_rate": 0.0001, + "loss": 64.1451, + "step": 5640 + }, + { + "epoch": 9.098352395828085, + "grad_norm": 0.9457297921180725, + "learning_rate": 0.0001, + "loss": 65.3036, + "step": 5650 + }, + { + "epoch": 9.11447573940646, + "grad_norm": 0.8315348625183105, + "learning_rate": 0.0001, + "loss": 66.0585, + "step": 5660 + }, + { + "epoch": 9.130599082984833, + "grad_norm": 0.9666195511817932, + "learning_rate": 0.0001, + "loss": 64.6802, + "step": 5670 + }, + { + "epoch": 9.146722426563208, + "grad_norm": 0.7416576147079468, + "learning_rate": 0.0001, + "loss": 65.1557, + "step": 5680 + }, + { + "epoch": 9.162845770141583, + "grad_norm": 0.7723799347877502, + "learning_rate": 0.0001, + "loss": 66.2871, + "step": 5690 + }, + { + "epoch": 9.178969113719958, + "grad_norm": 0.7461305856704712, + "learning_rate": 0.0001, + "loss": 65.5795, + "step": 5700 + }, + { + "epoch": 9.195092457298331, + "grad_norm": 0.6663411259651184, + "learning_rate": 0.0001, + "loss": 66.4094, + "step": 5710 + }, + { + "epoch": 9.211215800876706, + "grad_norm": 0.9340134859085083, + "learning_rate": 0.0001, + "loss": 63.5753, + "step": 5720 + }, + { + "epoch": 9.227339144455081, + "grad_norm": 0.7643963694572449, + "learning_rate": 0.0001, + "loss": 65.3359, + "step": 5730 + }, + { + "epoch": 9.243462488033456, + "grad_norm": 0.6931570768356323, + "learning_rate": 0.0001, + "loss": 65.0755, + "step": 5740 + }, + { + "epoch": 9.259585831611831, + "grad_norm": 0.9096988439559937, + "learning_rate": 0.0001, + "loss": 64.7983, + "step": 5750 + }, + { + "epoch": 9.275709175190205, + "grad_norm": 0.8360128402709961, + "learning_rate": 0.0001, + "loss": 64.3604, + "step": 5760 + }, + { + "epoch": 9.29183251876858, + "grad_norm": 0.9302591681480408, + "learning_rate": 0.0001, + "loss": 66.2754, + "step": 5770 + }, + { + "epoch": 9.307955862346954, + "grad_norm": 0.7000735998153687, + "learning_rate": 0.0001, + "loss": 65.2241, + "step": 5780 + }, + { + "epoch": 9.32407920592533, + "grad_norm": 0.7321175932884216, + "learning_rate": 0.0001, + "loss": 65.3558, + "step": 5790 + }, + { + "epoch": 9.340202549503703, + "grad_norm": 0.7948715090751648, + "learning_rate": 0.0001, + "loss": 64.2793, + "step": 5800 + }, + { + "epoch": 9.356325893082078, + "grad_norm": 0.7676129341125488, + "learning_rate": 0.0001, + "loss": 66.4778, + "step": 5810 + }, + { + "epoch": 9.372449236660453, + "grad_norm": 0.977236270904541, + "learning_rate": 0.0001, + "loss": 64.5587, + "step": 5820 + }, + { + "epoch": 9.388572580238828, + "grad_norm": 0.86808842420578, + "learning_rate": 0.0001, + "loss": 65.8026, + "step": 5830 + }, + { + "epoch": 9.4046959238172, + "grad_norm": 0.7455324530601501, + "learning_rate": 0.0001, + "loss": 65.066, + "step": 5840 + }, + { + "epoch": 9.420819267395576, + "grad_norm": 0.6784212589263916, + "learning_rate": 0.0001, + "loss": 65.4167, + "step": 5850 + }, + { + "epoch": 9.43694261097395, + "grad_norm": 1.033731460571289, + "learning_rate": 0.0001, + "loss": 66.4502, + "step": 5860 + }, + { + "epoch": 9.453065954552326, + "grad_norm": 0.7946107983589172, + "learning_rate": 0.0001, + "loss": 63.7567, + "step": 5870 + }, + { + "epoch": 9.4691892981307, + "grad_norm": 0.7826364636421204, + "learning_rate": 0.0001, + "loss": 65.4484, + "step": 5880 + }, + { + "epoch": 9.485312641709074, + "grad_norm": 0.7605016827583313, + "learning_rate": 0.0001, + "loss": 66.8459, + "step": 5890 + }, + { + "epoch": 9.501435985287449, + "grad_norm": 0.7383126020431519, + "learning_rate": 0.0001, + "loss": 64.6873, + "step": 5900 + }, + { + "epoch": 9.517559328865824, + "grad_norm": 0.8355686664581299, + "learning_rate": 0.0001, + "loss": 63.5145, + "step": 5910 + }, + { + "epoch": 9.533682672444199, + "grad_norm": 0.8827091455459595, + "learning_rate": 0.0001, + "loss": 67.2207, + "step": 5920 + }, + { + "epoch": 9.549806016022572, + "grad_norm": 0.6988142728805542, + "learning_rate": 0.0001, + "loss": 64.2289, + "step": 5930 + }, + { + "epoch": 9.565929359600947, + "grad_norm": 0.7609000205993652, + "learning_rate": 0.0001, + "loss": 63.9324, + "step": 5940 + }, + { + "epoch": 9.582052703179322, + "grad_norm": 0.8053853511810303, + "learning_rate": 0.0001, + "loss": 62.5564, + "step": 5950 + }, + { + "epoch": 9.598176046757697, + "grad_norm": 0.7684671878814697, + "learning_rate": 0.0001, + "loss": 66.2669, + "step": 5960 + }, + { + "epoch": 9.614299390336072, + "grad_norm": 0.8411065340042114, + "learning_rate": 0.0001, + "loss": 64.5087, + "step": 5970 + }, + { + "epoch": 9.630422733914445, + "grad_norm": 0.963498592376709, + "learning_rate": 0.0001, + "loss": 68.1736, + "step": 5980 + }, + { + "epoch": 9.64654607749282, + "grad_norm": 0.8715981841087341, + "learning_rate": 0.0001, + "loss": 64.1686, + "step": 5990 + }, + { + "epoch": 9.662669421071195, + "grad_norm": 0.8217800855636597, + "learning_rate": 0.0001, + "loss": 64.2211, + "step": 6000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 33, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.338397362934933e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ade09d5233289ba4bc5076af39b8652078582dee --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016a4e9b5d71a1947317d3801b00fe5d0604edbef1f981d7b6c92f219131717c +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-6000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/latest new file mode 100644 index 0000000000000000000000000000000000000000..6c558c76ac9e7515267f79d24d9fde4a7f8688f1 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/latest @@ -0,0 +1 @@ +global_step8000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0c7608c0a01255f3400f896c7d749b774145848f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a234eb639bac0cb4091177999141d481e0014f281d8667682a73f651a307951f +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..081ed8c8e845a002bb61628b48dba5ad148e890d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f1fe005ffd024c663b9c0f100fa0c40e65862ebfb96fde2c48354e8581556d7 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..cd5b83b4e89d3f3e489c65aff95727fbf8d4c8ae --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc2f6e1bdd25c260d578e2b7149793c9279413ec9abfa50ebaee073e177c8576 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..81bd6771012d19582a4c9e84d6bca22e770357f0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a6d0f90c8145353cfb28d25eb8b6300e8ad9b39c0192cad984db3fd6f5c29e +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..19742ea7c2670aac4c14d46a9bf7a72374888b3d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42709833a872ee434c4f84901d41493f80628935e55570d0ebec96d2d173878f +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..43d4ab4ef98bcb9d3e83c70c06c49b5657e6e08b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60368742ad1721cd02b9754cbc0368c7001e2805784ba2b912b2766d9d4531a2 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..c296e03b0bfbe50569909442d8fcde48085e2b39 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08555a4e32fe62db6d6655682dbb0f47148c0671d69006ff6bf9591a3f218853 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..31b9dcf22ba9450ace3e1751e5fc54314a4642bb --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f47b9a31f83c9e9b31b5dd8e47363cf072ab1c8c9f1b375a1c7718b2c1347bd4 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..4ef13d935382582c199d52516c43a327d0cf75d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ebd649d30cd455fc3e6bd517bf1bd1ed079c11d38b3b16594b698cb39223e16 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8fb4ef8eda5769247453e7f4c1cbe7f89242b5c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f130956eca2bbfbb04376b671db10d86b648fabcae3263ea3116e90a644e7ab1 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b13260f924b93cadad950f2069df09f434c62c19 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/trainer_state.json @@ -0,0 +1,5634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 12.883559228094926, + "eval_steps": 2000, + "global_step": 8000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.016123343578374565, + "grad_norm": 48.11856460571289, + "learning_rate": 0.0001, + "loss": 299.1088, + "step": 10 + }, + { + "epoch": 0.03224668715674913, + "grad_norm": 9.07711410522461, + "learning_rate": 0.0001, + "loss": 234.0888, + "step": 20 + }, + { + "epoch": 0.0483700307351237, + "grad_norm": 3.5935330390930176, + "learning_rate": 0.0001, + "loss": 231.4619, + "step": 30 + }, + { + "epoch": 0.06449337431349826, + "grad_norm": 3.5784752368927, + "learning_rate": 0.0001, + "loss": 229.7082, + "step": 40 + }, + { + "epoch": 0.08061671789187283, + "grad_norm": 3.5757956504821777, + "learning_rate": 0.0001, + "loss": 230.1062, + "step": 50 + }, + { + "epoch": 0.0967400614702474, + "grad_norm": 3.8763978481292725, + "learning_rate": 0.0001, + "loss": 229.1914, + "step": 60 + }, + { + "epoch": 0.11286340504862195, + "grad_norm": 3.006988525390625, + "learning_rate": 0.0001, + "loss": 229.6482, + "step": 70 + }, + { + "epoch": 0.12898674862699652, + "grad_norm": 2.3527095317840576, + "learning_rate": 0.0001, + "loss": 228.1141, + "step": 80 + }, + { + "epoch": 0.14511009220537108, + "grad_norm": 5.353086471557617, + "learning_rate": 0.0001, + "loss": 228.1448, + "step": 90 + }, + { + "epoch": 0.16123343578374566, + "grad_norm": 2.251854419708252, + "learning_rate": 0.0001, + "loss": 228.3897, + "step": 100 + }, + { + "epoch": 0.17735677936212021, + "grad_norm": 1.5789915323257446, + "learning_rate": 0.0001, + "loss": 227.7662, + "step": 110 + }, + { + "epoch": 0.1934801229404948, + "grad_norm": 3.7646398544311523, + "learning_rate": 0.0001, + "loss": 223.6411, + "step": 120 + }, + { + "epoch": 0.20960346651886935, + "grad_norm": 3.444000720977783, + "learning_rate": 0.0001, + "loss": 221.2073, + "step": 130 + }, + { + "epoch": 0.2257268100972439, + "grad_norm": 2.1871185302734375, + "learning_rate": 0.0001, + "loss": 215.9944, + "step": 140 + }, + { + "epoch": 0.2418501536756185, + "grad_norm": 2.266244411468506, + "learning_rate": 0.0001, + "loss": 212.3066, + "step": 150 + }, + { + "epoch": 0.25797349725399304, + "grad_norm": 2.043832778930664, + "learning_rate": 0.0001, + "loss": 208.1343, + "step": 160 + }, + { + "epoch": 0.2740968408323676, + "grad_norm": 2.2641494274139404, + "learning_rate": 0.0001, + "loss": 203.1485, + "step": 170 + }, + { + "epoch": 0.29022018441074215, + "grad_norm": 2.9727790355682373, + "learning_rate": 0.0001, + "loss": 198.181, + "step": 180 + }, + { + "epoch": 0.30634352798911674, + "grad_norm": 2.751523494720459, + "learning_rate": 0.0001, + "loss": 194.8422, + "step": 190 + }, + { + "epoch": 0.3224668715674913, + "grad_norm": 1.9561562538146973, + "learning_rate": 0.0001, + "loss": 189.9401, + "step": 200 + }, + { + "epoch": 0.33859021514586585, + "grad_norm": 2.3264107704162598, + "learning_rate": 0.0001, + "loss": 186.1309, + "step": 210 + }, + { + "epoch": 0.35471355872424043, + "grad_norm": 5.212426662445068, + "learning_rate": 0.0001, + "loss": 182.2255, + "step": 220 + }, + { + "epoch": 0.370836902302615, + "grad_norm": 2.488032102584839, + "learning_rate": 0.0001, + "loss": 181.0103, + "step": 230 + }, + { + "epoch": 0.3869602458809896, + "grad_norm": 2.6173202991485596, + "learning_rate": 0.0001, + "loss": 177.6162, + "step": 240 + }, + { + "epoch": 0.4030835894593641, + "grad_norm": 2.6245291233062744, + "learning_rate": 0.0001, + "loss": 173.0431, + "step": 250 + }, + { + "epoch": 0.4192069330377387, + "grad_norm": 5.356006145477295, + "learning_rate": 0.0001, + "loss": 172.2371, + "step": 260 + }, + { + "epoch": 0.4353302766161133, + "grad_norm": 2.0453829765319824, + "learning_rate": 0.0001, + "loss": 169.7781, + "step": 270 + }, + { + "epoch": 0.4514536201944878, + "grad_norm": 1.7002843618392944, + "learning_rate": 0.0001, + "loss": 166.9564, + "step": 280 + }, + { + "epoch": 0.4675769637728624, + "grad_norm": 2.181410312652588, + "learning_rate": 0.0001, + "loss": 166.0094, + "step": 290 + }, + { + "epoch": 0.483700307351237, + "grad_norm": 2.2948265075683594, + "learning_rate": 0.0001, + "loss": 162.4388, + "step": 300 + }, + { + "epoch": 0.4998236509296115, + "grad_norm": 3.0272789001464844, + "learning_rate": 0.0001, + "loss": 160.3446, + "step": 310 + }, + { + "epoch": 0.5159469945079861, + "grad_norm": 1.948184847831726, + "learning_rate": 0.0001, + "loss": 158.488, + "step": 320 + }, + { + "epoch": 0.5320703380863606, + "grad_norm": 1.7981538772583008, + "learning_rate": 0.0001, + "loss": 156.2107, + "step": 330 + }, + { + "epoch": 0.5481936816647353, + "grad_norm": 2.049323797225952, + "learning_rate": 0.0001, + "loss": 155.5434, + "step": 340 + }, + { + "epoch": 0.5643170252431098, + "grad_norm": 1.9415442943572998, + "learning_rate": 0.0001, + "loss": 153.8682, + "step": 350 + }, + { + "epoch": 0.5804403688214843, + "grad_norm": 2.2168045043945312, + "learning_rate": 0.0001, + "loss": 150.4087, + "step": 360 + }, + { + "epoch": 0.596563712399859, + "grad_norm": 1.510750651359558, + "learning_rate": 0.0001, + "loss": 151.7655, + "step": 370 + }, + { + "epoch": 0.6126870559782335, + "grad_norm": 2.7469751834869385, + "learning_rate": 0.0001, + "loss": 147.4665, + "step": 380 + }, + { + "epoch": 0.628810399556608, + "grad_norm": 2.0436975955963135, + "learning_rate": 0.0001, + "loss": 147.4405, + "step": 390 + }, + { + "epoch": 0.6449337431349826, + "grad_norm": 1.8219181299209595, + "learning_rate": 0.0001, + "loss": 146.6332, + "step": 400 + }, + { + "epoch": 0.6610570867133572, + "grad_norm": 1.948431134223938, + "learning_rate": 0.0001, + "loss": 143.3298, + "step": 410 + }, + { + "epoch": 0.6771804302917317, + "grad_norm": 1.8887908458709717, + "learning_rate": 0.0001, + "loss": 140.5058, + "step": 420 + }, + { + "epoch": 0.6933037738701063, + "grad_norm": 1.8739320039749146, + "learning_rate": 0.0001, + "loss": 137.6334, + "step": 430 + }, + { + "epoch": 0.7094271174484809, + "grad_norm": 2.057689666748047, + "learning_rate": 0.0001, + "loss": 135.0343, + "step": 440 + }, + { + "epoch": 0.7255504610268555, + "grad_norm": 1.6926478147506714, + "learning_rate": 0.0001, + "loss": 132.8766, + "step": 450 + }, + { + "epoch": 0.74167380460523, + "grad_norm": 1.5039501190185547, + "learning_rate": 0.0001, + "loss": 131.6774, + "step": 460 + }, + { + "epoch": 0.7577971481836046, + "grad_norm": 1.6903547048568726, + "learning_rate": 0.0001, + "loss": 129.3116, + "step": 470 + }, + { + "epoch": 0.7739204917619792, + "grad_norm": 3.7062244415283203, + "learning_rate": 0.0001, + "loss": 127.9314, + "step": 480 + }, + { + "epoch": 0.7900438353403537, + "grad_norm": 2.0586934089660645, + "learning_rate": 0.0001, + "loss": 126.6398, + "step": 490 + }, + { + "epoch": 0.8061671789187282, + "grad_norm": 1.7598215341567993, + "learning_rate": 0.0001, + "loss": 125.1501, + "step": 500 + }, + { + "epoch": 0.8222905224971029, + "grad_norm": 1.5992189645767212, + "learning_rate": 0.0001, + "loss": 124.7039, + "step": 510 + }, + { + "epoch": 0.8384138660754774, + "grad_norm": 2.8735458850860596, + "learning_rate": 0.0001, + "loss": 122.1619, + "step": 520 + }, + { + "epoch": 0.8545372096538519, + "grad_norm": 1.867098093032837, + "learning_rate": 0.0001, + "loss": 122.4506, + "step": 530 + }, + { + "epoch": 0.8706605532322266, + "grad_norm": 2.2088418006896973, + "learning_rate": 0.0001, + "loss": 122.1047, + "step": 540 + }, + { + "epoch": 0.8867838968106011, + "grad_norm": 1.642657995223999, + "learning_rate": 0.0001, + "loss": 120.1141, + "step": 550 + }, + { + "epoch": 0.9029072403889756, + "grad_norm": 1.5567114353179932, + "learning_rate": 0.0001, + "loss": 118.532, + "step": 560 + }, + { + "epoch": 0.9190305839673503, + "grad_norm": 1.667843222618103, + "learning_rate": 0.0001, + "loss": 118.0734, + "step": 570 + }, + { + "epoch": 0.9351539275457248, + "grad_norm": 1.5304442644119263, + "learning_rate": 0.0001, + "loss": 114.4925, + "step": 580 + }, + { + "epoch": 0.9512772711240993, + "grad_norm": 1.6920255422592163, + "learning_rate": 0.0001, + "loss": 114.0729, + "step": 590 + }, + { + "epoch": 0.967400614702474, + "grad_norm": 1.6313318014144897, + "learning_rate": 0.0001, + "loss": 114.2879, + "step": 600 + }, + { + "epoch": 0.9835239582808485, + "grad_norm": 1.9033657312393188, + "learning_rate": 0.0001, + "loss": 110.5299, + "step": 610 + }, + { + "epoch": 0.999647301859223, + "grad_norm": 1.9010754823684692, + "learning_rate": 0.0001, + "loss": 109.1145, + "step": 620 + }, + { + "epoch": 1.0145110092205372, + "grad_norm": 3.5390989780426025, + "learning_rate": 0.0001, + "loss": 101.0258, + "step": 630 + }, + { + "epoch": 1.0306343527989117, + "grad_norm": 2.2925171852111816, + "learning_rate": 0.0001, + "loss": 107.3693, + "step": 640 + }, + { + "epoch": 1.0467576963772862, + "grad_norm": 2.329108238220215, + "learning_rate": 0.0001, + "loss": 104.3778, + "step": 650 + }, + { + "epoch": 1.0628810399556607, + "grad_norm": 1.7325661182403564, + "learning_rate": 0.0001, + "loss": 102.8409, + "step": 660 + }, + { + "epoch": 1.0790043835340353, + "grad_norm": 1.989586353302002, + "learning_rate": 0.0001, + "loss": 104.2577, + "step": 670 + }, + { + "epoch": 1.09512772711241, + "grad_norm": 1.900484323501587, + "learning_rate": 0.0001, + "loss": 102.3051, + "step": 680 + }, + { + "epoch": 1.1112510706907845, + "grad_norm": 2.015859842300415, + "learning_rate": 0.0001, + "loss": 100.893, + "step": 690 + }, + { + "epoch": 1.127374414269159, + "grad_norm": 1.7695508003234863, + "learning_rate": 0.0001, + "loss": 99.215, + "step": 700 + }, + { + "epoch": 1.1434977578475336, + "grad_norm": 2.025923490524292, + "learning_rate": 0.0001, + "loss": 98.8768, + "step": 710 + }, + { + "epoch": 1.1596211014259081, + "grad_norm": 2.946708917617798, + "learning_rate": 0.0001, + "loss": 96.5588, + "step": 720 + }, + { + "epoch": 1.1757444450042827, + "grad_norm": 2.4511258602142334, + "learning_rate": 0.0001, + "loss": 95.0545, + "step": 730 + }, + { + "epoch": 1.1918677885826574, + "grad_norm": 2.3094682693481445, + "learning_rate": 0.0001, + "loss": 97.004, + "step": 740 + }, + { + "epoch": 1.207991132161032, + "grad_norm": 2.1313347816467285, + "learning_rate": 0.0001, + "loss": 94.8819, + "step": 750 + }, + { + "epoch": 1.2241144757394065, + "grad_norm": 1.614182710647583, + "learning_rate": 0.0001, + "loss": 95.5339, + "step": 760 + }, + { + "epoch": 1.240237819317781, + "grad_norm": 2.029953956604004, + "learning_rate": 0.0001, + "loss": 93.7303, + "step": 770 + }, + { + "epoch": 1.2563611628961555, + "grad_norm": 1.7871663570404053, + "learning_rate": 0.0001, + "loss": 93.8354, + "step": 780 + }, + { + "epoch": 1.2724845064745303, + "grad_norm": 1.72455632686615, + "learning_rate": 0.0001, + "loss": 94.7348, + "step": 790 + }, + { + "epoch": 1.2886078500529048, + "grad_norm": 1.9776639938354492, + "learning_rate": 0.0001, + "loss": 93.5957, + "step": 800 + }, + { + "epoch": 1.3047311936312793, + "grad_norm": 1.7187533378601074, + "learning_rate": 0.0001, + "loss": 92.9583, + "step": 810 + }, + { + "epoch": 1.3208545372096538, + "grad_norm": 1.5949162244796753, + "learning_rate": 0.0001, + "loss": 93.3088, + "step": 820 + }, + { + "epoch": 1.3369778807880284, + "grad_norm": 1.4616206884384155, + "learning_rate": 0.0001, + "loss": 93.2125, + "step": 830 + }, + { + "epoch": 1.353101224366403, + "grad_norm": 2.832163095474243, + "learning_rate": 0.0001, + "loss": 90.3301, + "step": 840 + }, + { + "epoch": 1.3692245679447774, + "grad_norm": 2.174501419067383, + "learning_rate": 0.0001, + "loss": 91.6988, + "step": 850 + }, + { + "epoch": 1.3853479115231522, + "grad_norm": 1.2156339883804321, + "learning_rate": 0.0001, + "loss": 92.7419, + "step": 860 + }, + { + "epoch": 1.4014712551015267, + "grad_norm": 1.608261227607727, + "learning_rate": 0.0001, + "loss": 89.8255, + "step": 870 + }, + { + "epoch": 1.4175945986799012, + "grad_norm": 1.6671923398971558, + "learning_rate": 0.0001, + "loss": 90.9683, + "step": 880 + }, + { + "epoch": 1.4337179422582758, + "grad_norm": 1.2346433401107788, + "learning_rate": 0.0001, + "loss": 90.6092, + "step": 890 + }, + { + "epoch": 1.4498412858366505, + "grad_norm": 1.561085820198059, + "learning_rate": 0.0001, + "loss": 88.0842, + "step": 900 + }, + { + "epoch": 1.465964629415025, + "grad_norm": 1.5475201606750488, + "learning_rate": 0.0001, + "loss": 89.8786, + "step": 910 + }, + { + "epoch": 1.4820879729933996, + "grad_norm": 1.6044800281524658, + "learning_rate": 0.0001, + "loss": 88.0364, + "step": 920 + }, + { + "epoch": 1.498211316571774, + "grad_norm": 1.4081908464431763, + "learning_rate": 0.0001, + "loss": 88.8799, + "step": 930 + }, + { + "epoch": 1.5143346601501486, + "grad_norm": 2.341053009033203, + "learning_rate": 0.0001, + "loss": 88.2023, + "step": 940 + }, + { + "epoch": 1.5304580037285231, + "grad_norm": 1.3976799249649048, + "learning_rate": 0.0001, + "loss": 86.6493, + "step": 950 + }, + { + "epoch": 1.5465813473068977, + "grad_norm": 1.2485483884811401, + "learning_rate": 0.0001, + "loss": 88.9224, + "step": 960 + }, + { + "epoch": 1.5627046908852722, + "grad_norm": 1.6824235916137695, + "learning_rate": 0.0001, + "loss": 89.6484, + "step": 970 + }, + { + "epoch": 1.578828034463647, + "grad_norm": 1.64790940284729, + "learning_rate": 0.0001, + "loss": 87.2479, + "step": 980 + }, + { + "epoch": 1.5949513780420215, + "grad_norm": 1.2093443870544434, + "learning_rate": 0.0001, + "loss": 86.8241, + "step": 990 + }, + { + "epoch": 1.611074721620396, + "grad_norm": 1.5306812524795532, + "learning_rate": 0.0001, + "loss": 88.0006, + "step": 1000 + }, + { + "epoch": 1.6271980651987707, + "grad_norm": 1.2465763092041016, + "learning_rate": 0.0001, + "loss": 86.8143, + "step": 1010 + }, + { + "epoch": 1.6433214087771453, + "grad_norm": 1.5224069356918335, + "learning_rate": 0.0001, + "loss": 87.0102, + "step": 1020 + }, + { + "epoch": 1.6594447523555198, + "grad_norm": 1.312179684638977, + "learning_rate": 0.0001, + "loss": 88.7395, + "step": 1030 + }, + { + "epoch": 1.6755680959338943, + "grad_norm": 1.2343158721923828, + "learning_rate": 0.0001, + "loss": 87.2868, + "step": 1040 + }, + { + "epoch": 1.6916914395122689, + "grad_norm": 1.2778265476226807, + "learning_rate": 0.0001, + "loss": 87.043, + "step": 1050 + }, + { + "epoch": 1.7078147830906434, + "grad_norm": 1.2799627780914307, + "learning_rate": 0.0001, + "loss": 87.7843, + "step": 1060 + }, + { + "epoch": 1.723938126669018, + "grad_norm": 1.3688952922821045, + "learning_rate": 0.0001, + "loss": 85.8985, + "step": 1070 + }, + { + "epoch": 1.7400614702473924, + "grad_norm": 1.4317796230316162, + "learning_rate": 0.0001, + "loss": 85.5408, + "step": 1080 + }, + { + "epoch": 1.756184813825767, + "grad_norm": 1.164801001548767, + "learning_rate": 0.0001, + "loss": 85.8484, + "step": 1090 + }, + { + "epoch": 1.7723081574041417, + "grad_norm": 1.3929173946380615, + "learning_rate": 0.0001, + "loss": 86.9031, + "step": 1100 + }, + { + "epoch": 1.7884315009825162, + "grad_norm": 2.116119861602783, + "learning_rate": 0.0001, + "loss": 86.5343, + "step": 1110 + }, + { + "epoch": 1.8045548445608908, + "grad_norm": 1.4117276668548584, + "learning_rate": 0.0001, + "loss": 85.2678, + "step": 1120 + }, + { + "epoch": 1.8206781881392655, + "grad_norm": 1.3213504552841187, + "learning_rate": 0.0001, + "loss": 85.8935, + "step": 1130 + }, + { + "epoch": 1.83680153171764, + "grad_norm": 1.3589770793914795, + "learning_rate": 0.0001, + "loss": 85.032, + "step": 1140 + }, + { + "epoch": 1.8529248752960146, + "grad_norm": 1.3336007595062256, + "learning_rate": 0.0001, + "loss": 84.0993, + "step": 1150 + }, + { + "epoch": 1.869048218874389, + "grad_norm": 1.1081451177597046, + "learning_rate": 0.0001, + "loss": 84.2225, + "step": 1160 + }, + { + "epoch": 1.8851715624527636, + "grad_norm": 1.482038974761963, + "learning_rate": 0.0001, + "loss": 82.1439, + "step": 1170 + }, + { + "epoch": 1.9012949060311382, + "grad_norm": 1.1571478843688965, + "learning_rate": 0.0001, + "loss": 82.5045, + "step": 1180 + }, + { + "epoch": 1.9174182496095127, + "grad_norm": 1.2665468454360962, + "learning_rate": 0.0001, + "loss": 82.8761, + "step": 1190 + }, + { + "epoch": 1.9335415931878872, + "grad_norm": 1.0771509408950806, + "learning_rate": 0.0001, + "loss": 81.5162, + "step": 1200 + }, + { + "epoch": 1.949664936766262, + "grad_norm": 1.2992299795150757, + "learning_rate": 0.0001, + "loss": 82.4038, + "step": 1210 + }, + { + "epoch": 1.9657882803446365, + "grad_norm": 1.2946668863296509, + "learning_rate": 0.0001, + "loss": 83.4837, + "step": 1220 + }, + { + "epoch": 1.981911623923011, + "grad_norm": 1.3505686521530151, + "learning_rate": 0.0001, + "loss": 81.8993, + "step": 1230 + }, + { + "epoch": 1.9980349675013858, + "grad_norm": 1.1247715950012207, + "learning_rate": 0.0001, + "loss": 82.9006, + "step": 1240 + }, + { + "epoch": 2.0128986748627, + "grad_norm": 1.5652127265930176, + "learning_rate": 0.0001, + "loss": 75.8347, + "step": 1250 + }, + { + "epoch": 2.0290220184410743, + "grad_norm": 7.311964988708496, + "learning_rate": 0.0001, + "loss": 80.5435, + "step": 1260 + }, + { + "epoch": 2.045145362019449, + "grad_norm": 1.2229893207550049, + "learning_rate": 0.0001, + "loss": 83.3277, + "step": 1270 + }, + { + "epoch": 2.0612687055978234, + "grad_norm": 1.1387693881988525, + "learning_rate": 0.0001, + "loss": 81.3252, + "step": 1280 + }, + { + "epoch": 2.077392049176198, + "grad_norm": 1.307201623916626, + "learning_rate": 0.0001, + "loss": 82.3311, + "step": 1290 + }, + { + "epoch": 2.0935153927545724, + "grad_norm": 1.1502896547317505, + "learning_rate": 0.0001, + "loss": 81.1682, + "step": 1300 + }, + { + "epoch": 2.109638736332947, + "grad_norm": 0.9291583299636841, + "learning_rate": 0.0001, + "loss": 81.7017, + "step": 1310 + }, + { + "epoch": 2.1257620799113215, + "grad_norm": 1.0334666967391968, + "learning_rate": 0.0001, + "loss": 81.718, + "step": 1320 + }, + { + "epoch": 2.141885423489696, + "grad_norm": 1.1099056005477905, + "learning_rate": 0.0001, + "loss": 81.2232, + "step": 1330 + }, + { + "epoch": 2.1580087670680705, + "grad_norm": 1.289478063583374, + "learning_rate": 0.0001, + "loss": 81.8736, + "step": 1340 + }, + { + "epoch": 2.174132110646445, + "grad_norm": 1.1634794473648071, + "learning_rate": 0.0001, + "loss": 79.9434, + "step": 1350 + }, + { + "epoch": 2.19025545422482, + "grad_norm": 0.9563440680503845, + "learning_rate": 0.0001, + "loss": 80.628, + "step": 1360 + }, + { + "epoch": 2.2063787978031946, + "grad_norm": 1.1268844604492188, + "learning_rate": 0.0001, + "loss": 82.1389, + "step": 1370 + }, + { + "epoch": 2.222502141381569, + "grad_norm": 1.1050443649291992, + "learning_rate": 0.0001, + "loss": 78.9023, + "step": 1380 + }, + { + "epoch": 2.2386254849599436, + "grad_norm": 1.1936142444610596, + "learning_rate": 0.0001, + "loss": 81.2118, + "step": 1390 + }, + { + "epoch": 2.254748828538318, + "grad_norm": 1.7825465202331543, + "learning_rate": 0.0001, + "loss": 80.8804, + "step": 1400 + }, + { + "epoch": 2.2708721721166927, + "grad_norm": 2.437251329421997, + "learning_rate": 0.0001, + "loss": 79.3807, + "step": 1410 + }, + { + "epoch": 2.286995515695067, + "grad_norm": 1.0604854822158813, + "learning_rate": 0.0001, + "loss": 79.5881, + "step": 1420 + }, + { + "epoch": 2.3031188592734417, + "grad_norm": 1.2285529375076294, + "learning_rate": 0.0001, + "loss": 79.5517, + "step": 1430 + }, + { + "epoch": 2.3192422028518163, + "grad_norm": 1.1916660070419312, + "learning_rate": 0.0001, + "loss": 80.5887, + "step": 1440 + }, + { + "epoch": 2.335365546430191, + "grad_norm": 1.224304437637329, + "learning_rate": 0.0001, + "loss": 79.8109, + "step": 1450 + }, + { + "epoch": 2.3514888900085653, + "grad_norm": 0.9228448867797852, + "learning_rate": 0.0001, + "loss": 79.2787, + "step": 1460 + }, + { + "epoch": 2.3676122335869403, + "grad_norm": 1.2085226774215698, + "learning_rate": 0.0001, + "loss": 78.4983, + "step": 1470 + }, + { + "epoch": 2.383735577165315, + "grad_norm": 2.424083948135376, + "learning_rate": 0.0001, + "loss": 79.8657, + "step": 1480 + }, + { + "epoch": 2.3998589207436893, + "grad_norm": 1.0277849435806274, + "learning_rate": 0.0001, + "loss": 80.6915, + "step": 1490 + }, + { + "epoch": 2.415982264322064, + "grad_norm": 1.0226075649261475, + "learning_rate": 0.0001, + "loss": 78.9751, + "step": 1500 + }, + { + "epoch": 2.4321056079004384, + "grad_norm": 0.8860388398170471, + "learning_rate": 0.0001, + "loss": 78.3915, + "step": 1510 + }, + { + "epoch": 2.448228951478813, + "grad_norm": 1.265637755393982, + "learning_rate": 0.0001, + "loss": 78.8737, + "step": 1520 + }, + { + "epoch": 2.4643522950571874, + "grad_norm": 1.2518186569213867, + "learning_rate": 0.0001, + "loss": 76.5169, + "step": 1530 + }, + { + "epoch": 2.480475638635562, + "grad_norm": 1.074277400970459, + "learning_rate": 0.0001, + "loss": 77.9156, + "step": 1540 + }, + { + "epoch": 2.4965989822139365, + "grad_norm": 1.0666711330413818, + "learning_rate": 0.0001, + "loss": 78.4103, + "step": 1550 + }, + { + "epoch": 2.512722325792311, + "grad_norm": 0.9492248892784119, + "learning_rate": 0.0001, + "loss": 76.7393, + "step": 1560 + }, + { + "epoch": 2.5288456693706856, + "grad_norm": 1.127238154411316, + "learning_rate": 0.0001, + "loss": 79.729, + "step": 1570 + }, + { + "epoch": 2.5449690129490605, + "grad_norm": 1.1520992517471313, + "learning_rate": 0.0001, + "loss": 78.5618, + "step": 1580 + }, + { + "epoch": 2.5610923565274346, + "grad_norm": 0.8632035255432129, + "learning_rate": 0.0001, + "loss": 79.032, + "step": 1590 + }, + { + "epoch": 2.5772157001058096, + "grad_norm": 1.0506329536437988, + "learning_rate": 0.0001, + "loss": 77.6204, + "step": 1600 + }, + { + "epoch": 2.593339043684184, + "grad_norm": 1.019895315170288, + "learning_rate": 0.0001, + "loss": 79.2634, + "step": 1610 + }, + { + "epoch": 2.6094623872625586, + "grad_norm": 1.1685571670532227, + "learning_rate": 0.0001, + "loss": 78.8371, + "step": 1620 + }, + { + "epoch": 2.625585730840933, + "grad_norm": 1.147546648979187, + "learning_rate": 0.0001, + "loss": 78.2113, + "step": 1630 + }, + { + "epoch": 2.6417090744193077, + "grad_norm": 1.7336841821670532, + "learning_rate": 0.0001, + "loss": 78.7866, + "step": 1640 + }, + { + "epoch": 2.657832417997682, + "grad_norm": 1.2150614261627197, + "learning_rate": 0.0001, + "loss": 78.5621, + "step": 1650 + }, + { + "epoch": 2.6739557615760567, + "grad_norm": 1.2257503271102905, + "learning_rate": 0.0001, + "loss": 77.5835, + "step": 1660 + }, + { + "epoch": 2.6900791051544313, + "grad_norm": 1.0375311374664307, + "learning_rate": 0.0001, + "loss": 77.7509, + "step": 1670 + }, + { + "epoch": 2.706202448732806, + "grad_norm": 1.2551004886627197, + "learning_rate": 0.0001, + "loss": 77.8615, + "step": 1680 + }, + { + "epoch": 2.7223257923111808, + "grad_norm": 1.0464223623275757, + "learning_rate": 0.0001, + "loss": 77.2837, + "step": 1690 + }, + { + "epoch": 2.738449135889555, + "grad_norm": 1.0254806280136108, + "learning_rate": 0.0001, + "loss": 76.9616, + "step": 1700 + }, + { + "epoch": 2.75457247946793, + "grad_norm": 1.1665682792663574, + "learning_rate": 0.0001, + "loss": 77.9108, + "step": 1710 + }, + { + "epoch": 2.7706958230463044, + "grad_norm": 1.1349453926086426, + "learning_rate": 0.0001, + "loss": 78.7636, + "step": 1720 + }, + { + "epoch": 2.786819166624679, + "grad_norm": 1.4961057901382446, + "learning_rate": 0.0001, + "loss": 76.3862, + "step": 1730 + }, + { + "epoch": 2.8029425102030534, + "grad_norm": 1.10721755027771, + "learning_rate": 0.0001, + "loss": 77.4105, + "step": 1740 + }, + { + "epoch": 2.819065853781428, + "grad_norm": 0.9878576397895813, + "learning_rate": 0.0001, + "loss": 78.4855, + "step": 1750 + }, + { + "epoch": 2.8351891973598025, + "grad_norm": 1.1722687482833862, + "learning_rate": 0.0001, + "loss": 75.1589, + "step": 1760 + }, + { + "epoch": 2.851312540938177, + "grad_norm": 1.6624362468719482, + "learning_rate": 0.0001, + "loss": 77.1647, + "step": 1770 + }, + { + "epoch": 2.8674358845165515, + "grad_norm": 0.9542295932769775, + "learning_rate": 0.0001, + "loss": 77.1656, + "step": 1780 + }, + { + "epoch": 2.883559228094926, + "grad_norm": 2.328542947769165, + "learning_rate": 0.0001, + "loss": 76.4686, + "step": 1790 + }, + { + "epoch": 2.899682571673301, + "grad_norm": 1.2532029151916504, + "learning_rate": 0.0001, + "loss": 76.1727, + "step": 1800 + }, + { + "epoch": 2.915805915251675, + "grad_norm": 0.9604716897010803, + "learning_rate": 0.0001, + "loss": 74.3132, + "step": 1810 + }, + { + "epoch": 2.93192925883005, + "grad_norm": 0.9182595610618591, + "learning_rate": 0.0001, + "loss": 77.8689, + "step": 1820 + }, + { + "epoch": 2.9480526024084246, + "grad_norm": 0.9623042941093445, + "learning_rate": 0.0001, + "loss": 77.7531, + "step": 1830 + }, + { + "epoch": 2.964175945986799, + "grad_norm": 1.0500119924545288, + "learning_rate": 0.0001, + "loss": 77.6007, + "step": 1840 + }, + { + "epoch": 2.9802992895651736, + "grad_norm": 1.0874103307724, + "learning_rate": 0.0001, + "loss": 77.042, + "step": 1850 + }, + { + "epoch": 2.996422633143548, + "grad_norm": 0.9955475330352783, + "learning_rate": 0.0001, + "loss": 75.468, + "step": 1860 + }, + { + "epoch": 3.011286340504862, + "grad_norm": 1.4281760454177856, + "learning_rate": 0.0001, + "loss": 69.5972, + "step": 1870 + }, + { + "epoch": 3.0274096840832367, + "grad_norm": 1.0292123556137085, + "learning_rate": 0.0001, + "loss": 76.0186, + "step": 1880 + }, + { + "epoch": 3.0435330276616113, + "grad_norm": 0.9535816311836243, + "learning_rate": 0.0001, + "loss": 76.0979, + "step": 1890 + }, + { + "epoch": 3.059656371239986, + "grad_norm": 0.9099674820899963, + "learning_rate": 0.0001, + "loss": 75.9641, + "step": 1900 + }, + { + "epoch": 3.0757797148183603, + "grad_norm": 0.9568380117416382, + "learning_rate": 0.0001, + "loss": 75.0656, + "step": 1910 + }, + { + "epoch": 3.091903058396735, + "grad_norm": 0.9701060652732849, + "learning_rate": 0.0001, + "loss": 75.1422, + "step": 1920 + }, + { + "epoch": 3.1080264019751094, + "grad_norm": 1.3879098892211914, + "learning_rate": 0.0001, + "loss": 76.0731, + "step": 1930 + }, + { + "epoch": 3.1241497455534843, + "grad_norm": 0.9782084226608276, + "learning_rate": 0.0001, + "loss": 75.9666, + "step": 1940 + }, + { + "epoch": 3.140273089131859, + "grad_norm": 1.0856964588165283, + "learning_rate": 0.0001, + "loss": 75.0217, + "step": 1950 + }, + { + "epoch": 3.1563964327102334, + "grad_norm": 0.8506444692611694, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 1960 + }, + { + "epoch": 3.172519776288608, + "grad_norm": 1.044980525970459, + "learning_rate": 0.0001, + "loss": 75.3471, + "step": 1970 + }, + { + "epoch": 3.1886431198669825, + "grad_norm": 0.8432291746139526, + "learning_rate": 0.0001, + "loss": 75.3563, + "step": 1980 + }, + { + "epoch": 3.204766463445357, + "grad_norm": 0.9651834964752197, + "learning_rate": 0.0001, + "loss": 76.3991, + "step": 1990 + }, + { + "epoch": 3.2208898070237315, + "grad_norm": 0.957943320274353, + "learning_rate": 0.0001, + "loss": 77.2613, + "step": 2000 + }, + { + "epoch": 3.237013150602106, + "grad_norm": 0.9495745301246643, + "learning_rate": 0.0001, + "loss": 73.3801, + "step": 2010 + }, + { + "epoch": 3.2531364941804806, + "grad_norm": 1.182983160018921, + "learning_rate": 0.0001, + "loss": 74.5695, + "step": 2020 + }, + { + "epoch": 3.269259837758855, + "grad_norm": 0.828452467918396, + "learning_rate": 0.0001, + "loss": 75.4313, + "step": 2030 + }, + { + "epoch": 3.2853831813372296, + "grad_norm": 0.8340104818344116, + "learning_rate": 0.0001, + "loss": 75.1086, + "step": 2040 + }, + { + "epoch": 3.301506524915604, + "grad_norm": 1.1700762510299683, + "learning_rate": 0.0001, + "loss": 75.7496, + "step": 2050 + }, + { + "epoch": 3.317629868493979, + "grad_norm": 0.8709110617637634, + "learning_rate": 0.0001, + "loss": 73.8403, + "step": 2060 + }, + { + "epoch": 3.3337532120723536, + "grad_norm": 0.8928954601287842, + "learning_rate": 0.0001, + "loss": 75.6134, + "step": 2070 + }, + { + "epoch": 3.349876555650728, + "grad_norm": 0.9787353277206421, + "learning_rate": 0.0001, + "loss": 75.8116, + "step": 2080 + }, + { + "epoch": 3.3659998992291027, + "grad_norm": 0.9515123963356018, + "learning_rate": 0.0001, + "loss": 75.5024, + "step": 2090 + }, + { + "epoch": 3.3821232428074772, + "grad_norm": 0.9529262185096741, + "learning_rate": 0.0001, + "loss": 74.1553, + "step": 2100 + }, + { + "epoch": 3.3982465863858518, + "grad_norm": 0.8934710621833801, + "learning_rate": 0.0001, + "loss": 74.1501, + "step": 2110 + }, + { + "epoch": 3.4143699299642263, + "grad_norm": 1.089447021484375, + "learning_rate": 0.0001, + "loss": 74.9123, + "step": 2120 + }, + { + "epoch": 3.430493273542601, + "grad_norm": 1.2569622993469238, + "learning_rate": 0.0001, + "loss": 74.3962, + "step": 2130 + }, + { + "epoch": 3.4466166171209753, + "grad_norm": 1.020094394683838, + "learning_rate": 0.0001, + "loss": 74.6617, + "step": 2140 + }, + { + "epoch": 3.46273996069935, + "grad_norm": 1.1579667329788208, + "learning_rate": 0.0001, + "loss": 74.2745, + "step": 2150 + }, + { + "epoch": 3.4788633042777244, + "grad_norm": 1.448935627937317, + "learning_rate": 0.0001, + "loss": 71.5285, + "step": 2160 + }, + { + "epoch": 3.4949866478560994, + "grad_norm": 1.0030494928359985, + "learning_rate": 0.0001, + "loss": 74.6666, + "step": 2170 + }, + { + "epoch": 3.511109991434474, + "grad_norm": 0.9010888934135437, + "learning_rate": 0.0001, + "loss": 74.9355, + "step": 2180 + }, + { + "epoch": 3.5272333350128484, + "grad_norm": 1.1595124006271362, + "learning_rate": 0.0001, + "loss": 74.5049, + "step": 2190 + }, + { + "epoch": 3.543356678591223, + "grad_norm": 1.066405177116394, + "learning_rate": 0.0001, + "loss": 74.6336, + "step": 2200 + }, + { + "epoch": 3.5594800221695975, + "grad_norm": 1.2301596403121948, + "learning_rate": 0.0001, + "loss": 74.6664, + "step": 2210 + }, + { + "epoch": 3.575603365747972, + "grad_norm": 1.0073511600494385, + "learning_rate": 0.0001, + "loss": 73.1405, + "step": 2220 + }, + { + "epoch": 3.5917267093263465, + "grad_norm": 0.9913995862007141, + "learning_rate": 0.0001, + "loss": 75.7443, + "step": 2230 + }, + { + "epoch": 3.607850052904721, + "grad_norm": 0.9555648565292358, + "learning_rate": 0.0001, + "loss": 74.8036, + "step": 2240 + }, + { + "epoch": 3.6239733964830956, + "grad_norm": 0.9350649118423462, + "learning_rate": 0.0001, + "loss": 73.5783, + "step": 2250 + }, + { + "epoch": 3.64009674006147, + "grad_norm": 0.935840368270874, + "learning_rate": 0.0001, + "loss": 75.1694, + "step": 2260 + }, + { + "epoch": 3.6562200836398446, + "grad_norm": 0.9303803443908691, + "learning_rate": 0.0001, + "loss": 72.8282, + "step": 2270 + }, + { + "epoch": 3.6723434272182196, + "grad_norm": 0.937164843082428, + "learning_rate": 0.0001, + "loss": 70.3305, + "step": 2280 + }, + { + "epoch": 3.6884667707965937, + "grad_norm": 1.027329444885254, + "learning_rate": 0.0001, + "loss": 73.748, + "step": 2290 + }, + { + "epoch": 3.7045901143749687, + "grad_norm": 1.0915215015411377, + "learning_rate": 0.0001, + "loss": 72.2265, + "step": 2300 + }, + { + "epoch": 3.720713457953343, + "grad_norm": 1.1031261682510376, + "learning_rate": 0.0001, + "loss": 71.2795, + "step": 2310 + }, + { + "epoch": 3.7368368015317177, + "grad_norm": 1.1466140747070312, + "learning_rate": 0.0001, + "loss": 72.0116, + "step": 2320 + }, + { + "epoch": 3.7529601451100922, + "grad_norm": 1.0441662073135376, + "learning_rate": 0.0001, + "loss": 73.3735, + "step": 2330 + }, + { + "epoch": 3.7690834886884668, + "grad_norm": 0.8610432744026184, + "learning_rate": 0.0001, + "loss": 73.3433, + "step": 2340 + }, + { + "epoch": 3.7852068322668413, + "grad_norm": 0.885720431804657, + "learning_rate": 0.0001, + "loss": 73.575, + "step": 2350 + }, + { + "epoch": 3.801330175845216, + "grad_norm": 1.0023950338363647, + "learning_rate": 0.0001, + "loss": 73.4166, + "step": 2360 + }, + { + "epoch": 3.8174535194235903, + "grad_norm": 1.3820304870605469, + "learning_rate": 0.0001, + "loss": 72.9599, + "step": 2370 + }, + { + "epoch": 3.833576863001965, + "grad_norm": 0.9645710587501526, + "learning_rate": 0.0001, + "loss": 72.3922, + "step": 2380 + }, + { + "epoch": 3.84970020658034, + "grad_norm": 1.273369312286377, + "learning_rate": 0.0001, + "loss": 72.0158, + "step": 2390 + }, + { + "epoch": 3.865823550158714, + "grad_norm": 0.859863817691803, + "learning_rate": 0.0001, + "loss": 74.6642, + "step": 2400 + }, + { + "epoch": 3.881946893737089, + "grad_norm": 0.9396682381629944, + "learning_rate": 0.0001, + "loss": 71.9003, + "step": 2410 + }, + { + "epoch": 3.8980702373154634, + "grad_norm": 0.9976151585578918, + "learning_rate": 0.0001, + "loss": 73.0716, + "step": 2420 + }, + { + "epoch": 3.914193580893838, + "grad_norm": 1.0730767250061035, + "learning_rate": 0.0001, + "loss": 72.7847, + "step": 2430 + }, + { + "epoch": 3.9303169244722125, + "grad_norm": 1.002018690109253, + "learning_rate": 0.0001, + "loss": 71.7309, + "step": 2440 + }, + { + "epoch": 3.946440268050587, + "grad_norm": 0.9102753400802612, + "learning_rate": 0.0001, + "loss": 71.616, + "step": 2450 + }, + { + "epoch": 3.9625636116289615, + "grad_norm": 0.9827467799186707, + "learning_rate": 0.0001, + "loss": 72.22, + "step": 2460 + }, + { + "epoch": 3.978686955207336, + "grad_norm": 1.2781893014907837, + "learning_rate": 0.0001, + "loss": 73.2048, + "step": 2470 + }, + { + "epoch": 3.9948102987857106, + "grad_norm": 0.9042410850524902, + "learning_rate": 0.0001, + "loss": 72.9843, + "step": 2480 + }, + { + "epoch": 4.009674006147025, + "grad_norm": 0.8662117123603821, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 2490 + }, + { + "epoch": 4.0257973497254, + "grad_norm": 0.9448756575584412, + "learning_rate": 0.0001, + "loss": 73.36, + "step": 2500 + }, + { + "epoch": 4.041920693303774, + "grad_norm": 0.9054566621780396, + "learning_rate": 0.0001, + "loss": 73.94, + "step": 2510 + }, + { + "epoch": 4.058044036882149, + "grad_norm": 0.957398533821106, + "learning_rate": 0.0001, + "loss": 73.2019, + "step": 2520 + }, + { + "epoch": 4.074167380460523, + "grad_norm": 0.9444408416748047, + "learning_rate": 0.0001, + "loss": 70.6553, + "step": 2530 + }, + { + "epoch": 4.090290724038898, + "grad_norm": 0.8716063499450684, + "learning_rate": 0.0001, + "loss": 70.1048, + "step": 2540 + }, + { + "epoch": 4.106414067617272, + "grad_norm": 0.949249267578125, + "learning_rate": 0.0001, + "loss": 72.4453, + "step": 2550 + }, + { + "epoch": 4.122537411195647, + "grad_norm": 0.9843282699584961, + "learning_rate": 0.0001, + "loss": 72.1725, + "step": 2560 + }, + { + "epoch": 4.138660754774021, + "grad_norm": 0.8977480530738831, + "learning_rate": 0.0001, + "loss": 72.5272, + "step": 2570 + }, + { + "epoch": 4.154784098352396, + "grad_norm": 0.9957827925682068, + "learning_rate": 0.0001, + "loss": 72.289, + "step": 2580 + }, + { + "epoch": 4.170907441930771, + "grad_norm": 1.1358791589736938, + "learning_rate": 0.0001, + "loss": 72.1949, + "step": 2590 + }, + { + "epoch": 4.187030785509145, + "grad_norm": 0.9953081607818604, + "learning_rate": 0.0001, + "loss": 72.6004, + "step": 2600 + }, + { + "epoch": 4.20315412908752, + "grad_norm": 1.019061803817749, + "learning_rate": 0.0001, + "loss": 72.3772, + "step": 2610 + }, + { + "epoch": 4.219277472665894, + "grad_norm": 0.9062562584877014, + "learning_rate": 0.0001, + "loss": 72.0566, + "step": 2620 + }, + { + "epoch": 4.235400816244269, + "grad_norm": 1.0877611637115479, + "learning_rate": 0.0001, + "loss": 72.854, + "step": 2630 + }, + { + "epoch": 4.251524159822643, + "grad_norm": 1.2419648170471191, + "learning_rate": 0.0001, + "loss": 72.6498, + "step": 2640 + }, + { + "epoch": 4.267647503401018, + "grad_norm": 0.9507225751876831, + "learning_rate": 0.0001, + "loss": 68.9583, + "step": 2650 + }, + { + "epoch": 4.283770846979392, + "grad_norm": 1.121191382408142, + "learning_rate": 0.0001, + "loss": 72.8464, + "step": 2660 + }, + { + "epoch": 4.299894190557767, + "grad_norm": 1.0543361902236938, + "learning_rate": 0.0001, + "loss": 71.9882, + "step": 2670 + }, + { + "epoch": 4.316017534136141, + "grad_norm": 0.8622955083847046, + "learning_rate": 0.0001, + "loss": 71.7373, + "step": 2680 + }, + { + "epoch": 4.332140877714516, + "grad_norm": 0.911253809928894, + "learning_rate": 0.0001, + "loss": 71.1968, + "step": 2690 + }, + { + "epoch": 4.34826422129289, + "grad_norm": 4.405472755432129, + "learning_rate": 0.0001, + "loss": 70.4128, + "step": 2700 + }, + { + "epoch": 4.364387564871265, + "grad_norm": 0.9142985939979553, + "learning_rate": 0.0001, + "loss": 69.7548, + "step": 2710 + }, + { + "epoch": 4.38051090844964, + "grad_norm": 1.1022522449493408, + "learning_rate": 0.0001, + "loss": 71.8282, + "step": 2720 + }, + { + "epoch": 4.396634252028014, + "grad_norm": 1.018386721611023, + "learning_rate": 0.0001, + "loss": 71.0853, + "step": 2730 + }, + { + "epoch": 4.412757595606389, + "grad_norm": 1.0483211278915405, + "learning_rate": 0.0001, + "loss": 72.1478, + "step": 2740 + }, + { + "epoch": 4.428880939184763, + "grad_norm": 1.026861548423767, + "learning_rate": 0.0001, + "loss": 71.8947, + "step": 2750 + }, + { + "epoch": 4.445004282763138, + "grad_norm": 1.1858552694320679, + "learning_rate": 0.0001, + "loss": 71.7016, + "step": 2760 + }, + { + "epoch": 4.461127626341512, + "grad_norm": 1.1212221384048462, + "learning_rate": 0.0001, + "loss": 71.4454, + "step": 2770 + }, + { + "epoch": 4.477250969919887, + "grad_norm": 0.8062102794647217, + "learning_rate": 0.0001, + "loss": 73.2672, + "step": 2780 + }, + { + "epoch": 4.493374313498261, + "grad_norm": 1.104483962059021, + "learning_rate": 0.0001, + "loss": 70.548, + "step": 2790 + }, + { + "epoch": 4.509497657076636, + "grad_norm": 0.942057728767395, + "learning_rate": 0.0001, + "loss": 72.3584, + "step": 2800 + }, + { + "epoch": 4.525621000655011, + "grad_norm": 0.9028333425521851, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 2810 + }, + { + "epoch": 4.541744344233385, + "grad_norm": 0.9096795916557312, + "learning_rate": 0.0001, + "loss": 70.6596, + "step": 2820 + }, + { + "epoch": 4.55786768781176, + "grad_norm": 0.9038829207420349, + "learning_rate": 0.0001, + "loss": 72.6148, + "step": 2830 + }, + { + "epoch": 4.573991031390134, + "grad_norm": 1.8511613607406616, + "learning_rate": 0.0001, + "loss": 71.7205, + "step": 2840 + }, + { + "epoch": 4.590114374968509, + "grad_norm": 1.3492343425750732, + "learning_rate": 0.0001, + "loss": 72.1492, + "step": 2850 + }, + { + "epoch": 4.6062377185468835, + "grad_norm": 1.1469528675079346, + "learning_rate": 0.0001, + "loss": 70.5892, + "step": 2860 + }, + { + "epoch": 4.622361062125258, + "grad_norm": 1.1374379396438599, + "learning_rate": 0.0001, + "loss": 70.061, + "step": 2870 + }, + { + "epoch": 4.6384844057036325, + "grad_norm": 0.9597565531730652, + "learning_rate": 0.0001, + "loss": 71.0869, + "step": 2880 + }, + { + "epoch": 4.6546077492820075, + "grad_norm": 0.8356521129608154, + "learning_rate": 0.0001, + "loss": 72.1891, + "step": 2890 + }, + { + "epoch": 4.670731092860382, + "grad_norm": 0.8882647752761841, + "learning_rate": 0.0001, + "loss": 71.3845, + "step": 2900 + }, + { + "epoch": 4.6868544364387565, + "grad_norm": 1.1904003620147705, + "learning_rate": 0.0001, + "loss": 72.3086, + "step": 2910 + }, + { + "epoch": 4.702977780017131, + "grad_norm": 0.8010085821151733, + "learning_rate": 0.0001, + "loss": 71.3051, + "step": 2920 + }, + { + "epoch": 4.719101123595506, + "grad_norm": 0.7572135329246521, + "learning_rate": 0.0001, + "loss": 72.5605, + "step": 2930 + }, + { + "epoch": 4.735224467173881, + "grad_norm": 1.0866942405700684, + "learning_rate": 0.0001, + "loss": 70.0935, + "step": 2940 + }, + { + "epoch": 4.751347810752255, + "grad_norm": 0.9358721971511841, + "learning_rate": 0.0001, + "loss": 71.0391, + "step": 2950 + }, + { + "epoch": 4.76747115433063, + "grad_norm": 1.0276161432266235, + "learning_rate": 0.0001, + "loss": 69.8462, + "step": 2960 + }, + { + "epoch": 4.783594497909004, + "grad_norm": 0.9573063850402832, + "learning_rate": 0.0001, + "loss": 69.7089, + "step": 2970 + }, + { + "epoch": 4.799717841487379, + "grad_norm": 1.0638892650604248, + "learning_rate": 0.0001, + "loss": 71.8376, + "step": 2980 + }, + { + "epoch": 4.815841185065753, + "grad_norm": 0.9894681572914124, + "learning_rate": 0.0001, + "loss": 70.1764, + "step": 2990 + }, + { + "epoch": 4.831964528644128, + "grad_norm": 1.1265769004821777, + "learning_rate": 0.0001, + "loss": 71.7918, + "step": 3000 + }, + { + "epoch": 4.848087872222502, + "grad_norm": 2.2792513370513916, + "learning_rate": 0.0001, + "loss": 70.4341, + "step": 3010 + }, + { + "epoch": 4.864211215800877, + "grad_norm": 0.8469329476356506, + "learning_rate": 0.0001, + "loss": 72.1137, + "step": 3020 + }, + { + "epoch": 4.880334559379252, + "grad_norm": 0.9333013296127319, + "learning_rate": 0.0001, + "loss": 71.0789, + "step": 3030 + }, + { + "epoch": 4.896457902957626, + "grad_norm": 0.7888994216918945, + "learning_rate": 0.0001, + "loss": 70.8197, + "step": 3040 + }, + { + "epoch": 4.912581246536, + "grad_norm": 0.9194874167442322, + "learning_rate": 0.0001, + "loss": 72.071, + "step": 3050 + }, + { + "epoch": 4.928704590114375, + "grad_norm": 1.1252421140670776, + "learning_rate": 0.0001, + "loss": 71.6943, + "step": 3060 + }, + { + "epoch": 4.94482793369275, + "grad_norm": 1.1801071166992188, + "learning_rate": 0.0001, + "loss": 70.8937, + "step": 3070 + }, + { + "epoch": 4.960951277271124, + "grad_norm": 0.9162415266036987, + "learning_rate": 0.0001, + "loss": 70.024, + "step": 3080 + }, + { + "epoch": 4.977074620849499, + "grad_norm": 0.8833363652229309, + "learning_rate": 0.0001, + "loss": 71.3835, + "step": 3090 + }, + { + "epoch": 4.993197964427873, + "grad_norm": 0.9985883831977844, + "learning_rate": 0.0001, + "loss": 71.635, + "step": 3100 + }, + { + "epoch": 5.0080616717891875, + "grad_norm": 1.097298264503479, + "learning_rate": 0.0001, + "loss": 65.3542, + "step": 3110 + }, + { + "epoch": 5.024185015367562, + "grad_norm": 0.9703903198242188, + "learning_rate": 0.0001, + "loss": 69.8587, + "step": 3120 + }, + { + "epoch": 5.0403083589459365, + "grad_norm": 0.8894070386886597, + "learning_rate": 0.0001, + "loss": 71.2868, + "step": 3130 + }, + { + "epoch": 5.056431702524311, + "grad_norm": 0.6984127163887024, + "learning_rate": 0.0001, + "loss": 69.0425, + "step": 3140 + }, + { + "epoch": 5.072555046102686, + "grad_norm": 0.8607762455940247, + "learning_rate": 0.0001, + "loss": 69.5354, + "step": 3150 + }, + { + "epoch": 5.08867838968106, + "grad_norm": 1.1215914487838745, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 3160 + }, + { + "epoch": 5.104801733259435, + "grad_norm": 0.9114409685134888, + "learning_rate": 0.0001, + "loss": 68.4186, + "step": 3170 + }, + { + "epoch": 5.12092507683781, + "grad_norm": 1.010400652885437, + "learning_rate": 0.0001, + "loss": 68.3965, + "step": 3180 + }, + { + "epoch": 5.137048420416184, + "grad_norm": 0.8741958141326904, + "learning_rate": 0.0001, + "loss": 70.6728, + "step": 3190 + }, + { + "epoch": 5.153171763994559, + "grad_norm": 0.8760207891464233, + "learning_rate": 0.0001, + "loss": 72.0704, + "step": 3200 + }, + { + "epoch": 5.169295107572933, + "grad_norm": 1.328247308731079, + "learning_rate": 0.0001, + "loss": 69.9082, + "step": 3210 + }, + { + "epoch": 5.185418451151308, + "grad_norm": 0.8263765573501587, + "learning_rate": 0.0001, + "loss": 70.3539, + "step": 3220 + }, + { + "epoch": 5.201541794729682, + "grad_norm": 0.9358834028244019, + "learning_rate": 0.0001, + "loss": 70.4221, + "step": 3230 + }, + { + "epoch": 5.217665138308057, + "grad_norm": 1.0769531726837158, + "learning_rate": 0.0001, + "loss": 68.8167, + "step": 3240 + }, + { + "epoch": 5.233788481886431, + "grad_norm": 1.1438790559768677, + "learning_rate": 0.0001, + "loss": 68.9288, + "step": 3250 + }, + { + "epoch": 5.249911825464806, + "grad_norm": 0.9534218311309814, + "learning_rate": 0.0001, + "loss": 70.3277, + "step": 3260 + }, + { + "epoch": 5.26603516904318, + "grad_norm": 0.88532954454422, + "learning_rate": 0.0001, + "loss": 67.5781, + "step": 3270 + }, + { + "epoch": 5.282158512621555, + "grad_norm": 0.7986133694648743, + "learning_rate": 0.0001, + "loss": 69.759, + "step": 3280 + }, + { + "epoch": 5.29828185619993, + "grad_norm": 0.9100566506385803, + "learning_rate": 0.0001, + "loss": 68.6368, + "step": 3290 + }, + { + "epoch": 5.314405199778304, + "grad_norm": 0.8861745595932007, + "learning_rate": 0.0001, + "loss": 70.5756, + "step": 3300 + }, + { + "epoch": 5.330528543356679, + "grad_norm": 1.0729680061340332, + "learning_rate": 0.0001, + "loss": 70.3178, + "step": 3310 + }, + { + "epoch": 5.346651886935053, + "grad_norm": 1.0288280248641968, + "learning_rate": 0.0001, + "loss": 69.422, + "step": 3320 + }, + { + "epoch": 5.362775230513428, + "grad_norm": 0.8335140347480774, + "learning_rate": 0.0001, + "loss": 69.8281, + "step": 3330 + }, + { + "epoch": 5.378898574091802, + "grad_norm": 0.9789717197418213, + "learning_rate": 0.0001, + "loss": 71.1532, + "step": 3340 + }, + { + "epoch": 5.395021917670177, + "grad_norm": 0.9884677529335022, + "learning_rate": 0.0001, + "loss": 69.5673, + "step": 3350 + }, + { + "epoch": 5.411145261248551, + "grad_norm": 1.050154447555542, + "learning_rate": 0.0001, + "loss": 70.8015, + "step": 3360 + }, + { + "epoch": 5.427268604826926, + "grad_norm": 0.9645342230796814, + "learning_rate": 0.0001, + "loss": 69.1615, + "step": 3370 + }, + { + "epoch": 5.4433919484053, + "grad_norm": 0.8383159041404724, + "learning_rate": 0.0001, + "loss": 70.6299, + "step": 3380 + }, + { + "epoch": 5.459515291983675, + "grad_norm": 0.9250556826591492, + "learning_rate": 0.0001, + "loss": 69.231, + "step": 3390 + }, + { + "epoch": 5.475638635562049, + "grad_norm": 0.8108687400817871, + "learning_rate": 0.0001, + "loss": 70.5858, + "step": 3400 + }, + { + "epoch": 5.491761979140424, + "grad_norm": 0.8748818039894104, + "learning_rate": 0.0001, + "loss": 69.9001, + "step": 3410 + }, + { + "epoch": 5.507885322718799, + "grad_norm": 0.7411381006240845, + "learning_rate": 0.0001, + "loss": 68.3248, + "step": 3420 + }, + { + "epoch": 5.524008666297173, + "grad_norm": 1.0753326416015625, + "learning_rate": 0.0001, + "loss": 69.5831, + "step": 3430 + }, + { + "epoch": 5.540132009875548, + "grad_norm": 0.875254213809967, + "learning_rate": 0.0001, + "loss": 69.7516, + "step": 3440 + }, + { + "epoch": 5.556255353453922, + "grad_norm": 0.7637733817100525, + "learning_rate": 0.0001, + "loss": 68.0078, + "step": 3450 + }, + { + "epoch": 5.572378697032297, + "grad_norm": 0.968203604221344, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 3460 + }, + { + "epoch": 5.588502040610671, + "grad_norm": 0.8903141617774963, + "learning_rate": 0.0001, + "loss": 68.5835, + "step": 3470 + }, + { + "epoch": 5.604625384189046, + "grad_norm": 0.9536195397377014, + "learning_rate": 0.0001, + "loss": 71.0259, + "step": 3480 + }, + { + "epoch": 5.62074872776742, + "grad_norm": 0.8951714038848877, + "learning_rate": 0.0001, + "loss": 69.4422, + "step": 3490 + }, + { + "epoch": 5.636872071345795, + "grad_norm": 0.9639522433280945, + "learning_rate": 0.0001, + "loss": 66.5173, + "step": 3500 + }, + { + "epoch": 5.65299541492417, + "grad_norm": 1.0022730827331543, + "learning_rate": 0.0001, + "loss": 68.3443, + "step": 3510 + }, + { + "epoch": 5.669118758502544, + "grad_norm": 1.0654487609863281, + "learning_rate": 0.0001, + "loss": 69.0835, + "step": 3520 + }, + { + "epoch": 5.685242102080919, + "grad_norm": 0.9934629797935486, + "learning_rate": 0.0001, + "loss": 69.2278, + "step": 3530 + }, + { + "epoch": 5.7013654456592935, + "grad_norm": 0.8773566484451294, + "learning_rate": 0.0001, + "loss": 68.9339, + "step": 3540 + }, + { + "epoch": 5.7174887892376685, + "grad_norm": 0.7756998538970947, + "learning_rate": 0.0001, + "loss": 69.2683, + "step": 3550 + }, + { + "epoch": 5.7336121328160425, + "grad_norm": 0.8819359540939331, + "learning_rate": 0.0001, + "loss": 68.5071, + "step": 3560 + }, + { + "epoch": 5.7497354763944175, + "grad_norm": 1.1069825887680054, + "learning_rate": 0.0001, + "loss": 68.5236, + "step": 3570 + }, + { + "epoch": 5.765858819972792, + "grad_norm": 0.8006312847137451, + "learning_rate": 0.0001, + "loss": 68.7192, + "step": 3580 + }, + { + "epoch": 5.781982163551167, + "grad_norm": 1.0278719663619995, + "learning_rate": 0.0001, + "loss": 69.2545, + "step": 3590 + }, + { + "epoch": 5.798105507129541, + "grad_norm": 0.9205041527748108, + "learning_rate": 0.0001, + "loss": 69.7698, + "step": 3600 + }, + { + "epoch": 5.814228850707916, + "grad_norm": 1.0769038200378418, + "learning_rate": 0.0001, + "loss": 67.6568, + "step": 3610 + }, + { + "epoch": 5.83035219428629, + "grad_norm": 1.0565663576126099, + "learning_rate": 0.0001, + "loss": 68.9166, + "step": 3620 + }, + { + "epoch": 5.846475537864665, + "grad_norm": 0.9008663296699524, + "learning_rate": 0.0001, + "loss": 69.5837, + "step": 3630 + }, + { + "epoch": 5.86259888144304, + "grad_norm": 0.8369411826133728, + "learning_rate": 0.0001, + "loss": 68.5661, + "step": 3640 + }, + { + "epoch": 5.878722225021414, + "grad_norm": 1.0472379922866821, + "learning_rate": 0.0001, + "loss": 68.3122, + "step": 3650 + }, + { + "epoch": 5.894845568599789, + "grad_norm": 0.965277373790741, + "learning_rate": 0.0001, + "loss": 69.8062, + "step": 3660 + }, + { + "epoch": 5.910968912178163, + "grad_norm": 0.8936495780944824, + "learning_rate": 0.0001, + "loss": 67.1016, + "step": 3670 + }, + { + "epoch": 5.927092255756538, + "grad_norm": 0.8559162020683289, + "learning_rate": 0.0001, + "loss": 69.9986, + "step": 3680 + }, + { + "epoch": 5.943215599334912, + "grad_norm": 1.4815773963928223, + "learning_rate": 0.0001, + "loss": 66.7648, + "step": 3690 + }, + { + "epoch": 5.959338942913287, + "grad_norm": 0.9034469723701477, + "learning_rate": 0.0001, + "loss": 68.0767, + "step": 3700 + }, + { + "epoch": 5.975462286491661, + "grad_norm": 1.838799238204956, + "learning_rate": 0.0001, + "loss": 66.5848, + "step": 3710 + }, + { + "epoch": 5.991585630070036, + "grad_norm": 0.9274787902832031, + "learning_rate": 0.0001, + "loss": 70.0396, + "step": 3720 + }, + { + "epoch": 6.0064493374313495, + "grad_norm": 1.016974687576294, + "learning_rate": 0.0001, + "loss": 62.2268, + "step": 3730 + }, + { + "epoch": 6.022572681009724, + "grad_norm": 1.0522642135620117, + "learning_rate": 0.0001, + "loss": 69.3381, + "step": 3740 + }, + { + "epoch": 6.0386960245880985, + "grad_norm": 1.1157186031341553, + "learning_rate": 0.0001, + "loss": 68.8969, + "step": 3750 + }, + { + "epoch": 6.0548193681664735, + "grad_norm": 0.9913288950920105, + "learning_rate": 0.0001, + "loss": 67.3634, + "step": 3760 + }, + { + "epoch": 6.0709427117448485, + "grad_norm": 0.8823490142822266, + "learning_rate": 0.0001, + "loss": 70.978, + "step": 3770 + }, + { + "epoch": 6.0870660553232225, + "grad_norm": 0.853837788105011, + "learning_rate": 0.0001, + "loss": 68.9924, + "step": 3780 + }, + { + "epoch": 6.1031893989015975, + "grad_norm": 0.7937766909599304, + "learning_rate": 0.0001, + "loss": 68.1994, + "step": 3790 + }, + { + "epoch": 6.119312742479972, + "grad_norm": 1.7638057470321655, + "learning_rate": 0.0001, + "loss": 68.5078, + "step": 3800 + }, + { + "epoch": 6.135436086058347, + "grad_norm": 1.0542887449264526, + "learning_rate": 0.0001, + "loss": 68.9362, + "step": 3810 + }, + { + "epoch": 6.151559429636721, + "grad_norm": 1.06019926071167, + "learning_rate": 0.0001, + "loss": 67.2493, + "step": 3820 + }, + { + "epoch": 6.167682773215096, + "grad_norm": 0.9765111804008484, + "learning_rate": 0.0001, + "loss": 70.4797, + "step": 3830 + }, + { + "epoch": 6.18380611679347, + "grad_norm": 1.006372332572937, + "learning_rate": 0.0001, + "loss": 69.4233, + "step": 3840 + }, + { + "epoch": 6.199929460371845, + "grad_norm": 0.8274241089820862, + "learning_rate": 0.0001, + "loss": 67.9907, + "step": 3850 + }, + { + "epoch": 6.216052803950219, + "grad_norm": 0.9905765056610107, + "learning_rate": 0.0001, + "loss": 69.6988, + "step": 3860 + }, + { + "epoch": 6.232176147528594, + "grad_norm": 1.0065921545028687, + "learning_rate": 0.0001, + "loss": 68.7678, + "step": 3870 + }, + { + "epoch": 6.248299491106969, + "grad_norm": 0.8845255970954895, + "learning_rate": 0.0001, + "loss": 68.3039, + "step": 3880 + }, + { + "epoch": 6.264422834685343, + "grad_norm": 0.9030968546867371, + "learning_rate": 0.0001, + "loss": 68.671, + "step": 3890 + }, + { + "epoch": 6.280546178263718, + "grad_norm": 0.8650258183479309, + "learning_rate": 0.0001, + "loss": 68.5309, + "step": 3900 + }, + { + "epoch": 6.296669521842092, + "grad_norm": 0.8961272835731506, + "learning_rate": 0.0001, + "loss": 67.5611, + "step": 3910 + }, + { + "epoch": 6.312792865420467, + "grad_norm": 1.1702407598495483, + "learning_rate": 0.0001, + "loss": 68.2768, + "step": 3920 + }, + { + "epoch": 6.328916208998841, + "grad_norm": 0.8598962426185608, + "learning_rate": 0.0001, + "loss": 69.1961, + "step": 3930 + }, + { + "epoch": 6.345039552577216, + "grad_norm": 0.8714122176170349, + "learning_rate": 0.0001, + "loss": 66.8628, + "step": 3940 + }, + { + "epoch": 6.36116289615559, + "grad_norm": 0.7757895588874817, + "learning_rate": 0.0001, + "loss": 68.6908, + "step": 3950 + }, + { + "epoch": 6.377286239733965, + "grad_norm": 0.9189490675926208, + "learning_rate": 0.0001, + "loss": 67.8685, + "step": 3960 + }, + { + "epoch": 6.393409583312339, + "grad_norm": 0.8161299228668213, + "learning_rate": 0.0001, + "loss": 66.816, + "step": 3970 + }, + { + "epoch": 6.409532926890714, + "grad_norm": 0.8559791445732117, + "learning_rate": 0.0001, + "loss": 68.4302, + "step": 3980 + }, + { + "epoch": 6.425656270469089, + "grad_norm": 0.8087984919548035, + "learning_rate": 0.0001, + "loss": 66.2646, + "step": 3990 + }, + { + "epoch": 6.441779614047463, + "grad_norm": 0.7733110785484314, + "learning_rate": 0.0001, + "loss": 69.2769, + "step": 4000 + }, + { + "epoch": 6.457902957625838, + "grad_norm": 0.9335984587669373, + "learning_rate": 0.0001, + "loss": 67.1199, + "step": 4010 + }, + { + "epoch": 6.474026301204212, + "grad_norm": 0.9546418786048889, + "learning_rate": 0.0001, + "loss": 68.8706, + "step": 4020 + }, + { + "epoch": 6.490149644782587, + "grad_norm": 0.9670593738555908, + "learning_rate": 0.0001, + "loss": 68.2329, + "step": 4030 + }, + { + "epoch": 6.506272988360961, + "grad_norm": 0.8428588509559631, + "learning_rate": 0.0001, + "loss": 68.0105, + "step": 4040 + }, + { + "epoch": 6.522396331939336, + "grad_norm": 0.9755622744560242, + "learning_rate": 0.0001, + "loss": 68.656, + "step": 4050 + }, + { + "epoch": 6.53851967551771, + "grad_norm": 0.8972346186637878, + "learning_rate": 0.0001, + "loss": 69.3665, + "step": 4060 + }, + { + "epoch": 6.554643019096085, + "grad_norm": 0.9716668128967285, + "learning_rate": 0.0001, + "loss": 68.1053, + "step": 4070 + }, + { + "epoch": 6.570766362674459, + "grad_norm": 0.8147910833358765, + "learning_rate": 0.0001, + "loss": 67.9778, + "step": 4080 + }, + { + "epoch": 6.586889706252834, + "grad_norm": 0.8304234147071838, + "learning_rate": 0.0001, + "loss": 67.7069, + "step": 4090 + }, + { + "epoch": 6.603013049831208, + "grad_norm": 0.9356856346130371, + "learning_rate": 0.0001, + "loss": 68.1942, + "step": 4100 + }, + { + "epoch": 6.619136393409583, + "grad_norm": 1.017134189605713, + "learning_rate": 0.0001, + "loss": 67.4211, + "step": 4110 + }, + { + "epoch": 6.635259736987958, + "grad_norm": 1.142558217048645, + "learning_rate": 0.0001, + "loss": 68.4413, + "step": 4120 + }, + { + "epoch": 6.651383080566332, + "grad_norm": 0.9071153998374939, + "learning_rate": 0.0001, + "loss": 69.3666, + "step": 4130 + }, + { + "epoch": 6.667506424144707, + "grad_norm": 0.9620397686958313, + "learning_rate": 0.0001, + "loss": 68.208, + "step": 4140 + }, + { + "epoch": 6.683629767723081, + "grad_norm": 0.8282128572463989, + "learning_rate": 0.0001, + "loss": 67.1702, + "step": 4150 + }, + { + "epoch": 6.699753111301456, + "grad_norm": 0.8838199973106384, + "learning_rate": 0.0001, + "loss": 69.0074, + "step": 4160 + }, + { + "epoch": 6.71587645487983, + "grad_norm": 0.8864356279373169, + "learning_rate": 0.0001, + "loss": 67.8052, + "step": 4170 + }, + { + "epoch": 6.731999798458205, + "grad_norm": 0.9373385906219482, + "learning_rate": 0.0001, + "loss": 67.446, + "step": 4180 + }, + { + "epoch": 6.7481231420365795, + "grad_norm": 0.8308956027030945, + "learning_rate": 0.0001, + "loss": 68.716, + "step": 4190 + }, + { + "epoch": 6.7642464856149545, + "grad_norm": 0.8031372427940369, + "learning_rate": 0.0001, + "loss": 68.0482, + "step": 4200 + }, + { + "epoch": 6.780369829193329, + "grad_norm": 0.9258586764335632, + "learning_rate": 0.0001, + "loss": 68.5607, + "step": 4210 + }, + { + "epoch": 6.7964931727717035, + "grad_norm": 0.8147327899932861, + "learning_rate": 0.0001, + "loss": 67.7265, + "step": 4220 + }, + { + "epoch": 6.8126165163500785, + "grad_norm": 0.8717535734176636, + "learning_rate": 0.0001, + "loss": 67.8097, + "step": 4230 + }, + { + "epoch": 6.828739859928453, + "grad_norm": 0.9876209497451782, + "learning_rate": 0.0001, + "loss": 67.9068, + "step": 4240 + }, + { + "epoch": 6.8448632035068275, + "grad_norm": 0.718376874923706, + "learning_rate": 0.0001, + "loss": 68.1762, + "step": 4250 + }, + { + "epoch": 6.860986547085202, + "grad_norm": 0.8264320492744446, + "learning_rate": 0.0001, + "loss": 67.8391, + "step": 4260 + }, + { + "epoch": 6.877109890663577, + "grad_norm": 0.7478179335594177, + "learning_rate": 0.0001, + "loss": 67.6635, + "step": 4270 + }, + { + "epoch": 6.893233234241951, + "grad_norm": 0.8394957184791565, + "learning_rate": 0.0001, + "loss": 68.6073, + "step": 4280 + }, + { + "epoch": 6.909356577820326, + "grad_norm": 0.8534071445465088, + "learning_rate": 0.0001, + "loss": 69.6206, + "step": 4290 + }, + { + "epoch": 6.9254799213987, + "grad_norm": 0.9137582778930664, + "learning_rate": 0.0001, + "loss": 67.4362, + "step": 4300 + }, + { + "epoch": 6.941603264977075, + "grad_norm": 0.8869971036911011, + "learning_rate": 0.0001, + "loss": 67.6811, + "step": 4310 + }, + { + "epoch": 6.957726608555449, + "grad_norm": 0.889025866985321, + "learning_rate": 0.0001, + "loss": 65.3354, + "step": 4320 + }, + { + "epoch": 6.973849952133824, + "grad_norm": 0.7968549728393555, + "learning_rate": 0.0001, + "loss": 66.192, + "step": 4330 + }, + { + "epoch": 6.989973295712199, + "grad_norm": 1.0077433586120605, + "learning_rate": 0.0001, + "loss": 67.5216, + "step": 4340 + }, + { + "epoch": 7.004837003073512, + "grad_norm": 0.7900649309158325, + "learning_rate": 0.0001, + "loss": 62.2421, + "step": 4350 + }, + { + "epoch": 7.020960346651887, + "grad_norm": 1.0160410404205322, + "learning_rate": 0.0001, + "loss": 68.0978, + "step": 4360 + }, + { + "epoch": 7.037083690230261, + "grad_norm": 0.841387927532196, + "learning_rate": 0.0001, + "loss": 68.1008, + "step": 4370 + }, + { + "epoch": 7.053207033808636, + "grad_norm": 0.87884122133255, + "learning_rate": 0.0001, + "loss": 66.4667, + "step": 4380 + }, + { + "epoch": 7.06933037738701, + "grad_norm": 0.8580564856529236, + "learning_rate": 0.0001, + "loss": 67.2002, + "step": 4390 + }, + { + "epoch": 7.085453720965385, + "grad_norm": 0.7560063600540161, + "learning_rate": 0.0001, + "loss": 67.6743, + "step": 4400 + }, + { + "epoch": 7.1015770645437595, + "grad_norm": 0.855478048324585, + "learning_rate": 0.0001, + "loss": 67.822, + "step": 4410 + }, + { + "epoch": 7.1177004081221344, + "grad_norm": 0.7994911670684814, + "learning_rate": 0.0001, + "loss": 66.0094, + "step": 4420 + }, + { + "epoch": 7.1338237517005085, + "grad_norm": 0.8728616833686829, + "learning_rate": 0.0001, + "loss": 66.405, + "step": 4430 + }, + { + "epoch": 7.1499470952788835, + "grad_norm": 0.8434081673622131, + "learning_rate": 0.0001, + "loss": 67.0535, + "step": 4440 + }, + { + "epoch": 7.166070438857258, + "grad_norm": 0.8543094396591187, + "learning_rate": 0.0001, + "loss": 66.2417, + "step": 4450 + }, + { + "epoch": 7.182193782435633, + "grad_norm": 0.7209319472312927, + "learning_rate": 0.0001, + "loss": 67.3958, + "step": 4460 + }, + { + "epoch": 7.1983171260140075, + "grad_norm": 0.6735154390335083, + "learning_rate": 0.0001, + "loss": 67.4086, + "step": 4470 + }, + { + "epoch": 7.214440469592382, + "grad_norm": 0.9126104116439819, + "learning_rate": 0.0001, + "loss": 68.4191, + "step": 4480 + }, + { + "epoch": 7.230563813170757, + "grad_norm": 0.9334300756454468, + "learning_rate": 0.0001, + "loss": 68.3247, + "step": 4490 + }, + { + "epoch": 7.246687156749131, + "grad_norm": 1.5956038236618042, + "learning_rate": 0.0001, + "loss": 68.2779, + "step": 4500 + }, + { + "epoch": 7.262810500327506, + "grad_norm": 1.5012775659561157, + "learning_rate": 0.0001, + "loss": 68.3533, + "step": 4510 + }, + { + "epoch": 7.27893384390588, + "grad_norm": 1.1376632452011108, + "learning_rate": 0.0001, + "loss": 65.3627, + "step": 4520 + }, + { + "epoch": 7.295057187484255, + "grad_norm": 0.8597291111946106, + "learning_rate": 0.0001, + "loss": 67.8653, + "step": 4530 + }, + { + "epoch": 7.311180531062629, + "grad_norm": 0.7645048499107361, + "learning_rate": 0.0001, + "loss": 65.4592, + "step": 4540 + }, + { + "epoch": 7.327303874641004, + "grad_norm": 0.7717318534851074, + "learning_rate": 0.0001, + "loss": 66.902, + "step": 4550 + }, + { + "epoch": 7.343427218219378, + "grad_norm": 0.8884950876235962, + "learning_rate": 0.0001, + "loss": 67.1858, + "step": 4560 + }, + { + "epoch": 7.359550561797753, + "grad_norm": 0.7826898694038391, + "learning_rate": 0.0001, + "loss": 68.4673, + "step": 4570 + }, + { + "epoch": 7.375673905376128, + "grad_norm": 0.7484182119369507, + "learning_rate": 0.0001, + "loss": 67.4859, + "step": 4580 + }, + { + "epoch": 7.391797248954502, + "grad_norm": 0.918194591999054, + "learning_rate": 0.0001, + "loss": 67.7393, + "step": 4590 + }, + { + "epoch": 7.407920592532877, + "grad_norm": 1.0696896314620972, + "learning_rate": 0.0001, + "loss": 67.0101, + "step": 4600 + }, + { + "epoch": 7.424043936111251, + "grad_norm": 0.8190692067146301, + "learning_rate": 0.0001, + "loss": 68.5358, + "step": 4610 + }, + { + "epoch": 7.440167279689626, + "grad_norm": 0.9063783884048462, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 4620 + }, + { + "epoch": 7.456290623268, + "grad_norm": 0.7588100433349609, + "learning_rate": 0.0001, + "loss": 68.0066, + "step": 4630 + }, + { + "epoch": 7.472413966846375, + "grad_norm": 0.897281289100647, + "learning_rate": 0.0001, + "loss": 67.2499, + "step": 4640 + }, + { + "epoch": 7.488537310424749, + "grad_norm": 0.9505569338798523, + "learning_rate": 0.0001, + "loss": 67.5452, + "step": 4650 + }, + { + "epoch": 7.504660654003124, + "grad_norm": 0.8409159183502197, + "learning_rate": 0.0001, + "loss": 66.6293, + "step": 4660 + }, + { + "epoch": 7.520783997581498, + "grad_norm": 0.9878574013710022, + "learning_rate": 0.0001, + "loss": 65.3379, + "step": 4670 + }, + { + "epoch": 7.536907341159873, + "grad_norm": 1.0410685539245605, + "learning_rate": 0.0001, + "loss": 65.8028, + "step": 4680 + }, + { + "epoch": 7.553030684738248, + "grad_norm": 0.8417725563049316, + "learning_rate": 0.0001, + "loss": 65.4283, + "step": 4690 + }, + { + "epoch": 7.569154028316622, + "grad_norm": 0.7660863399505615, + "learning_rate": 0.0001, + "loss": 66.6765, + "step": 4700 + }, + { + "epoch": 7.585277371894997, + "grad_norm": 0.8074233531951904, + "learning_rate": 0.0001, + "loss": 67.1082, + "step": 4710 + }, + { + "epoch": 7.601400715473371, + "grad_norm": 0.7653326988220215, + "learning_rate": 0.0001, + "loss": 66.028, + "step": 4720 + }, + { + "epoch": 7.617524059051746, + "grad_norm": 1.1764041185379028, + "learning_rate": 0.0001, + "loss": 65.2587, + "step": 4730 + }, + { + "epoch": 7.63364740263012, + "grad_norm": 1.7604950666427612, + "learning_rate": 0.0001, + "loss": 66.0543, + "step": 4740 + }, + { + "epoch": 7.649770746208495, + "grad_norm": 0.941438615322113, + "learning_rate": 0.0001, + "loss": 65.6954, + "step": 4750 + }, + { + "epoch": 7.665894089786869, + "grad_norm": 0.740069568157196, + "learning_rate": 0.0001, + "loss": 67.9701, + "step": 4760 + }, + { + "epoch": 7.682017433365244, + "grad_norm": 0.9867332577705383, + "learning_rate": 0.0001, + "loss": 66.1445, + "step": 4770 + }, + { + "epoch": 7.698140776943618, + "grad_norm": 0.8156166672706604, + "learning_rate": 0.0001, + "loss": 67.2647, + "step": 4780 + }, + { + "epoch": 7.714264120521993, + "grad_norm": 0.7689579129219055, + "learning_rate": 0.0001, + "loss": 67.0961, + "step": 4790 + }, + { + "epoch": 7.730387464100367, + "grad_norm": 0.8159700036048889, + "learning_rate": 0.0001, + "loss": 66.4974, + "step": 4800 + }, + { + "epoch": 7.746510807678742, + "grad_norm": 0.8554222583770752, + "learning_rate": 0.0001, + "loss": 64.5057, + "step": 4810 + }, + { + "epoch": 7.762634151257117, + "grad_norm": 0.856629490852356, + "learning_rate": 0.0001, + "loss": 67.7819, + "step": 4820 + }, + { + "epoch": 7.778757494835491, + "grad_norm": 1.157179832458496, + "learning_rate": 0.0001, + "loss": 66.0122, + "step": 4830 + }, + { + "epoch": 7.794880838413866, + "grad_norm": 0.9597711563110352, + "learning_rate": 0.0001, + "loss": 66.8248, + "step": 4840 + }, + { + "epoch": 7.8110041819922404, + "grad_norm": 0.9175744652748108, + "learning_rate": 0.0001, + "loss": 65.7998, + "step": 4850 + }, + { + "epoch": 7.827127525570615, + "grad_norm": 0.8467298150062561, + "learning_rate": 0.0001, + "loss": 67.5363, + "step": 4860 + }, + { + "epoch": 7.8432508691489895, + "grad_norm": 0.863922655582428, + "learning_rate": 0.0001, + "loss": 67.7219, + "step": 4870 + }, + { + "epoch": 7.8593742127273645, + "grad_norm": 0.9896290302276611, + "learning_rate": 0.0001, + "loss": 66.7098, + "step": 4880 + }, + { + "epoch": 7.8754975563057386, + "grad_norm": 0.8828520774841309, + "learning_rate": 0.0001, + "loss": 66.0192, + "step": 4890 + }, + { + "epoch": 7.8916208998841135, + "grad_norm": 0.9093656539916992, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 4900 + }, + { + "epoch": 7.9077442434624885, + "grad_norm": 0.8665091395378113, + "learning_rate": 0.0001, + "loss": 66.5851, + "step": 4910 + }, + { + "epoch": 7.923867587040863, + "grad_norm": 0.8027398586273193, + "learning_rate": 0.0001, + "loss": 64.8607, + "step": 4920 + }, + { + "epoch": 7.9399909306192376, + "grad_norm": 0.7808436751365662, + "learning_rate": 0.0001, + "loss": 65.052, + "step": 4930 + }, + { + "epoch": 7.956114274197612, + "grad_norm": 0.7246070504188538, + "learning_rate": 0.0001, + "loss": 64.789, + "step": 4940 + }, + { + "epoch": 7.972237617775987, + "grad_norm": 1.0405027866363525, + "learning_rate": 0.0001, + "loss": 66.7735, + "step": 4950 + }, + { + "epoch": 7.988360961354361, + "grad_norm": 0.812921404838562, + "learning_rate": 0.0001, + "loss": 64.1286, + "step": 4960 + }, + { + "epoch": 8.003224668715674, + "grad_norm": 0.884215772151947, + "learning_rate": 0.0001, + "loss": 61.0812, + "step": 4970 + }, + { + "epoch": 8.01934801229405, + "grad_norm": 0.8519641757011414, + "learning_rate": 0.0001, + "loss": 66.2784, + "step": 4980 + }, + { + "epoch": 8.035471355872424, + "grad_norm": 0.8331199884414673, + "learning_rate": 0.0001, + "loss": 67.1405, + "step": 4990 + }, + { + "epoch": 8.0515946994508, + "grad_norm": 0.8568746447563171, + "learning_rate": 0.0001, + "loss": 66.7835, + "step": 5000 + }, + { + "epoch": 8.067718043029172, + "grad_norm": 1.170482873916626, + "learning_rate": 0.0001, + "loss": 64.2269, + "step": 5010 + }, + { + "epoch": 8.083841386607547, + "grad_norm": 0.9171113967895508, + "learning_rate": 0.0001, + "loss": 66.5567, + "step": 5020 + }, + { + "epoch": 8.099964730185922, + "grad_norm": 0.8988714814186096, + "learning_rate": 0.0001, + "loss": 66.0471, + "step": 5030 + }, + { + "epoch": 8.116088073764297, + "grad_norm": 0.7651671171188354, + "learning_rate": 0.0001, + "loss": 65.7185, + "step": 5040 + }, + { + "epoch": 8.132211417342672, + "grad_norm": 0.6890843510627747, + "learning_rate": 0.0001, + "loss": 65.0261, + "step": 5050 + }, + { + "epoch": 8.148334760921045, + "grad_norm": 0.7683818340301514, + "learning_rate": 0.0001, + "loss": 65.895, + "step": 5060 + }, + { + "epoch": 8.16445810449942, + "grad_norm": 0.9398573637008667, + "learning_rate": 0.0001, + "loss": 64.674, + "step": 5070 + }, + { + "epoch": 8.180581448077795, + "grad_norm": 0.7708531022071838, + "learning_rate": 0.0001, + "loss": 65.0637, + "step": 5080 + }, + { + "epoch": 8.19670479165617, + "grad_norm": 0.8840665221214294, + "learning_rate": 0.0001, + "loss": 65.6449, + "step": 5090 + }, + { + "epoch": 8.212828135234544, + "grad_norm": 0.8390524983406067, + "learning_rate": 0.0001, + "loss": 65.4738, + "step": 5100 + }, + { + "epoch": 8.228951478812919, + "grad_norm": 1.7559128999710083, + "learning_rate": 0.0001, + "loss": 68.911, + "step": 5110 + }, + { + "epoch": 8.245074822391294, + "grad_norm": 0.8293817043304443, + "learning_rate": 0.0001, + "loss": 65.2601, + "step": 5120 + }, + { + "epoch": 8.261198165969668, + "grad_norm": 0.8947123289108276, + "learning_rate": 0.0001, + "loss": 65.8611, + "step": 5130 + }, + { + "epoch": 8.277321509548042, + "grad_norm": 0.8526618480682373, + "learning_rate": 0.0001, + "loss": 65.559, + "step": 5140 + }, + { + "epoch": 8.293444853126417, + "grad_norm": 1.7423408031463623, + "learning_rate": 0.0001, + "loss": 64.5007, + "step": 5150 + }, + { + "epoch": 8.309568196704792, + "grad_norm": 1.0512502193450928, + "learning_rate": 0.0001, + "loss": 66.2593, + "step": 5160 + }, + { + "epoch": 8.325691540283167, + "grad_norm": 0.8129869103431702, + "learning_rate": 0.0001, + "loss": 64.7989, + "step": 5170 + }, + { + "epoch": 8.341814883861542, + "grad_norm": 0.852199912071228, + "learning_rate": 0.0001, + "loss": 65.9627, + "step": 5180 + }, + { + "epoch": 8.357938227439915, + "grad_norm": 0.8932474851608276, + "learning_rate": 0.0001, + "loss": 64.1035, + "step": 5190 + }, + { + "epoch": 8.37406157101829, + "grad_norm": 1.2631597518920898, + "learning_rate": 0.0001, + "loss": 65.3406, + "step": 5200 + }, + { + "epoch": 8.390184914596665, + "grad_norm": 0.9062619209289551, + "learning_rate": 0.0001, + "loss": 65.1841, + "step": 5210 + }, + { + "epoch": 8.40630825817504, + "grad_norm": 0.9153940081596375, + "learning_rate": 0.0001, + "loss": 66.3852, + "step": 5220 + }, + { + "epoch": 8.422431601753413, + "grad_norm": 0.8056164979934692, + "learning_rate": 0.0001, + "loss": 66.511, + "step": 5230 + }, + { + "epoch": 8.438554945331788, + "grad_norm": 0.7506036162376404, + "learning_rate": 0.0001, + "loss": 65.275, + "step": 5240 + }, + { + "epoch": 8.454678288910163, + "grad_norm": 0.7994081377983093, + "learning_rate": 0.0001, + "loss": 64.9946, + "step": 5250 + }, + { + "epoch": 8.470801632488538, + "grad_norm": 0.9887873530387878, + "learning_rate": 0.0001, + "loss": 67.2057, + "step": 5260 + }, + { + "epoch": 8.486924976066913, + "grad_norm": 0.8495044112205505, + "learning_rate": 0.0001, + "loss": 66.2481, + "step": 5270 + }, + { + "epoch": 8.503048319645286, + "grad_norm": 0.7353820204734802, + "learning_rate": 0.0001, + "loss": 65.6163, + "step": 5280 + }, + { + "epoch": 8.519171663223661, + "grad_norm": 0.795216977596283, + "learning_rate": 0.0001, + "loss": 64.808, + "step": 5290 + }, + { + "epoch": 8.535295006802036, + "grad_norm": 0.8121851682662964, + "learning_rate": 0.0001, + "loss": 65.9755, + "step": 5300 + }, + { + "epoch": 8.55141835038041, + "grad_norm": 0.8164243698120117, + "learning_rate": 0.0001, + "loss": 66.0583, + "step": 5310 + }, + { + "epoch": 8.567541693958784, + "grad_norm": 0.9893137216567993, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 5320 + }, + { + "epoch": 8.583665037537159, + "grad_norm": 1.3616434335708618, + "learning_rate": 0.0001, + "loss": 64.1444, + "step": 5330 + }, + { + "epoch": 8.599788381115534, + "grad_norm": 0.7002740502357483, + "learning_rate": 0.0001, + "loss": 65.0906, + "step": 5340 + }, + { + "epoch": 8.615911724693909, + "grad_norm": 0.838424563407898, + "learning_rate": 0.0001, + "loss": 65.4578, + "step": 5350 + }, + { + "epoch": 8.632035068272282, + "grad_norm": 0.988902747631073, + "learning_rate": 0.0001, + "loss": 64.3405, + "step": 5360 + }, + { + "epoch": 8.648158411850657, + "grad_norm": 0.7941252589225769, + "learning_rate": 0.0001, + "loss": 67.0146, + "step": 5370 + }, + { + "epoch": 8.664281755429032, + "grad_norm": 0.7727184295654297, + "learning_rate": 0.0001, + "loss": 64.8011, + "step": 5380 + }, + { + "epoch": 8.680405099007407, + "grad_norm": 0.9514367580413818, + "learning_rate": 0.0001, + "loss": 65.8165, + "step": 5390 + }, + { + "epoch": 8.69652844258578, + "grad_norm": 0.870544970035553, + "learning_rate": 0.0001, + "loss": 65.803, + "step": 5400 + }, + { + "epoch": 8.712651786164155, + "grad_norm": 0.7348082065582275, + "learning_rate": 0.0001, + "loss": 65.8409, + "step": 5410 + }, + { + "epoch": 8.72877512974253, + "grad_norm": 0.7543814778327942, + "learning_rate": 0.0001, + "loss": 65.5958, + "step": 5420 + }, + { + "epoch": 8.744898473320905, + "grad_norm": 0.7520368099212646, + "learning_rate": 0.0001, + "loss": 65.3323, + "step": 5430 + }, + { + "epoch": 8.76102181689928, + "grad_norm": 0.7610031366348267, + "learning_rate": 0.0001, + "loss": 65.4386, + "step": 5440 + }, + { + "epoch": 8.777145160477653, + "grad_norm": 0.8854930400848389, + "learning_rate": 0.0001, + "loss": 66.181, + "step": 5450 + }, + { + "epoch": 8.793268504056028, + "grad_norm": 0.7659342885017395, + "learning_rate": 0.0001, + "loss": 65.2586, + "step": 5460 + }, + { + "epoch": 8.809391847634403, + "grad_norm": 0.8592572212219238, + "learning_rate": 0.0001, + "loss": 66.3306, + "step": 5470 + }, + { + "epoch": 8.825515191212778, + "grad_norm": 0.8169172406196594, + "learning_rate": 0.0001, + "loss": 65.8283, + "step": 5480 + }, + { + "epoch": 8.841638534791151, + "grad_norm": 0.7380614876747131, + "learning_rate": 0.0001, + "loss": 65.8711, + "step": 5490 + }, + { + "epoch": 8.857761878369526, + "grad_norm": 0.9993331432342529, + "learning_rate": 0.0001, + "loss": 64.9378, + "step": 5500 + }, + { + "epoch": 8.873885221947901, + "grad_norm": 0.6252862811088562, + "learning_rate": 0.0001, + "loss": 64.1655, + "step": 5510 + }, + { + "epoch": 8.890008565526276, + "grad_norm": 0.8441852331161499, + "learning_rate": 0.0001, + "loss": 64.9795, + "step": 5520 + }, + { + "epoch": 8.906131909104651, + "grad_norm": 0.7075499892234802, + "learning_rate": 0.0001, + "loss": 65.5843, + "step": 5530 + }, + { + "epoch": 8.922255252683025, + "grad_norm": 0.7374144196510315, + "learning_rate": 0.0001, + "loss": 67.5064, + "step": 5540 + }, + { + "epoch": 8.9383785962614, + "grad_norm": 0.8073632121086121, + "learning_rate": 0.0001, + "loss": 67.0182, + "step": 5550 + }, + { + "epoch": 8.954501939839774, + "grad_norm": 0.7576195001602173, + "learning_rate": 0.0001, + "loss": 65.9139, + "step": 5560 + }, + { + "epoch": 8.97062528341815, + "grad_norm": 1.6479626893997192, + "learning_rate": 0.0001, + "loss": 66.3517, + "step": 5570 + }, + { + "epoch": 8.986748626996523, + "grad_norm": 0.846513569355011, + "learning_rate": 0.0001, + "loss": 64.2908, + "step": 5580 + }, + { + "epoch": 9.001612334357837, + "grad_norm": 0.9551037549972534, + "learning_rate": 0.0001, + "loss": 61.7312, + "step": 5590 + }, + { + "epoch": 9.017735677936212, + "grad_norm": 0.8250304460525513, + "learning_rate": 0.0001, + "loss": 66.3351, + "step": 5600 + }, + { + "epoch": 9.033859021514587, + "grad_norm": 0.7932960391044617, + "learning_rate": 0.0001, + "loss": 66.7753, + "step": 5610 + }, + { + "epoch": 9.04998236509296, + "grad_norm": 0.9839881658554077, + "learning_rate": 0.0001, + "loss": 64.5146, + "step": 5620 + }, + { + "epoch": 9.066105708671335, + "grad_norm": 0.7337850332260132, + "learning_rate": 0.0001, + "loss": 65.0544, + "step": 5630 + }, + { + "epoch": 9.08222905224971, + "grad_norm": 0.7496634721755981, + "learning_rate": 0.0001, + "loss": 64.1451, + "step": 5640 + }, + { + "epoch": 9.098352395828085, + "grad_norm": 0.9457297921180725, + "learning_rate": 0.0001, + "loss": 65.3036, + "step": 5650 + }, + { + "epoch": 9.11447573940646, + "grad_norm": 0.8315348625183105, + "learning_rate": 0.0001, + "loss": 66.0585, + "step": 5660 + }, + { + "epoch": 9.130599082984833, + "grad_norm": 0.9666195511817932, + "learning_rate": 0.0001, + "loss": 64.6802, + "step": 5670 + }, + { + "epoch": 9.146722426563208, + "grad_norm": 0.7416576147079468, + "learning_rate": 0.0001, + "loss": 65.1557, + "step": 5680 + }, + { + "epoch": 9.162845770141583, + "grad_norm": 0.7723799347877502, + "learning_rate": 0.0001, + "loss": 66.2871, + "step": 5690 + }, + { + "epoch": 9.178969113719958, + "grad_norm": 0.7461305856704712, + "learning_rate": 0.0001, + "loss": 65.5795, + "step": 5700 + }, + { + "epoch": 9.195092457298331, + "grad_norm": 0.6663411259651184, + "learning_rate": 0.0001, + "loss": 66.4094, + "step": 5710 + }, + { + "epoch": 9.211215800876706, + "grad_norm": 0.9340134859085083, + "learning_rate": 0.0001, + "loss": 63.5753, + "step": 5720 + }, + { + "epoch": 9.227339144455081, + "grad_norm": 0.7643963694572449, + "learning_rate": 0.0001, + "loss": 65.3359, + "step": 5730 + }, + { + "epoch": 9.243462488033456, + "grad_norm": 0.6931570768356323, + "learning_rate": 0.0001, + "loss": 65.0755, + "step": 5740 + }, + { + "epoch": 9.259585831611831, + "grad_norm": 0.9096988439559937, + "learning_rate": 0.0001, + "loss": 64.7983, + "step": 5750 + }, + { + "epoch": 9.275709175190205, + "grad_norm": 0.8360128402709961, + "learning_rate": 0.0001, + "loss": 64.3604, + "step": 5760 + }, + { + "epoch": 9.29183251876858, + "grad_norm": 0.9302591681480408, + "learning_rate": 0.0001, + "loss": 66.2754, + "step": 5770 + }, + { + "epoch": 9.307955862346954, + "grad_norm": 0.7000735998153687, + "learning_rate": 0.0001, + "loss": 65.2241, + "step": 5780 + }, + { + "epoch": 9.32407920592533, + "grad_norm": 0.7321175932884216, + "learning_rate": 0.0001, + "loss": 65.3558, + "step": 5790 + }, + { + "epoch": 9.340202549503703, + "grad_norm": 0.7948715090751648, + "learning_rate": 0.0001, + "loss": 64.2793, + "step": 5800 + }, + { + "epoch": 9.356325893082078, + "grad_norm": 0.7676129341125488, + "learning_rate": 0.0001, + "loss": 66.4778, + "step": 5810 + }, + { + "epoch": 9.372449236660453, + "grad_norm": 0.977236270904541, + "learning_rate": 0.0001, + "loss": 64.5587, + "step": 5820 + }, + { + "epoch": 9.388572580238828, + "grad_norm": 0.86808842420578, + "learning_rate": 0.0001, + "loss": 65.8026, + "step": 5830 + }, + { + "epoch": 9.4046959238172, + "grad_norm": 0.7455324530601501, + "learning_rate": 0.0001, + "loss": 65.066, + "step": 5840 + }, + { + "epoch": 9.420819267395576, + "grad_norm": 0.6784212589263916, + "learning_rate": 0.0001, + "loss": 65.4167, + "step": 5850 + }, + { + "epoch": 9.43694261097395, + "grad_norm": 1.033731460571289, + "learning_rate": 0.0001, + "loss": 66.4502, + "step": 5860 + }, + { + "epoch": 9.453065954552326, + "grad_norm": 0.7946107983589172, + "learning_rate": 0.0001, + "loss": 63.7567, + "step": 5870 + }, + { + "epoch": 9.4691892981307, + "grad_norm": 0.7826364636421204, + "learning_rate": 0.0001, + "loss": 65.4484, + "step": 5880 + }, + { + "epoch": 9.485312641709074, + "grad_norm": 0.7605016827583313, + "learning_rate": 0.0001, + "loss": 66.8459, + "step": 5890 + }, + { + "epoch": 9.501435985287449, + "grad_norm": 0.7383126020431519, + "learning_rate": 0.0001, + "loss": 64.6873, + "step": 5900 + }, + { + "epoch": 9.517559328865824, + "grad_norm": 0.8355686664581299, + "learning_rate": 0.0001, + "loss": 63.5145, + "step": 5910 + }, + { + "epoch": 9.533682672444199, + "grad_norm": 0.8827091455459595, + "learning_rate": 0.0001, + "loss": 67.2207, + "step": 5920 + }, + { + "epoch": 9.549806016022572, + "grad_norm": 0.6988142728805542, + "learning_rate": 0.0001, + "loss": 64.2289, + "step": 5930 + }, + { + "epoch": 9.565929359600947, + "grad_norm": 0.7609000205993652, + "learning_rate": 0.0001, + "loss": 63.9324, + "step": 5940 + }, + { + "epoch": 9.582052703179322, + "grad_norm": 0.8053853511810303, + "learning_rate": 0.0001, + "loss": 62.5564, + "step": 5950 + }, + { + "epoch": 9.598176046757697, + "grad_norm": 0.7684671878814697, + "learning_rate": 0.0001, + "loss": 66.2669, + "step": 5960 + }, + { + "epoch": 9.614299390336072, + "grad_norm": 0.8411065340042114, + "learning_rate": 0.0001, + "loss": 64.5087, + "step": 5970 + }, + { + "epoch": 9.630422733914445, + "grad_norm": 0.963498592376709, + "learning_rate": 0.0001, + "loss": 68.1736, + "step": 5980 + }, + { + "epoch": 9.64654607749282, + "grad_norm": 0.8715981841087341, + "learning_rate": 0.0001, + "loss": 64.1686, + "step": 5990 + }, + { + "epoch": 9.662669421071195, + "grad_norm": 0.8217800855636597, + "learning_rate": 0.0001, + "loss": 64.2211, + "step": 6000 + }, + { + "epoch": 9.67879276464957, + "grad_norm": 0.7824742197990417, + "learning_rate": 0.0001, + "loss": 65.9052, + "step": 6010 + }, + { + "epoch": 9.694916108227943, + "grad_norm": 0.7440009117126465, + "learning_rate": 0.0001, + "loss": 63.3749, + "step": 6020 + }, + { + "epoch": 9.711039451806318, + "grad_norm": 0.8759479522705078, + "learning_rate": 0.0001, + "loss": 63.4488, + "step": 6030 + }, + { + "epoch": 9.727162795384693, + "grad_norm": 3.662144184112549, + "learning_rate": 0.0001, + "loss": 62.8787, + "step": 6040 + }, + { + "epoch": 9.743286138963068, + "grad_norm": 1.007149577140808, + "learning_rate": 0.0001, + "loss": 66.0053, + "step": 6050 + }, + { + "epoch": 9.759409482541441, + "grad_norm": 0.8369242548942566, + "learning_rate": 0.0001, + "loss": 67.233, + "step": 6060 + }, + { + "epoch": 9.775532826119816, + "grad_norm": 0.8160434365272522, + "learning_rate": 0.0001, + "loss": 65.2089, + "step": 6070 + }, + { + "epoch": 9.791656169698191, + "grad_norm": 0.8482319712638855, + "learning_rate": 0.0001, + "loss": 65.0718, + "step": 6080 + }, + { + "epoch": 9.807779513276566, + "grad_norm": 0.8800129890441895, + "learning_rate": 0.0001, + "loss": 65.145, + "step": 6090 + }, + { + "epoch": 9.82390285685494, + "grad_norm": 0.8911405801773071, + "learning_rate": 0.0001, + "loss": 65.7952, + "step": 6100 + }, + { + "epoch": 9.840026200433314, + "grad_norm": 0.8108358979225159, + "learning_rate": 0.0001, + "loss": 65.1928, + "step": 6110 + }, + { + "epoch": 9.85614954401169, + "grad_norm": 0.8919743895530701, + "learning_rate": 0.0001, + "loss": 65.8129, + "step": 6120 + }, + { + "epoch": 9.872272887590064, + "grad_norm": 0.73152756690979, + "learning_rate": 0.0001, + "loss": 65.7141, + "step": 6130 + }, + { + "epoch": 9.88839623116844, + "grad_norm": 0.8583903312683105, + "learning_rate": 0.0001, + "loss": 64.8231, + "step": 6140 + }, + { + "epoch": 9.904519574746812, + "grad_norm": 0.798543393611908, + "learning_rate": 0.0001, + "loss": 64.5073, + "step": 6150 + }, + { + "epoch": 9.920642918325187, + "grad_norm": 0.8657358288764954, + "learning_rate": 0.0001, + "loss": 64.1207, + "step": 6160 + }, + { + "epoch": 9.936766261903562, + "grad_norm": 0.9324677586555481, + "learning_rate": 0.0001, + "loss": 64.7129, + "step": 6170 + }, + { + "epoch": 9.952889605481937, + "grad_norm": 0.7371154427528381, + "learning_rate": 0.0001, + "loss": 64.0251, + "step": 6180 + }, + { + "epoch": 9.96901294906031, + "grad_norm": 0.8273237347602844, + "learning_rate": 0.0001, + "loss": 65.8633, + "step": 6190 + }, + { + "epoch": 9.985136292638686, + "grad_norm": 0.7219365239143372, + "learning_rate": 0.0001, + "loss": 63.4029, + "step": 6200 + }, + { + "epoch": 10.0, + "grad_norm": 0.76242995262146, + "learning_rate": 0.0001, + "loss": 60.9994, + "step": 6210 + }, + { + "epoch": 10.016123343578375, + "grad_norm": 0.8677287697792053, + "learning_rate": 0.0001, + "loss": 65.9687, + "step": 6220 + }, + { + "epoch": 10.03224668715675, + "grad_norm": 0.7073521018028259, + "learning_rate": 0.0001, + "loss": 63.0083, + "step": 6230 + }, + { + "epoch": 10.048370030735123, + "grad_norm": 0.794535756111145, + "learning_rate": 0.0001, + "loss": 63.0616, + "step": 6240 + }, + { + "epoch": 10.064493374313498, + "grad_norm": 0.6974631547927856, + "learning_rate": 0.0001, + "loss": 64.6579, + "step": 6250 + }, + { + "epoch": 10.080616717891873, + "grad_norm": 0.8279150128364563, + "learning_rate": 0.0001, + "loss": 65.1069, + "step": 6260 + }, + { + "epoch": 10.096740061470248, + "grad_norm": 0.8682932257652283, + "learning_rate": 0.0001, + "loss": 64.5019, + "step": 6270 + }, + { + "epoch": 10.112863405048621, + "grad_norm": 1.0158190727233887, + "learning_rate": 0.0001, + "loss": 65.8854, + "step": 6280 + }, + { + "epoch": 10.128986748626996, + "grad_norm": 0.6911011934280396, + "learning_rate": 0.0001, + "loss": 62.4879, + "step": 6290 + }, + { + "epoch": 10.145110092205371, + "grad_norm": 0.7394022941589355, + "learning_rate": 0.0001, + "loss": 64.2604, + "step": 6300 + }, + { + "epoch": 10.161233435783746, + "grad_norm": 0.7941621541976929, + "learning_rate": 0.0001, + "loss": 64.5133, + "step": 6310 + }, + { + "epoch": 10.17735677936212, + "grad_norm": 1.0238994359970093, + "learning_rate": 0.0001, + "loss": 63.8594, + "step": 6320 + }, + { + "epoch": 10.193480122940494, + "grad_norm": 0.7332146763801575, + "learning_rate": 0.0001, + "loss": 66.5434, + "step": 6330 + }, + { + "epoch": 10.20960346651887, + "grad_norm": 0.9375078082084656, + "learning_rate": 0.0001, + "loss": 63.4793, + "step": 6340 + }, + { + "epoch": 10.225726810097244, + "grad_norm": 0.715931236743927, + "learning_rate": 0.0001, + "loss": 65.8812, + "step": 6350 + }, + { + "epoch": 10.24185015367562, + "grad_norm": 1.291536808013916, + "learning_rate": 0.0001, + "loss": 64.9125, + "step": 6360 + }, + { + "epoch": 10.257973497253992, + "grad_norm": 0.8942781686782837, + "learning_rate": 0.0001, + "loss": 67.1706, + "step": 6370 + }, + { + "epoch": 10.274096840832367, + "grad_norm": 0.8272124528884888, + "learning_rate": 0.0001, + "loss": 62.1721, + "step": 6380 + }, + { + "epoch": 10.290220184410742, + "grad_norm": 0.8191152215003967, + "learning_rate": 0.0001, + "loss": 64.8931, + "step": 6390 + }, + { + "epoch": 10.306343527989117, + "grad_norm": 1.7500241994857788, + "learning_rate": 0.0001, + "loss": 62.9693, + "step": 6400 + }, + { + "epoch": 10.32246687156749, + "grad_norm": 1.0159430503845215, + "learning_rate": 0.0001, + "loss": 63.0889, + "step": 6410 + }, + { + "epoch": 10.338590215145866, + "grad_norm": 0.7300348281860352, + "learning_rate": 0.0001, + "loss": 65.1742, + "step": 6420 + }, + { + "epoch": 10.35471355872424, + "grad_norm": 1.3132387399673462, + "learning_rate": 0.0001, + "loss": 64.8867, + "step": 6430 + }, + { + "epoch": 10.370836902302615, + "grad_norm": 0.7502451539039612, + "learning_rate": 0.0001, + "loss": 66.5094, + "step": 6440 + }, + { + "epoch": 10.38696024588099, + "grad_norm": 0.9408686757087708, + "learning_rate": 0.0001, + "loss": 65.1515, + "step": 6450 + }, + { + "epoch": 10.403083589459364, + "grad_norm": 0.769598126411438, + "learning_rate": 0.0001, + "loss": 65.1478, + "step": 6460 + }, + { + "epoch": 10.419206933037739, + "grad_norm": 0.9086162447929382, + "learning_rate": 0.0001, + "loss": 63.5263, + "step": 6470 + }, + { + "epoch": 10.435330276616114, + "grad_norm": 0.7899322509765625, + "learning_rate": 0.0001, + "loss": 63.3939, + "step": 6480 + }, + { + "epoch": 10.451453620194489, + "grad_norm": 0.9135434031486511, + "learning_rate": 0.0001, + "loss": 63.867, + "step": 6490 + }, + { + "epoch": 10.467576963772862, + "grad_norm": 0.9312737584114075, + "learning_rate": 0.0001, + "loss": 63.6939, + "step": 6500 + }, + { + "epoch": 10.483700307351237, + "grad_norm": 0.861764132976532, + "learning_rate": 0.0001, + "loss": 63.4523, + "step": 6510 + }, + { + "epoch": 10.499823650929612, + "grad_norm": 0.7056366801261902, + "learning_rate": 0.0001, + "loss": 63.2889, + "step": 6520 + }, + { + "epoch": 10.515946994507987, + "grad_norm": 0.7007710337638855, + "learning_rate": 0.0001, + "loss": 62.2425, + "step": 6530 + }, + { + "epoch": 10.53207033808636, + "grad_norm": 0.7091413736343384, + "learning_rate": 0.0001, + "loss": 63.4185, + "step": 6540 + }, + { + "epoch": 10.548193681664735, + "grad_norm": 0.7147234678268433, + "learning_rate": 0.0001, + "loss": 65.0726, + "step": 6550 + }, + { + "epoch": 10.56431702524311, + "grad_norm": 0.7954673767089844, + "learning_rate": 0.0001, + "loss": 65.0334, + "step": 6560 + }, + { + "epoch": 10.580440368821485, + "grad_norm": 0.725352942943573, + "learning_rate": 0.0001, + "loss": 62.6663, + "step": 6570 + }, + { + "epoch": 10.59656371239986, + "grad_norm": 0.9563508629798889, + "learning_rate": 0.0001, + "loss": 64.0705, + "step": 6580 + }, + { + "epoch": 10.612687055978233, + "grad_norm": 0.7778734564781189, + "learning_rate": 0.0001, + "loss": 63.7619, + "step": 6590 + }, + { + "epoch": 10.628810399556608, + "grad_norm": 0.8414369821548462, + "learning_rate": 0.0001, + "loss": 63.9238, + "step": 6600 + }, + { + "epoch": 10.644933743134983, + "grad_norm": 0.8070562481880188, + "learning_rate": 0.0001, + "loss": 63.9591, + "step": 6610 + }, + { + "epoch": 10.661057086713358, + "grad_norm": 0.6493734121322632, + "learning_rate": 0.0001, + "loss": 63.7237, + "step": 6620 + }, + { + "epoch": 10.677180430291731, + "grad_norm": 0.8263372778892517, + "learning_rate": 0.0001, + "loss": 64.2262, + "step": 6630 + }, + { + "epoch": 10.693303773870106, + "grad_norm": 1.0090227127075195, + "learning_rate": 0.0001, + "loss": 63.5469, + "step": 6640 + }, + { + "epoch": 10.709427117448481, + "grad_norm": 0.998319685459137, + "learning_rate": 0.0001, + "loss": 62.9113, + "step": 6650 + }, + { + "epoch": 10.725550461026856, + "grad_norm": 1.006489634513855, + "learning_rate": 0.0001, + "loss": 66.1458, + "step": 6660 + }, + { + "epoch": 10.741673804605231, + "grad_norm": 0.9575757384300232, + "learning_rate": 0.0001, + "loss": 63.0913, + "step": 6670 + }, + { + "epoch": 10.757797148183604, + "grad_norm": 0.6832238435745239, + "learning_rate": 0.0001, + "loss": 66.7501, + "step": 6680 + }, + { + "epoch": 10.773920491761979, + "grad_norm": 0.8075717687606812, + "learning_rate": 0.0001, + "loss": 64.2839, + "step": 6690 + }, + { + "epoch": 10.790043835340354, + "grad_norm": 0.9402624368667603, + "learning_rate": 0.0001, + "loss": 64.9296, + "step": 6700 + }, + { + "epoch": 10.806167178918729, + "grad_norm": 0.7681158185005188, + "learning_rate": 0.0001, + "loss": 63.5686, + "step": 6710 + }, + { + "epoch": 10.822290522497102, + "grad_norm": 0.8451285362243652, + "learning_rate": 0.0001, + "loss": 64.5404, + "step": 6720 + }, + { + "epoch": 10.838413866075477, + "grad_norm": 0.7278769016265869, + "learning_rate": 0.0001, + "loss": 64.1341, + "step": 6730 + }, + { + "epoch": 10.854537209653852, + "grad_norm": 0.8524075746536255, + "learning_rate": 0.0001, + "loss": 64.0483, + "step": 6740 + }, + { + "epoch": 10.870660553232227, + "grad_norm": 0.703598141670227, + "learning_rate": 0.0001, + "loss": 64.0072, + "step": 6750 + }, + { + "epoch": 10.8867838968106, + "grad_norm": 0.7740384936332703, + "learning_rate": 0.0001, + "loss": 63.4171, + "step": 6760 + }, + { + "epoch": 10.902907240388975, + "grad_norm": 0.8302356004714966, + "learning_rate": 0.0001, + "loss": 65.5159, + "step": 6770 + }, + { + "epoch": 10.91903058396735, + "grad_norm": 1.0289733409881592, + "learning_rate": 0.0001, + "loss": 64.9785, + "step": 6780 + }, + { + "epoch": 10.935153927545725, + "grad_norm": 0.6721987128257751, + "learning_rate": 0.0001, + "loss": 63.965, + "step": 6790 + }, + { + "epoch": 10.951277271124098, + "grad_norm": 0.7614948749542236, + "learning_rate": 0.0001, + "loss": 64.1375, + "step": 6800 + }, + { + "epoch": 10.967400614702473, + "grad_norm": 0.7342216372489929, + "learning_rate": 0.0001, + "loss": 64.4342, + "step": 6810 + }, + { + "epoch": 10.983523958280848, + "grad_norm": 0.8808079957962036, + "learning_rate": 0.0001, + "loss": 65.2864, + "step": 6820 + }, + { + "epoch": 10.999647301859223, + "grad_norm": 0.818477988243103, + "learning_rate": 0.0001, + "loss": 63.9931, + "step": 6830 + }, + { + "epoch": 11.014511009220538, + "grad_norm": 0.7867474555969238, + "learning_rate": 0.0001, + "loss": 59.2426, + "step": 6840 + }, + { + "epoch": 11.030634352798911, + "grad_norm": 0.6163678765296936, + "learning_rate": 0.0001, + "loss": 65.0149, + "step": 6850 + }, + { + "epoch": 11.046757696377286, + "grad_norm": 0.8334609866142273, + "learning_rate": 0.0001, + "loss": 63.5017, + "step": 6860 + }, + { + "epoch": 11.062881039955661, + "grad_norm": 0.7335264086723328, + "learning_rate": 0.0001, + "loss": 64.6307, + "step": 6870 + }, + { + "epoch": 11.079004383534036, + "grad_norm": 0.7723846435546875, + "learning_rate": 0.0001, + "loss": 64.316, + "step": 6880 + }, + { + "epoch": 11.09512772711241, + "grad_norm": 0.7972145080566406, + "learning_rate": 0.0001, + "loss": 63.1469, + "step": 6890 + }, + { + "epoch": 11.111251070690784, + "grad_norm": 0.7696133852005005, + "learning_rate": 0.0001, + "loss": 64.0338, + "step": 6900 + }, + { + "epoch": 11.127374414269159, + "grad_norm": 0.8972133994102478, + "learning_rate": 0.0001, + "loss": 65.5727, + "step": 6910 + }, + { + "epoch": 11.143497757847534, + "grad_norm": 0.6776295900344849, + "learning_rate": 0.0001, + "loss": 66.176, + "step": 6920 + }, + { + "epoch": 11.159621101425909, + "grad_norm": 0.7345874905586243, + "learning_rate": 0.0001, + "loss": 64.0949, + "step": 6930 + }, + { + "epoch": 11.175744445004282, + "grad_norm": 0.7988374829292297, + "learning_rate": 0.0001, + "loss": 64.7721, + "step": 6940 + }, + { + "epoch": 11.191867788582657, + "grad_norm": 0.7552387714385986, + "learning_rate": 0.0001, + "loss": 63.1053, + "step": 6950 + }, + { + "epoch": 11.207991132161032, + "grad_norm": 0.6388214230537415, + "learning_rate": 0.0001, + "loss": 63.7871, + "step": 6960 + }, + { + "epoch": 11.224114475739407, + "grad_norm": 0.6837960481643677, + "learning_rate": 0.0001, + "loss": 63.906, + "step": 6970 + }, + { + "epoch": 11.24023781931778, + "grad_norm": 0.7050679326057434, + "learning_rate": 0.0001, + "loss": 64.2803, + "step": 6980 + }, + { + "epoch": 11.256361162896155, + "grad_norm": 0.8619478940963745, + "learning_rate": 0.0001, + "loss": 62.8786, + "step": 6990 + }, + { + "epoch": 11.27248450647453, + "grad_norm": 0.9759813547134399, + "learning_rate": 0.0001, + "loss": 65.2299, + "step": 7000 + }, + { + "epoch": 11.288607850052905, + "grad_norm": 0.7584514021873474, + "learning_rate": 0.0001, + "loss": 61.4178, + "step": 7010 + }, + { + "epoch": 11.304731193631278, + "grad_norm": 0.8467874526977539, + "learning_rate": 0.0001, + "loss": 65.6229, + "step": 7020 + }, + { + "epoch": 11.320854537209653, + "grad_norm": 0.7121654152870178, + "learning_rate": 0.0001, + "loss": 63.9061, + "step": 7030 + }, + { + "epoch": 11.336977880788028, + "grad_norm": 0.8885557055473328, + "learning_rate": 0.0001, + "loss": 65.1436, + "step": 7040 + }, + { + "epoch": 11.353101224366403, + "grad_norm": 0.6320177912712097, + "learning_rate": 0.0001, + "loss": 62.8535, + "step": 7050 + }, + { + "epoch": 11.369224567944778, + "grad_norm": 0.9287629723548889, + "learning_rate": 0.0001, + "loss": 62.7407, + "step": 7060 + }, + { + "epoch": 11.385347911523152, + "grad_norm": 0.7705608606338501, + "learning_rate": 0.0001, + "loss": 63.469, + "step": 7070 + }, + { + "epoch": 11.401471255101526, + "grad_norm": 0.6687246561050415, + "learning_rate": 0.0001, + "loss": 62.722, + "step": 7080 + }, + { + "epoch": 11.417594598679901, + "grad_norm": 0.7774648666381836, + "learning_rate": 0.0001, + "loss": 63.1145, + "step": 7090 + }, + { + "epoch": 11.433717942258276, + "grad_norm": 0.7808363437652588, + "learning_rate": 0.0001, + "loss": 63.9342, + "step": 7100 + }, + { + "epoch": 11.44984128583665, + "grad_norm": 0.8033562302589417, + "learning_rate": 0.0001, + "loss": 65.7876, + "step": 7110 + }, + { + "epoch": 11.465964629415025, + "grad_norm": 0.8143486976623535, + "learning_rate": 0.0001, + "loss": 64.8782, + "step": 7120 + }, + { + "epoch": 11.4820879729934, + "grad_norm": 0.7855561971664429, + "learning_rate": 0.0001, + "loss": 63.4944, + "step": 7130 + }, + { + "epoch": 11.498211316571775, + "grad_norm": 0.7525351047515869, + "learning_rate": 0.0001, + "loss": 62.4088, + "step": 7140 + }, + { + "epoch": 11.514334660150148, + "grad_norm": 0.7193494439125061, + "learning_rate": 0.0001, + "loss": 64.2386, + "step": 7150 + }, + { + "epoch": 11.530458003728523, + "grad_norm": 0.892281174659729, + "learning_rate": 0.0001, + "loss": 63.8075, + "step": 7160 + }, + { + "epoch": 11.546581347306898, + "grad_norm": 0.720287561416626, + "learning_rate": 0.0001, + "loss": 63.3259, + "step": 7170 + }, + { + "epoch": 11.562704690885273, + "grad_norm": 1.4404689073562622, + "learning_rate": 0.0001, + "loss": 62.4105, + "step": 7180 + }, + { + "epoch": 11.578828034463648, + "grad_norm": 0.9082726240158081, + "learning_rate": 0.0001, + "loss": 64.6877, + "step": 7190 + }, + { + "epoch": 11.59495137804202, + "grad_norm": 0.7137670516967773, + "learning_rate": 0.0001, + "loss": 63.4832, + "step": 7200 + }, + { + "epoch": 11.611074721620396, + "grad_norm": 0.6848503947257996, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 7210 + }, + { + "epoch": 11.62719806519877, + "grad_norm": 0.7483687996864319, + "learning_rate": 0.0001, + "loss": 63.294, + "step": 7220 + }, + { + "epoch": 11.643321408777146, + "grad_norm": 0.935321569442749, + "learning_rate": 0.0001, + "loss": 62.8669, + "step": 7230 + }, + { + "epoch": 11.659444752355519, + "grad_norm": 0.8359326124191284, + "learning_rate": 0.0001, + "loss": 64.4482, + "step": 7240 + }, + { + "epoch": 11.675568095933894, + "grad_norm": 0.7838515043258667, + "learning_rate": 0.0001, + "loss": 64.1975, + "step": 7250 + }, + { + "epoch": 11.691691439512269, + "grad_norm": 0.9128443598747253, + "learning_rate": 0.0001, + "loss": 63.1822, + "step": 7260 + }, + { + "epoch": 11.707814783090644, + "grad_norm": 1.0545419454574585, + "learning_rate": 0.0001, + "loss": 63.3112, + "step": 7270 + }, + { + "epoch": 11.723938126669019, + "grad_norm": 0.8572486639022827, + "learning_rate": 0.0001, + "loss": 64.9605, + "step": 7280 + }, + { + "epoch": 11.740061470247392, + "grad_norm": 1.0894116163253784, + "learning_rate": 0.0001, + "loss": 64.3671, + "step": 7290 + }, + { + "epoch": 11.756184813825767, + "grad_norm": 0.7500355839729309, + "learning_rate": 0.0001, + "loss": 62.6381, + "step": 7300 + }, + { + "epoch": 11.772308157404142, + "grad_norm": 0.7315185070037842, + "learning_rate": 0.0001, + "loss": 66.1438, + "step": 7310 + }, + { + "epoch": 11.788431500982517, + "grad_norm": 0.8123629093170166, + "learning_rate": 0.0001, + "loss": 62.6012, + "step": 7320 + }, + { + "epoch": 11.80455484456089, + "grad_norm": 0.8072763085365295, + "learning_rate": 0.0001, + "loss": 63.4424, + "step": 7330 + }, + { + "epoch": 11.820678188139265, + "grad_norm": 1.1077874898910522, + "learning_rate": 0.0001, + "loss": 63.2812, + "step": 7340 + }, + { + "epoch": 11.83680153171764, + "grad_norm": 0.7116726636886597, + "learning_rate": 0.0001, + "loss": 63.6077, + "step": 7350 + }, + { + "epoch": 11.852924875296015, + "grad_norm": 0.8817790746688843, + "learning_rate": 0.0001, + "loss": 63.1938, + "step": 7360 + }, + { + "epoch": 11.86904821887439, + "grad_norm": 1.021730661392212, + "learning_rate": 0.0001, + "loss": 65.2215, + "step": 7370 + }, + { + "epoch": 11.885171562452763, + "grad_norm": 0.9871250987052917, + "learning_rate": 0.0001, + "loss": 64.6468, + "step": 7380 + }, + { + "epoch": 11.901294906031138, + "grad_norm": 1.3591430187225342, + "learning_rate": 0.0001, + "loss": 61.5546, + "step": 7390 + }, + { + "epoch": 11.917418249609513, + "grad_norm": 0.8114227056503296, + "learning_rate": 0.0001, + "loss": 61.8215, + "step": 7400 + }, + { + "epoch": 11.933541593187888, + "grad_norm": 0.7151288986206055, + "learning_rate": 0.0001, + "loss": 61.4856, + "step": 7410 + }, + { + "epoch": 11.949664936766261, + "grad_norm": 0.7494287490844727, + "learning_rate": 0.0001, + "loss": 63.6451, + "step": 7420 + }, + { + "epoch": 11.965788280344636, + "grad_norm": 0.7377745509147644, + "learning_rate": 0.0001, + "loss": 63.002, + "step": 7430 + }, + { + "epoch": 11.981911623923011, + "grad_norm": 0.6534659266471863, + "learning_rate": 0.0001, + "loss": 63.6404, + "step": 7440 + }, + { + "epoch": 11.998034967501386, + "grad_norm": 0.7548736333847046, + "learning_rate": 0.0001, + "loss": 64.1986, + "step": 7450 + }, + { + "epoch": 12.012898674862699, + "grad_norm": 0.7227553129196167, + "learning_rate": 0.0001, + "loss": 56.8327, + "step": 7460 + }, + { + "epoch": 12.029022018441074, + "grad_norm": 0.6348695158958435, + "learning_rate": 0.0001, + "loss": 62.4472, + "step": 7470 + }, + { + "epoch": 12.045145362019449, + "grad_norm": 0.8249921202659607, + "learning_rate": 0.0001, + "loss": 65.1877, + "step": 7480 + }, + { + "epoch": 12.061268705597824, + "grad_norm": 0.7152777314186096, + "learning_rate": 0.0001, + "loss": 64.2609, + "step": 7490 + }, + { + "epoch": 12.077392049176197, + "grad_norm": 0.7399184703826904, + "learning_rate": 0.0001, + "loss": 63.3997, + "step": 7500 + }, + { + "epoch": 12.093515392754572, + "grad_norm": 0.6972354650497437, + "learning_rate": 0.0001, + "loss": 61.3573, + "step": 7510 + }, + { + "epoch": 12.109638736332947, + "grad_norm": 0.7666720151901245, + "learning_rate": 0.0001, + "loss": 63.8438, + "step": 7520 + }, + { + "epoch": 12.125762079911322, + "grad_norm": 0.7309302687644958, + "learning_rate": 0.0001, + "loss": 61.6775, + "step": 7530 + }, + { + "epoch": 12.141885423489697, + "grad_norm": 1.3805177211761475, + "learning_rate": 0.0001, + "loss": 64.68, + "step": 7540 + }, + { + "epoch": 12.15800876706807, + "grad_norm": 1.0315922498703003, + "learning_rate": 0.0001, + "loss": 63.6372, + "step": 7550 + }, + { + "epoch": 12.174132110646445, + "grad_norm": 1.2012155055999756, + "learning_rate": 0.0001, + "loss": 62.5765, + "step": 7560 + }, + { + "epoch": 12.19025545422482, + "grad_norm": 0.7769846320152283, + "learning_rate": 0.0001, + "loss": 63.9737, + "step": 7570 + }, + { + "epoch": 12.206378797803195, + "grad_norm": 0.6587379574775696, + "learning_rate": 0.0001, + "loss": 63.382, + "step": 7580 + }, + { + "epoch": 12.222502141381568, + "grad_norm": 0.8309435248374939, + "learning_rate": 0.0001, + "loss": 64.4027, + "step": 7590 + }, + { + "epoch": 12.238625484959943, + "grad_norm": 0.8221393823623657, + "learning_rate": 0.0001, + "loss": 62.8561, + "step": 7600 + }, + { + "epoch": 12.254748828538318, + "grad_norm": 0.8629311323165894, + "learning_rate": 0.0001, + "loss": 64.3673, + "step": 7610 + }, + { + "epoch": 12.270872172116693, + "grad_norm": 0.698113739490509, + "learning_rate": 0.0001, + "loss": 63.6153, + "step": 7620 + }, + { + "epoch": 12.286995515695068, + "grad_norm": 0.72906094789505, + "learning_rate": 0.0001, + "loss": 64.8439, + "step": 7630 + }, + { + "epoch": 12.303118859273441, + "grad_norm": 0.7402384877204895, + "learning_rate": 0.0001, + "loss": 64.2103, + "step": 7640 + }, + { + "epoch": 12.319242202851816, + "grad_norm": 0.9972009062767029, + "learning_rate": 0.0001, + "loss": 63.803, + "step": 7650 + }, + { + "epoch": 12.335365546430191, + "grad_norm": 0.7852274775505066, + "learning_rate": 0.0001, + "loss": 64.8144, + "step": 7660 + }, + { + "epoch": 12.351488890008566, + "grad_norm": 0.7433444857597351, + "learning_rate": 0.0001, + "loss": 63.0123, + "step": 7670 + }, + { + "epoch": 12.36761223358694, + "grad_norm": 0.8548759818077087, + "learning_rate": 0.0001, + "loss": 63.2215, + "step": 7680 + }, + { + "epoch": 12.383735577165314, + "grad_norm": 0.6759565472602844, + "learning_rate": 0.0001, + "loss": 62.1246, + "step": 7690 + }, + { + "epoch": 12.39985892074369, + "grad_norm": 0.9079536199569702, + "learning_rate": 0.0001, + "loss": 64.1692, + "step": 7700 + }, + { + "epoch": 12.415982264322064, + "grad_norm": 0.6890071034431458, + "learning_rate": 0.0001, + "loss": 62.5602, + "step": 7710 + }, + { + "epoch": 12.432105607900438, + "grad_norm": 0.8096348643302917, + "learning_rate": 0.0001, + "loss": 61.0103, + "step": 7720 + }, + { + "epoch": 12.448228951478812, + "grad_norm": 0.7359873056411743, + "learning_rate": 0.0001, + "loss": 62.1914, + "step": 7730 + }, + { + "epoch": 12.464352295057187, + "grad_norm": 1.0701091289520264, + "learning_rate": 0.0001, + "loss": 61.7295, + "step": 7740 + }, + { + "epoch": 12.480475638635562, + "grad_norm": 0.791928768157959, + "learning_rate": 0.0001, + "loss": 62.6812, + "step": 7750 + }, + { + "epoch": 12.496598982213937, + "grad_norm": 0.7719446420669556, + "learning_rate": 0.0001, + "loss": 61.9114, + "step": 7760 + }, + { + "epoch": 12.51272232579231, + "grad_norm": 0.8720338344573975, + "learning_rate": 0.0001, + "loss": 64.8107, + "step": 7770 + }, + { + "epoch": 12.528845669370686, + "grad_norm": 0.9999920129776001, + "learning_rate": 0.0001, + "loss": 61.198, + "step": 7780 + }, + { + "epoch": 12.54496901294906, + "grad_norm": 0.710754930973053, + "learning_rate": 0.0001, + "loss": 64.1462, + "step": 7790 + }, + { + "epoch": 12.561092356527435, + "grad_norm": 0.7401363849639893, + "learning_rate": 0.0001, + "loss": 61.8988, + "step": 7800 + }, + { + "epoch": 12.577215700105809, + "grad_norm": 0.9183729290962219, + "learning_rate": 0.0001, + "loss": 62.6392, + "step": 7810 + }, + { + "epoch": 12.593339043684184, + "grad_norm": 0.7341399788856506, + "learning_rate": 0.0001, + "loss": 62.7068, + "step": 7820 + }, + { + "epoch": 12.609462387262559, + "grad_norm": 0.8066722750663757, + "learning_rate": 0.0001, + "loss": 64.0625, + "step": 7830 + }, + { + "epoch": 12.625585730840934, + "grad_norm": 0.6088835597038269, + "learning_rate": 0.0001, + "loss": 64.3208, + "step": 7840 + }, + { + "epoch": 12.641709074419307, + "grad_norm": 0.6969302892684937, + "learning_rate": 0.0001, + "loss": 62.5817, + "step": 7850 + }, + { + "epoch": 12.657832417997682, + "grad_norm": 0.82635098695755, + "learning_rate": 0.0001, + "loss": 62.0061, + "step": 7860 + }, + { + "epoch": 12.673955761576057, + "grad_norm": 0.7016922235488892, + "learning_rate": 0.0001, + "loss": 62.5166, + "step": 7870 + }, + { + "epoch": 12.690079105154432, + "grad_norm": 0.8035777807235718, + "learning_rate": 0.0001, + "loss": 63.6543, + "step": 7880 + }, + { + "epoch": 12.706202448732807, + "grad_norm": 0.8573809862136841, + "learning_rate": 0.0001, + "loss": 62.5258, + "step": 7890 + }, + { + "epoch": 12.72232579231118, + "grad_norm": 0.9497472643852234, + "learning_rate": 0.0001, + "loss": 63.1133, + "step": 7900 + }, + { + "epoch": 12.738449135889555, + "grad_norm": 0.7494457960128784, + "learning_rate": 0.0001, + "loss": 63.8157, + "step": 7910 + }, + { + "epoch": 12.75457247946793, + "grad_norm": 0.7657570242881775, + "learning_rate": 0.0001, + "loss": 62.6255, + "step": 7920 + }, + { + "epoch": 12.770695823046305, + "grad_norm": 0.7848380208015442, + "learning_rate": 0.0001, + "loss": 62.9095, + "step": 7930 + }, + { + "epoch": 12.786819166624678, + "grad_norm": 0.8386877775192261, + "learning_rate": 0.0001, + "loss": 63.2254, + "step": 7940 + }, + { + "epoch": 12.802942510203053, + "grad_norm": 0.6774776577949524, + "learning_rate": 0.0001, + "loss": 64.3929, + "step": 7950 + }, + { + "epoch": 12.819065853781428, + "grad_norm": 0.798547625541687, + "learning_rate": 0.0001, + "loss": 62.9884, + "step": 7960 + }, + { + "epoch": 12.835189197359803, + "grad_norm": 0.9340888261795044, + "learning_rate": 0.0001, + "loss": 62.6197, + "step": 7970 + }, + { + "epoch": 12.851312540938178, + "grad_norm": 0.8211432695388794, + "learning_rate": 0.0001, + "loss": 62.7756, + "step": 7980 + }, + { + "epoch": 12.867435884516551, + "grad_norm": 0.7612001299858093, + "learning_rate": 0.0001, + "loss": 62.0792, + "step": 7990 + }, + { + "epoch": 12.883559228094926, + "grad_norm": 0.7743386626243591, + "learning_rate": 0.0001, + "loss": 63.9203, + "step": 8000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 33, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.784529817246577e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ade09d5233289ba4bc5076af39b8652078582dee --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:016a4e9b5d71a1947317d3801b00fe5d0604edbef1f981d7b6c92f219131717c +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-8000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..99b5928dc4b23a286b8a39a81b7ba51cb3fc3a73 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc143181971049502f06632936eea9f3c8ccfa87a6b5982687ef9ab02141409f +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..ad66941d8a4a859aa18e8017cb879260493845ad --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros10sf/checkpoint-final/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d36614926d6321b92164636079011a7236956d2d61a604c0ebb3100bc78860fa +size 6904 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/latest new file mode 100644 index 0000000000000000000000000000000000000000..25c776ee3abcad1c4d1e16e8275e4e00984a237c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/latest @@ -0,0 +1 @@ +global_step10000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c2f74686326ccef50ff0af6ebf93b2769b2b4d95 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77c72818fb8c8d898295d500d25e07fb4a7b1f91b34d28ed942c695e2ffae6c7 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..b8f3c593c97eb33687ab822f91732861164eb883 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f93ecf18e5f49234ad4425959c9a65f367367dd29f36366a009152a300d8ed4 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..2cc9ab764364def6e01d6f644a8aef73b24aa839 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:980601bd92effdcbec46ac82cecec33a9962f42f0a65d8f5c1050f775368ed98 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..32abedfd5336cf958c8cbda31697c210b68f45cf --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b60b026e7a80ff966182fa6d98aa3d4019c05dfd005b1868317f74e2cf01c8ec +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..88d98c01af35b7ce4ff02b96d25999b2ea6d624f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33488a373f7fafcb77ef5c4e55504c60db9c6ae8d786eb445bac2b9f35209711 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..a11ba137302da18b4f1a53c2435282f55389334e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fcba3a0770c8a2dcd1217ff1069b9e54b336dfcb016ccbdec842decd7018d2f +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..56596485b03b83560d0f0f9d8cba6825cbf82a14 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88ecb2c8114089312e7f5884e5738949f72a2ccf077d48e92b45efb2b989ded6 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..fc73c199f7b9342fb9eb6aa63deebb86c975f7ca --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcac1b8be38aa917b0acbab7f05b5b7be06c875e29191ea5564ba9aaa4a1b34b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..e5cf4ff384ee03a24efc8d6c1f31a6d0d76b3ce4 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:212fc488ba643c605468f1cd2d6e19061b2b5384d97fc74fb6227294508bd6ea +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c3b38e2c4b51d5a99b693a86d505062af0a9098 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:034d71d0f0787f0c3ec923c15d690debaef12acfa289bd37c34adb31a082231b +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..89f78e20ca9589a8d52e9664dbc7dd5bd1a3c52a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/trainer_state.json @@ -0,0 +1,7034 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 161.29268292682926, + "eval_steps": 2000, + "global_step": 10000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.16260162601626016, + "grad_norm": 23.72986602783203, + "learning_rate": 0.0001, + "loss": 271.6598, + "step": 10 + }, + { + "epoch": 0.3252032520325203, + "grad_norm": 18.99613380432129, + "learning_rate": 0.0001, + "loss": 233.029, + "step": 20 + }, + { + "epoch": 0.4878048780487805, + "grad_norm": 3.9815304279327393, + "learning_rate": 0.0001, + "loss": 231.0776, + "step": 30 + }, + { + "epoch": 0.6504065040650406, + "grad_norm": 4.641753196716309, + "learning_rate": 0.0001, + "loss": 228.6894, + "step": 40 + }, + { + "epoch": 0.8130081300813008, + "grad_norm": 3.000852108001709, + "learning_rate": 0.0001, + "loss": 228.4694, + "step": 50 + }, + { + "epoch": 0.975609756097561, + "grad_norm": 12.213018417358398, + "learning_rate": 0.0001, + "loss": 226.4258, + "step": 60 + }, + { + "epoch": 1.1300813008130082, + "grad_norm": 2.535187005996704, + "learning_rate": 0.0001, + "loss": 209.9772, + "step": 70 + }, + { + "epoch": 1.2926829268292683, + "grad_norm": 2.0984582901000977, + "learning_rate": 0.0001, + "loss": 217.2421, + "step": 80 + }, + { + "epoch": 1.4552845528455285, + "grad_norm": 3.4802305698394775, + "learning_rate": 0.0001, + "loss": 210.9241, + "step": 90 + }, + { + "epoch": 1.6178861788617886, + "grad_norm": 3.813626527786255, + "learning_rate": 0.0001, + "loss": 205.7566, + "step": 100 + }, + { + "epoch": 1.7804878048780488, + "grad_norm": 2.6378512382507324, + "learning_rate": 0.0001, + "loss": 201.8789, + "step": 110 + }, + { + "epoch": 1.943089430894309, + "grad_norm": 2.7502875328063965, + "learning_rate": 0.0001, + "loss": 196.373, + "step": 120 + }, + { + "epoch": 2.097560975609756, + "grad_norm": 2.659212112426758, + "learning_rate": 0.0001, + "loss": 183.1456, + "step": 130 + }, + { + "epoch": 2.2601626016260163, + "grad_norm": 2.513044595718384, + "learning_rate": 0.0001, + "loss": 190.1661, + "step": 140 + }, + { + "epoch": 2.4227642276422765, + "grad_norm": 2.3192145824432373, + "learning_rate": 0.0001, + "loss": 184.9997, + "step": 150 + }, + { + "epoch": 2.5853658536585367, + "grad_norm": 3.056708574295044, + "learning_rate": 0.0001, + "loss": 182.0164, + "step": 160 + }, + { + "epoch": 2.747967479674797, + "grad_norm": 2.9687108993530273, + "learning_rate": 0.0001, + "loss": 179.6539, + "step": 170 + }, + { + "epoch": 2.910569105691057, + "grad_norm": 2.1498093605041504, + "learning_rate": 0.0001, + "loss": 176.7159, + "step": 180 + }, + { + "epoch": 3.065040650406504, + "grad_norm": 3.0103771686553955, + "learning_rate": 0.0001, + "loss": 166.1703, + "step": 190 + }, + { + "epoch": 3.227642276422764, + "grad_norm": 2.992462635040283, + "learning_rate": 0.0001, + "loss": 171.124, + "step": 200 + }, + { + "epoch": 3.3902439024390243, + "grad_norm": 4.8617119789123535, + "learning_rate": 0.0001, + "loss": 168.2646, + "step": 210 + }, + { + "epoch": 3.5528455284552845, + "grad_norm": 3.1047444343566895, + "learning_rate": 0.0001, + "loss": 163.0889, + "step": 220 + }, + { + "epoch": 3.7154471544715446, + "grad_norm": 6.473227500915527, + "learning_rate": 0.0001, + "loss": 151.3667, + "step": 230 + }, + { + "epoch": 3.8780487804878048, + "grad_norm": 6.16747522354126, + "learning_rate": 0.0001, + "loss": 137.5915, + "step": 240 + }, + { + "epoch": 4.032520325203252, + "grad_norm": 7.994515419006348, + "learning_rate": 0.0001, + "loss": 118.8875, + "step": 250 + }, + { + "epoch": 4.195121951219512, + "grad_norm": 5.882101058959961, + "learning_rate": 0.0001, + "loss": 119.4793, + "step": 260 + }, + { + "epoch": 4.357723577235772, + "grad_norm": 12.32968521118164, + "learning_rate": 0.0001, + "loss": 113.1209, + "step": 270 + }, + { + "epoch": 4.520325203252033, + "grad_norm": 5.697225570678711, + "learning_rate": 0.0001, + "loss": 107.51, + "step": 280 + }, + { + "epoch": 4.682926829268292, + "grad_norm": 4.654167175292969, + "learning_rate": 0.0001, + "loss": 107.0172, + "step": 290 + }, + { + "epoch": 4.845528455284553, + "grad_norm": 2.9595484733581543, + "learning_rate": 0.0001, + "loss": 100.9164, + "step": 300 + }, + { + "epoch": 5.0, + "grad_norm": 21.242591857910156, + "learning_rate": 0.0001, + "loss": 94.9884, + "step": 310 + }, + { + "epoch": 5.16260162601626, + "grad_norm": 5.760314464569092, + "learning_rate": 0.0001, + "loss": 97.5901, + "step": 320 + }, + { + "epoch": 5.32520325203252, + "grad_norm": 3.1373329162597656, + "learning_rate": 0.0001, + "loss": 94.8485, + "step": 330 + }, + { + "epoch": 5.487804878048781, + "grad_norm": 2.3777568340301514, + "learning_rate": 0.0001, + "loss": 96.9982, + "step": 340 + }, + { + "epoch": 5.650406504065041, + "grad_norm": 3.0607948303222656, + "learning_rate": 0.0001, + "loss": 96.1586, + "step": 350 + }, + { + "epoch": 5.8130081300813, + "grad_norm": 4.168269634246826, + "learning_rate": 0.0001, + "loss": 94.5743, + "step": 360 + }, + { + "epoch": 5.975609756097561, + "grad_norm": 2.4383699893951416, + "learning_rate": 0.0001, + "loss": 95.5197, + "step": 370 + }, + { + "epoch": 6.130081300813008, + "grad_norm": 2.302259683609009, + "learning_rate": 0.0001, + "loss": 85.6599, + "step": 380 + }, + { + "epoch": 6.2926829268292686, + "grad_norm": 2.552753210067749, + "learning_rate": 0.0001, + "loss": 91.2999, + "step": 390 + }, + { + "epoch": 6.455284552845528, + "grad_norm": 2.2092080116271973, + "learning_rate": 0.0001, + "loss": 90.7041, + "step": 400 + }, + { + "epoch": 6.617886178861789, + "grad_norm": 2.2935690879821777, + "learning_rate": 0.0001, + "loss": 89.9949, + "step": 410 + }, + { + "epoch": 6.780487804878049, + "grad_norm": 2.487510919570923, + "learning_rate": 0.0001, + "loss": 86.9056, + "step": 420 + }, + { + "epoch": 6.943089430894309, + "grad_norm": 2.8630545139312744, + "learning_rate": 0.0001, + "loss": 88.3347, + "step": 430 + }, + { + "epoch": 7.097560975609756, + "grad_norm": 5.87056303024292, + "learning_rate": 0.0001, + "loss": 84.9632, + "step": 440 + }, + { + "epoch": 7.260162601626016, + "grad_norm": 2.587661027908325, + "learning_rate": 0.0001, + "loss": 86.2024, + "step": 450 + }, + { + "epoch": 7.4227642276422765, + "grad_norm": 3.5671255588531494, + "learning_rate": 0.0001, + "loss": 86.218, + "step": 460 + }, + { + "epoch": 7.585365853658536, + "grad_norm": 2.5036680698394775, + "learning_rate": 0.0001, + "loss": 86.5914, + "step": 470 + }, + { + "epoch": 7.747967479674797, + "grad_norm": 3.2656471729278564, + "learning_rate": 0.0001, + "loss": 82.6081, + "step": 480 + }, + { + "epoch": 7.9105691056910565, + "grad_norm": 2.5053915977478027, + "learning_rate": 0.0001, + "loss": 84.6116, + "step": 490 + }, + { + "epoch": 8.065040650406504, + "grad_norm": 2.1669957637786865, + "learning_rate": 0.0001, + "loss": 80.4354, + "step": 500 + }, + { + "epoch": 8.227642276422765, + "grad_norm": 4.2003703117370605, + "learning_rate": 0.0001, + "loss": 85.2286, + "step": 510 + }, + { + "epoch": 8.390243902439025, + "grad_norm": 3.1909608840942383, + "learning_rate": 0.0001, + "loss": 85.0358, + "step": 520 + }, + { + "epoch": 8.552845528455284, + "grad_norm": 1.8420121669769287, + "learning_rate": 0.0001, + "loss": 85.4647, + "step": 530 + }, + { + "epoch": 8.715447154471544, + "grad_norm": 1.9822801351547241, + "learning_rate": 0.0001, + "loss": 84.7134, + "step": 540 + }, + { + "epoch": 8.878048780487806, + "grad_norm": 2.936758518218994, + "learning_rate": 0.0001, + "loss": 83.8068, + "step": 550 + }, + { + "epoch": 9.032520325203253, + "grad_norm": 1.5713881254196167, + "learning_rate": 0.0001, + "loss": 80.0695, + "step": 560 + }, + { + "epoch": 9.195121951219512, + "grad_norm": 2.0584616661071777, + "learning_rate": 0.0001, + "loss": 84.5158, + "step": 570 + }, + { + "epoch": 9.357723577235772, + "grad_norm": 2.101391553878784, + "learning_rate": 0.0001, + "loss": 82.4719, + "step": 580 + }, + { + "epoch": 9.520325203252032, + "grad_norm": 1.9956543445587158, + "learning_rate": 0.0001, + "loss": 81.6715, + "step": 590 + }, + { + "epoch": 9.682926829268293, + "grad_norm": 2.7622063159942627, + "learning_rate": 0.0001, + "loss": 85.4178, + "step": 600 + }, + { + "epoch": 9.845528455284553, + "grad_norm": 2.2084789276123047, + "learning_rate": 0.0001, + "loss": 84.5702, + "step": 610 + }, + { + "epoch": 10.0, + "grad_norm": 1.9635663032531738, + "learning_rate": 0.0001, + "loss": 77.3838, + "step": 620 + }, + { + "epoch": 10.16260162601626, + "grad_norm": 2.5895581245422363, + "learning_rate": 0.0001, + "loss": 81.723, + "step": 630 + }, + { + "epoch": 10.32520325203252, + "grad_norm": 2.743727445602417, + "learning_rate": 0.0001, + "loss": 83.1332, + "step": 640 + }, + { + "epoch": 10.487804878048781, + "grad_norm": 2.024585008621216, + "learning_rate": 0.0001, + "loss": 81.5968, + "step": 650 + }, + { + "epoch": 10.65040650406504, + "grad_norm": 1.8254153728485107, + "learning_rate": 0.0001, + "loss": 78.9589, + "step": 660 + }, + { + "epoch": 10.8130081300813, + "grad_norm": 3.2015740871429443, + "learning_rate": 0.0001, + "loss": 79.1829, + "step": 670 + }, + { + "epoch": 10.975609756097562, + "grad_norm": 1.788453459739685, + "learning_rate": 0.0001, + "loss": 81.527, + "step": 680 + }, + { + "epoch": 11.130081300813009, + "grad_norm": 1.5744423866271973, + "learning_rate": 0.0001, + "loss": 74.9869, + "step": 690 + }, + { + "epoch": 11.292682926829269, + "grad_norm": 2.006002187728882, + "learning_rate": 0.0001, + "loss": 80.2677, + "step": 700 + }, + { + "epoch": 11.455284552845528, + "grad_norm": 1.8847148418426514, + "learning_rate": 0.0001, + "loss": 78.8709, + "step": 710 + }, + { + "epoch": 11.617886178861788, + "grad_norm": 1.8384478092193604, + "learning_rate": 0.0001, + "loss": 79.008, + "step": 720 + }, + { + "epoch": 11.78048780487805, + "grad_norm": 1.9486687183380127, + "learning_rate": 0.0001, + "loss": 76.1525, + "step": 730 + }, + { + "epoch": 11.94308943089431, + "grad_norm": 1.8644076585769653, + "learning_rate": 0.0001, + "loss": 77.5802, + "step": 740 + }, + { + "epoch": 12.097560975609756, + "grad_norm": 1.6035499572753906, + "learning_rate": 0.0001, + "loss": 75.2422, + "step": 750 + }, + { + "epoch": 12.260162601626016, + "grad_norm": 1.6935622692108154, + "learning_rate": 0.0001, + "loss": 77.5838, + "step": 760 + }, + { + "epoch": 12.422764227642276, + "grad_norm": 1.9327987432479858, + "learning_rate": 0.0001, + "loss": 78.3237, + "step": 770 + }, + { + "epoch": 12.585365853658537, + "grad_norm": 1.7903852462768555, + "learning_rate": 0.0001, + "loss": 78.7459, + "step": 780 + }, + { + "epoch": 12.747967479674797, + "grad_norm": 1.4962575435638428, + "learning_rate": 0.0001, + "loss": 78.9193, + "step": 790 + }, + { + "epoch": 12.910569105691057, + "grad_norm": 42.10435104370117, + "learning_rate": 0.0001, + "loss": 80.1429, + "step": 800 + }, + { + "epoch": 13.065040650406504, + "grad_norm": 1.904065728187561, + "learning_rate": 0.0001, + "loss": 73.7798, + "step": 810 + }, + { + "epoch": 13.227642276422765, + "grad_norm": 1.8768481016159058, + "learning_rate": 0.0001, + "loss": 76.8692, + "step": 820 + }, + { + "epoch": 13.390243902439025, + "grad_norm": 16.140565872192383, + "learning_rate": 0.0001, + "loss": 78.122, + "step": 830 + }, + { + "epoch": 13.552845528455284, + "grad_norm": 2.160288095474243, + "learning_rate": 0.0001, + "loss": 76.2169, + "step": 840 + }, + { + "epoch": 13.715447154471544, + "grad_norm": 27.682231903076172, + "learning_rate": 0.0001, + "loss": 76.6266, + "step": 850 + }, + { + "epoch": 13.878048780487806, + "grad_norm": 1.779772162437439, + "learning_rate": 0.0001, + "loss": 80.0434, + "step": 860 + }, + { + "epoch": 14.032520325203253, + "grad_norm": 1.6516485214233398, + "learning_rate": 0.0001, + "loss": 73.3943, + "step": 870 + }, + { + "epoch": 14.195121951219512, + "grad_norm": 1.846498966217041, + "learning_rate": 0.0001, + "loss": 77.5157, + "step": 880 + }, + { + "epoch": 14.357723577235772, + "grad_norm": 1.942946434020996, + "learning_rate": 0.0001, + "loss": 77.6944, + "step": 890 + }, + { + "epoch": 14.520325203252032, + "grad_norm": 2.0929224491119385, + "learning_rate": 0.0001, + "loss": 76.9411, + "step": 900 + }, + { + "epoch": 14.682926829268293, + "grad_norm": 3.4498767852783203, + "learning_rate": 0.0001, + "loss": 74.9234, + "step": 910 + }, + { + "epoch": 14.845528455284553, + "grad_norm": 1.6880728006362915, + "learning_rate": 0.0001, + "loss": 75.3617, + "step": 920 + }, + { + "epoch": 15.0, + "grad_norm": 1.8403468132019043, + "learning_rate": 0.0001, + "loss": 73.2543, + "step": 930 + }, + { + "epoch": 15.16260162601626, + "grad_norm": 1.4368852376937866, + "learning_rate": 0.0001, + "loss": 76.6271, + "step": 940 + }, + { + "epoch": 15.32520325203252, + "grad_norm": 2.9467780590057373, + "learning_rate": 0.0001, + "loss": 75.8744, + "step": 950 + }, + { + "epoch": 15.487804878048781, + "grad_norm": 1.567134141921997, + "learning_rate": 0.0001, + "loss": 76.6629, + "step": 960 + }, + { + "epoch": 15.65040650406504, + "grad_norm": 1.4503871202468872, + "learning_rate": 0.0001, + "loss": 76.3525, + "step": 970 + }, + { + "epoch": 15.8130081300813, + "grad_norm": 1.8871147632598877, + "learning_rate": 0.0001, + "loss": 77.4684, + "step": 980 + }, + { + "epoch": 15.975609756097562, + "grad_norm": 1.6707026958465576, + "learning_rate": 0.0001, + "loss": 76.3063, + "step": 990 + }, + { + "epoch": 16.130081300813007, + "grad_norm": 3.6300716400146484, + "learning_rate": 0.0001, + "loss": 73.0586, + "step": 1000 + }, + { + "epoch": 16.29268292682927, + "grad_norm": 1.6669234037399292, + "learning_rate": 0.0001, + "loss": 75.8027, + "step": 1010 + }, + { + "epoch": 16.45528455284553, + "grad_norm": 1.6920552253723145, + "learning_rate": 0.0001, + "loss": 74.7252, + "step": 1020 + }, + { + "epoch": 16.617886178861788, + "grad_norm": 1.8184431791305542, + "learning_rate": 0.0001, + "loss": 75.3625, + "step": 1030 + }, + { + "epoch": 16.78048780487805, + "grad_norm": 1.5828496217727661, + "learning_rate": 0.0001, + "loss": 78.0302, + "step": 1040 + }, + { + "epoch": 16.943089430894307, + "grad_norm": 1.7372915744781494, + "learning_rate": 0.0001, + "loss": 78.6949, + "step": 1050 + }, + { + "epoch": 17.097560975609756, + "grad_norm": 1.837161898612976, + "learning_rate": 0.0001, + "loss": 74.4726, + "step": 1060 + }, + { + "epoch": 17.260162601626018, + "grad_norm": 3.8733808994293213, + "learning_rate": 0.0001, + "loss": 76.1711, + "step": 1070 + }, + { + "epoch": 17.422764227642276, + "grad_norm": 1.5907851457595825, + "learning_rate": 0.0001, + "loss": 75.8292, + "step": 1080 + }, + { + "epoch": 17.585365853658537, + "grad_norm": 1.8271747827529907, + "learning_rate": 0.0001, + "loss": 74.5316, + "step": 1090 + }, + { + "epoch": 17.747967479674795, + "grad_norm": 1.827980875968933, + "learning_rate": 0.0001, + "loss": 74.7281, + "step": 1100 + }, + { + "epoch": 17.910569105691057, + "grad_norm": 1.4184074401855469, + "learning_rate": 0.0001, + "loss": 75.736, + "step": 1110 + }, + { + "epoch": 18.065040650406505, + "grad_norm": 2.3459699153900146, + "learning_rate": 0.0001, + "loss": 71.9586, + "step": 1120 + }, + { + "epoch": 18.227642276422763, + "grad_norm": 47.673641204833984, + "learning_rate": 0.0001, + "loss": 75.6649, + "step": 1130 + }, + { + "epoch": 18.390243902439025, + "grad_norm": 1.677398443222046, + "learning_rate": 0.0001, + "loss": 76.0574, + "step": 1140 + }, + { + "epoch": 18.552845528455286, + "grad_norm": 1.4180850982666016, + "learning_rate": 0.0001, + "loss": 74.3077, + "step": 1150 + }, + { + "epoch": 18.715447154471544, + "grad_norm": 2.0041465759277344, + "learning_rate": 0.0001, + "loss": 73.5916, + "step": 1160 + }, + { + "epoch": 18.878048780487806, + "grad_norm": 1.9098238945007324, + "learning_rate": 0.0001, + "loss": 74.5002, + "step": 1170 + }, + { + "epoch": 19.03252032520325, + "grad_norm": 1.405555248260498, + "learning_rate": 0.0001, + "loss": 69.8115, + "step": 1180 + }, + { + "epoch": 19.195121951219512, + "grad_norm": 1.424198865890503, + "learning_rate": 0.0001, + "loss": 73.7289, + "step": 1190 + }, + { + "epoch": 19.357723577235774, + "grad_norm": 7.542550563812256, + "learning_rate": 0.0001, + "loss": 73.7696, + "step": 1200 + }, + { + "epoch": 19.520325203252032, + "grad_norm": 1.425329566001892, + "learning_rate": 0.0001, + "loss": 72.1261, + "step": 1210 + }, + { + "epoch": 19.682926829268293, + "grad_norm": 1.525724172592163, + "learning_rate": 0.0001, + "loss": 73.6578, + "step": 1220 + }, + { + "epoch": 19.84552845528455, + "grad_norm": 1.6387193202972412, + "learning_rate": 0.0001, + "loss": 74.3027, + "step": 1230 + }, + { + "epoch": 20.0, + "grad_norm": 1.8429840803146362, + "learning_rate": 0.0001, + "loss": 69.5056, + "step": 1240 + }, + { + "epoch": 20.16260162601626, + "grad_norm": 2.5166006088256836, + "learning_rate": 0.0001, + "loss": 73.5965, + "step": 1250 + }, + { + "epoch": 20.32520325203252, + "grad_norm": 1.6006672382354736, + "learning_rate": 0.0001, + "loss": 72.4581, + "step": 1260 + }, + { + "epoch": 20.48780487804878, + "grad_norm": 1.6722878217697144, + "learning_rate": 0.0001, + "loss": 72.3097, + "step": 1270 + }, + { + "epoch": 20.65040650406504, + "grad_norm": 1.8715728521347046, + "learning_rate": 0.0001, + "loss": 73.4752, + "step": 1280 + }, + { + "epoch": 20.8130081300813, + "grad_norm": 1.3280900716781616, + "learning_rate": 0.0001, + "loss": 73.2145, + "step": 1290 + }, + { + "epoch": 20.975609756097562, + "grad_norm": 1.941451907157898, + "learning_rate": 0.0001, + "loss": 73.2085, + "step": 1300 + }, + { + "epoch": 21.130081300813007, + "grad_norm": 4.277497291564941, + "learning_rate": 0.0001, + "loss": 68.8305, + "step": 1310 + }, + { + "epoch": 21.29268292682927, + "grad_norm": 1.9949846267700195, + "learning_rate": 0.0001, + "loss": 72.6689, + "step": 1320 + }, + { + "epoch": 21.45528455284553, + "grad_norm": 1.524044394493103, + "learning_rate": 0.0001, + "loss": 72.9705, + "step": 1330 + }, + { + "epoch": 21.617886178861788, + "grad_norm": 1.384194016456604, + "learning_rate": 0.0001, + "loss": 72.5465, + "step": 1340 + }, + { + "epoch": 21.78048780487805, + "grad_norm": 1.724225640296936, + "learning_rate": 0.0001, + "loss": 72.9203, + "step": 1350 + }, + { + "epoch": 21.943089430894307, + "grad_norm": 2.0856704711914062, + "learning_rate": 0.0001, + "loss": 72.4034, + "step": 1360 + }, + { + "epoch": 22.097560975609756, + "grad_norm": 5.128044128417969, + "learning_rate": 0.0001, + "loss": 68.7754, + "step": 1370 + }, + { + "epoch": 22.260162601626018, + "grad_norm": 1.7385634183883667, + "learning_rate": 0.0001, + "loss": 73.9349, + "step": 1380 + }, + { + "epoch": 22.422764227642276, + "grad_norm": 1.3946012258529663, + "learning_rate": 0.0001, + "loss": 70.9627, + "step": 1390 + }, + { + "epoch": 22.585365853658537, + "grad_norm": 1.285414457321167, + "learning_rate": 0.0001, + "loss": 72.4578, + "step": 1400 + }, + { + "epoch": 22.747967479674795, + "grad_norm": 1.4197710752487183, + "learning_rate": 0.0001, + "loss": 72.5252, + "step": 1410 + }, + { + "epoch": 22.910569105691057, + "grad_norm": 3.048942804336548, + "learning_rate": 0.0001, + "loss": 71.1107, + "step": 1420 + }, + { + "epoch": 23.065040650406505, + "grad_norm": 1.3114765882492065, + "learning_rate": 0.0001, + "loss": 68.2725, + "step": 1430 + }, + { + "epoch": 23.227642276422763, + "grad_norm": 1.2763880491256714, + "learning_rate": 0.0001, + "loss": 71.5783, + "step": 1440 + }, + { + "epoch": 23.390243902439025, + "grad_norm": 1.3913089036941528, + "learning_rate": 0.0001, + "loss": 72.5986, + "step": 1450 + }, + { + "epoch": 23.552845528455286, + "grad_norm": 1.628751516342163, + "learning_rate": 0.0001, + "loss": 71.8383, + "step": 1460 + }, + { + "epoch": 23.715447154471544, + "grad_norm": 1.4331274032592773, + "learning_rate": 0.0001, + "loss": 70.8051, + "step": 1470 + }, + { + "epoch": 23.878048780487806, + "grad_norm": 1.4154727458953857, + "learning_rate": 0.0001, + "loss": 71.3283, + "step": 1480 + }, + { + "epoch": 24.03252032520325, + "grad_norm": 1.7208784818649292, + "learning_rate": 0.0001, + "loss": 68.4719, + "step": 1490 + }, + { + "epoch": 24.195121951219512, + "grad_norm": 1.466123342514038, + "learning_rate": 0.0001, + "loss": 71.9597, + "step": 1500 + }, + { + "epoch": 24.357723577235774, + "grad_norm": 1.4823886156082153, + "learning_rate": 0.0001, + "loss": 72.4533, + "step": 1510 + }, + { + "epoch": 24.520325203252032, + "grad_norm": 1.8407355546951294, + "learning_rate": 0.0001, + "loss": 71.1468, + "step": 1520 + }, + { + "epoch": 24.682926829268293, + "grad_norm": 1.630565881729126, + "learning_rate": 0.0001, + "loss": 71.5192, + "step": 1530 + }, + { + "epoch": 24.84552845528455, + "grad_norm": 1.5248785018920898, + "learning_rate": 0.0001, + "loss": 68.4804, + "step": 1540 + }, + { + "epoch": 25.0, + "grad_norm": 1.4690219163894653, + "learning_rate": 0.0001, + "loss": 67.5716, + "step": 1550 + }, + { + "epoch": 25.16260162601626, + "grad_norm": 2.0246646404266357, + "learning_rate": 0.0001, + "loss": 71.2554, + "step": 1560 + }, + { + "epoch": 25.32520325203252, + "grad_norm": 1.3611458539962769, + "learning_rate": 0.0001, + "loss": 69.8558, + "step": 1570 + }, + { + "epoch": 25.48780487804878, + "grad_norm": 2.0006039142608643, + "learning_rate": 0.0001, + "loss": 72.5344, + "step": 1580 + }, + { + "epoch": 25.65040650406504, + "grad_norm": 1.7373167276382446, + "learning_rate": 0.0001, + "loss": 70.6253, + "step": 1590 + }, + { + "epoch": 25.8130081300813, + "grad_norm": 1.2878378629684448, + "learning_rate": 0.0001, + "loss": 72.165, + "step": 1600 + }, + { + "epoch": 25.975609756097562, + "grad_norm": 1.5337013006210327, + "learning_rate": 0.0001, + "loss": 70.6989, + "step": 1610 + }, + { + "epoch": 26.130081300813007, + "grad_norm": 1.364615559577942, + "learning_rate": 0.0001, + "loss": 67.3898, + "step": 1620 + }, + { + "epoch": 26.29268292682927, + "grad_norm": 1.3249316215515137, + "learning_rate": 0.0001, + "loss": 72.7102, + "step": 1630 + }, + { + "epoch": 26.45528455284553, + "grad_norm": 4.209898471832275, + "learning_rate": 0.0001, + "loss": 70.3185, + "step": 1640 + }, + { + "epoch": 26.617886178861788, + "grad_norm": 1.5488810539245605, + "learning_rate": 0.0001, + "loss": 72.3678, + "step": 1650 + }, + { + "epoch": 26.78048780487805, + "grad_norm": 1.7743597030639648, + "learning_rate": 0.0001, + "loss": 71.0917, + "step": 1660 + }, + { + "epoch": 26.943089430894307, + "grad_norm": 1.5433533191680908, + "learning_rate": 0.0001, + "loss": 71.4001, + "step": 1670 + }, + { + "epoch": 27.097560975609756, + "grad_norm": 1.6320385932922363, + "learning_rate": 0.0001, + "loss": 66.7441, + "step": 1680 + }, + { + "epoch": 27.260162601626018, + "grad_norm": 1.3137969970703125, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 1690 + }, + { + "epoch": 27.422764227642276, + "grad_norm": 1.9150484800338745, + "learning_rate": 0.0001, + "loss": 69.8956, + "step": 1700 + }, + { + "epoch": 27.585365853658537, + "grad_norm": 1.4442802667617798, + "learning_rate": 0.0001, + "loss": 70.7185, + "step": 1710 + }, + { + "epoch": 27.747967479674795, + "grad_norm": 2.572497606277466, + "learning_rate": 0.0001, + "loss": 70.8934, + "step": 1720 + }, + { + "epoch": 27.910569105691057, + "grad_norm": 1.6669617891311646, + "learning_rate": 0.0001, + "loss": 71.8221, + "step": 1730 + }, + { + "epoch": 28.065040650406505, + "grad_norm": 1.8449876308441162, + "learning_rate": 0.0001, + "loss": 66.3923, + "step": 1740 + }, + { + "epoch": 28.227642276422763, + "grad_norm": 1.6600004434585571, + "learning_rate": 0.0001, + "loss": 69.5301, + "step": 1750 + }, + { + "epoch": 28.390243902439025, + "grad_norm": 1.4655731916427612, + "learning_rate": 0.0001, + "loss": 72.1924, + "step": 1760 + }, + { + "epoch": 28.552845528455286, + "grad_norm": 1.134505033493042, + "learning_rate": 0.0001, + "loss": 69.5855, + "step": 1770 + }, + { + "epoch": 28.715447154471544, + "grad_norm": 1.301984190940857, + "learning_rate": 0.0001, + "loss": 70.7999, + "step": 1780 + }, + { + "epoch": 28.878048780487806, + "grad_norm": 1.5641013383865356, + "learning_rate": 0.0001, + "loss": 69.9091, + "step": 1790 + }, + { + "epoch": 29.03252032520325, + "grad_norm": 1.4850797653198242, + "learning_rate": 0.0001, + "loss": 66.6266, + "step": 1800 + }, + { + "epoch": 29.195121951219512, + "grad_norm": 1.2069118022918701, + "learning_rate": 0.0001, + "loss": 69.3802, + "step": 1810 + }, + { + "epoch": 29.357723577235774, + "grad_norm": 1.2890352010726929, + "learning_rate": 0.0001, + "loss": 68.5044, + "step": 1820 + }, + { + "epoch": 29.520325203252032, + "grad_norm": 1.4150917530059814, + "learning_rate": 0.0001, + "loss": 69.807, + "step": 1830 + }, + { + "epoch": 29.682926829268293, + "grad_norm": 1.514807939529419, + "learning_rate": 0.0001, + "loss": 71.3858, + "step": 1840 + }, + { + "epoch": 29.84552845528455, + "grad_norm": 1.4246330261230469, + "learning_rate": 0.0001, + "loss": 70.494, + "step": 1850 + }, + { + "epoch": 30.0, + "grad_norm": 1.1789597272872925, + "learning_rate": 0.0001, + "loss": 67.1056, + "step": 1860 + }, + { + "epoch": 30.16260162601626, + "grad_norm": 1.23678719997406, + "learning_rate": 0.0001, + "loss": 70.336, + "step": 1870 + }, + { + "epoch": 30.32520325203252, + "grad_norm": 1.1756274700164795, + "learning_rate": 0.0001, + "loss": 67.9909, + "step": 1880 + }, + { + "epoch": 30.48780487804878, + "grad_norm": 1.4175690412521362, + "learning_rate": 0.0001, + "loss": 69.2617, + "step": 1890 + }, + { + "epoch": 30.65040650406504, + "grad_norm": 1.3084465265274048, + "learning_rate": 0.0001, + "loss": 70.2963, + "step": 1900 + }, + { + "epoch": 30.8130081300813, + "grad_norm": 1.486464500427246, + "learning_rate": 0.0001, + "loss": 68.6317, + "step": 1910 + }, + { + "epoch": 30.975609756097562, + "grad_norm": 10.415762901306152, + "learning_rate": 0.0001, + "loss": 69.7767, + "step": 1920 + }, + { + "epoch": 31.130081300813007, + "grad_norm": 1.4484038352966309, + "learning_rate": 0.0001, + "loss": 65.6345, + "step": 1930 + }, + { + "epoch": 31.29268292682927, + "grad_norm": 2.297306537628174, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 1940 + }, + { + "epoch": 31.45528455284553, + "grad_norm": 1.223490595817566, + "learning_rate": 0.0001, + "loss": 69.2139, + "step": 1950 + }, + { + "epoch": 31.617886178861788, + "grad_norm": 1.2144125699996948, + "learning_rate": 0.0001, + "loss": 70.3061, + "step": 1960 + }, + { + "epoch": 31.78048780487805, + "grad_norm": 2.633528470993042, + "learning_rate": 0.0001, + "loss": 68.7096, + "step": 1970 + }, + { + "epoch": 31.943089430894307, + "grad_norm": 1.5048350095748901, + "learning_rate": 0.0001, + "loss": 68.6711, + "step": 1980 + }, + { + "epoch": 32.09756097560975, + "grad_norm": 1.5154467821121216, + "learning_rate": 0.0001, + "loss": 66.7989, + "step": 1990 + }, + { + "epoch": 32.260162601626014, + "grad_norm": 2.721277952194214, + "learning_rate": 0.0001, + "loss": 70.4987, + "step": 2000 + }, + { + "epoch": 32.422764227642276, + "grad_norm": 1.44464910030365, + "learning_rate": 0.0001, + "loss": 69.9712, + "step": 2010 + }, + { + "epoch": 32.58536585365854, + "grad_norm": 1.3462835550308228, + "learning_rate": 0.0001, + "loss": 68.9381, + "step": 2020 + }, + { + "epoch": 32.7479674796748, + "grad_norm": 1.3283029794692993, + "learning_rate": 0.0001, + "loss": 68.5301, + "step": 2030 + }, + { + "epoch": 32.91056910569106, + "grad_norm": 1.5172960758209229, + "learning_rate": 0.0001, + "loss": 70.1894, + "step": 2040 + }, + { + "epoch": 33.0650406504065, + "grad_norm": 1.1872576475143433, + "learning_rate": 0.0001, + "loss": 65.0158, + "step": 2050 + }, + { + "epoch": 33.22764227642276, + "grad_norm": 1.5414419174194336, + "learning_rate": 0.0001, + "loss": 69.1537, + "step": 2060 + }, + { + "epoch": 33.390243902439025, + "grad_norm": 1.3143430948257446, + "learning_rate": 0.0001, + "loss": 68.6792, + "step": 2070 + }, + { + "epoch": 33.552845528455286, + "grad_norm": 1.3863723278045654, + "learning_rate": 0.0001, + "loss": 70.3867, + "step": 2080 + }, + { + "epoch": 33.71544715447155, + "grad_norm": 1.4060479402542114, + "learning_rate": 0.0001, + "loss": 68.4981, + "step": 2090 + }, + { + "epoch": 33.8780487804878, + "grad_norm": 1.34132981300354, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 2100 + }, + { + "epoch": 34.03252032520325, + "grad_norm": 1.8303673267364502, + "learning_rate": 0.0001, + "loss": 65.819, + "step": 2110 + }, + { + "epoch": 34.19512195121951, + "grad_norm": 1.387339472770691, + "learning_rate": 0.0001, + "loss": 69.9825, + "step": 2120 + }, + { + "epoch": 34.357723577235774, + "grad_norm": 1.4125195741653442, + "learning_rate": 0.0001, + "loss": 67.5149, + "step": 2130 + }, + { + "epoch": 34.520325203252035, + "grad_norm": 1.2909412384033203, + "learning_rate": 0.0001, + "loss": 69.7848, + "step": 2140 + }, + { + "epoch": 34.68292682926829, + "grad_norm": 1.8860795497894287, + "learning_rate": 0.0001, + "loss": 68.6145, + "step": 2150 + }, + { + "epoch": 34.84552845528455, + "grad_norm": 1.5111079216003418, + "learning_rate": 0.0001, + "loss": 68.2612, + "step": 2160 + }, + { + "epoch": 35.0, + "grad_norm": 1.3275264501571655, + "learning_rate": 0.0001, + "loss": 64.0298, + "step": 2170 + }, + { + "epoch": 35.16260162601626, + "grad_norm": 1.174864411354065, + "learning_rate": 0.0001, + "loss": 66.6153, + "step": 2180 + }, + { + "epoch": 35.32520325203252, + "grad_norm": 1.6873915195465088, + "learning_rate": 0.0001, + "loss": 69.6278, + "step": 2190 + }, + { + "epoch": 35.48780487804878, + "grad_norm": 2.4457507133483887, + "learning_rate": 0.0001, + "loss": 68.3611, + "step": 2200 + }, + { + "epoch": 35.65040650406504, + "grad_norm": 1.4779958724975586, + "learning_rate": 0.0001, + "loss": 69.2609, + "step": 2210 + }, + { + "epoch": 35.8130081300813, + "grad_norm": 2.273709774017334, + "learning_rate": 0.0001, + "loss": 69.1312, + "step": 2220 + }, + { + "epoch": 35.97560975609756, + "grad_norm": 1.5741862058639526, + "learning_rate": 0.0001, + "loss": 69.1191, + "step": 2230 + }, + { + "epoch": 36.13008130081301, + "grad_norm": 1.2730036973953247, + "learning_rate": 0.0001, + "loss": 65.692, + "step": 2240 + }, + { + "epoch": 36.292682926829265, + "grad_norm": 1.2936429977416992, + "learning_rate": 0.0001, + "loss": 68.2962, + "step": 2250 + }, + { + "epoch": 36.45528455284553, + "grad_norm": 1.354279637336731, + "learning_rate": 0.0001, + "loss": 70.5507, + "step": 2260 + }, + { + "epoch": 36.61788617886179, + "grad_norm": 1.4165853261947632, + "learning_rate": 0.0001, + "loss": 68.0048, + "step": 2270 + }, + { + "epoch": 36.78048780487805, + "grad_norm": 1.2992445230484009, + "learning_rate": 0.0001, + "loss": 67.7072, + "step": 2280 + }, + { + "epoch": 36.94308943089431, + "grad_norm": 1.5053461790084839, + "learning_rate": 0.0001, + "loss": 66.7587, + "step": 2290 + }, + { + "epoch": 37.09756097560975, + "grad_norm": 1.337390661239624, + "learning_rate": 0.0001, + "loss": 63.786, + "step": 2300 + }, + { + "epoch": 37.260162601626014, + "grad_norm": 1.0407774448394775, + "learning_rate": 0.0001, + "loss": 66.6253, + "step": 2310 + }, + { + "epoch": 37.422764227642276, + "grad_norm": 1.564152479171753, + "learning_rate": 0.0001, + "loss": 68.3606, + "step": 2320 + }, + { + "epoch": 37.58536585365854, + "grad_norm": 1.2028454542160034, + "learning_rate": 0.0001, + "loss": 66.2841, + "step": 2330 + }, + { + "epoch": 37.7479674796748, + "grad_norm": 1.4528603553771973, + "learning_rate": 0.0001, + "loss": 67.4277, + "step": 2340 + }, + { + "epoch": 37.91056910569106, + "grad_norm": 1.4954335689544678, + "learning_rate": 0.0001, + "loss": 68.9954, + "step": 2350 + }, + { + "epoch": 38.0650406504065, + "grad_norm": 1.2820416688919067, + "learning_rate": 0.0001, + "loss": 65.2449, + "step": 2360 + }, + { + "epoch": 38.22764227642276, + "grad_norm": 1.2392579317092896, + "learning_rate": 0.0001, + "loss": 67.5006, + "step": 2370 + }, + { + "epoch": 38.390243902439025, + "grad_norm": 1.2135238647460938, + "learning_rate": 0.0001, + "loss": 69.2214, + "step": 2380 + }, + { + "epoch": 38.552845528455286, + "grad_norm": 1.5964744091033936, + "learning_rate": 0.0001, + "loss": 67.934, + "step": 2390 + }, + { + "epoch": 38.71544715447155, + "grad_norm": 1.2841702699661255, + "learning_rate": 0.0001, + "loss": 67.0848, + "step": 2400 + }, + { + "epoch": 38.8780487804878, + "grad_norm": 1.8026081323623657, + "learning_rate": 0.0001, + "loss": 67.5358, + "step": 2410 + }, + { + "epoch": 39.03252032520325, + "grad_norm": 1.4178807735443115, + "learning_rate": 0.0001, + "loss": 64.771, + "step": 2420 + }, + { + "epoch": 39.19512195121951, + "grad_norm": 1.3671854734420776, + "learning_rate": 0.0001, + "loss": 68.0531, + "step": 2430 + }, + { + "epoch": 39.357723577235774, + "grad_norm": 1.43596613407135, + "learning_rate": 0.0001, + "loss": 67.7735, + "step": 2440 + }, + { + "epoch": 39.520325203252035, + "grad_norm": 1.464250922203064, + "learning_rate": 0.0001, + "loss": 67.8331, + "step": 2450 + }, + { + "epoch": 39.68292682926829, + "grad_norm": 3.101461410522461, + "learning_rate": 0.0001, + "loss": 65.8956, + "step": 2460 + }, + { + "epoch": 39.84552845528455, + "grad_norm": 1.8332289457321167, + "learning_rate": 0.0001, + "loss": 66.8218, + "step": 2470 + }, + { + "epoch": 40.0, + "grad_norm": 1.2660984992980957, + "learning_rate": 0.0001, + "loss": 63.9969, + "step": 2480 + }, + { + "epoch": 40.16260162601626, + "grad_norm": 2.0309898853302, + "learning_rate": 0.0001, + "loss": 67.3538, + "step": 2490 + }, + { + "epoch": 40.32520325203252, + "grad_norm": 1.207000494003296, + "learning_rate": 0.0001, + "loss": 68.0898, + "step": 2500 + }, + { + "epoch": 40.48780487804878, + "grad_norm": 1.3609018325805664, + "learning_rate": 0.0001, + "loss": 67.0381, + "step": 2510 + }, + { + "epoch": 40.65040650406504, + "grad_norm": 1.310826301574707, + "learning_rate": 0.0001, + "loss": 69.0267, + "step": 2520 + }, + { + "epoch": 40.8130081300813, + "grad_norm": 1.4150960445404053, + "learning_rate": 0.0001, + "loss": 69.1934, + "step": 2530 + }, + { + "epoch": 40.97560975609756, + "grad_norm": 1.5414319038391113, + "learning_rate": 0.0001, + "loss": 68.0604, + "step": 2540 + }, + { + "epoch": 41.13008130081301, + "grad_norm": 2.3246140480041504, + "learning_rate": 0.0001, + "loss": 61.5698, + "step": 2550 + }, + { + "epoch": 41.292682926829265, + "grad_norm": 1.206296443939209, + "learning_rate": 0.0001, + "loss": 64.9898, + "step": 2560 + }, + { + "epoch": 41.45528455284553, + "grad_norm": 1.6080626249313354, + "learning_rate": 0.0001, + "loss": 68.2685, + "step": 2570 + }, + { + "epoch": 41.61788617886179, + "grad_norm": 1.2404112815856934, + "learning_rate": 0.0001, + "loss": 67.8457, + "step": 2580 + }, + { + "epoch": 41.78048780487805, + "grad_norm": 1.2304514646530151, + "learning_rate": 0.0001, + "loss": 67.9052, + "step": 2590 + }, + { + "epoch": 41.94308943089431, + "grad_norm": 1.5806187391281128, + "learning_rate": 0.0001, + "loss": 66.0663, + "step": 2600 + }, + { + "epoch": 42.09756097560975, + "grad_norm": 1.2564151287078857, + "learning_rate": 0.0001, + "loss": 65.7763, + "step": 2610 + }, + { + "epoch": 42.260162601626014, + "grad_norm": 1.4972114562988281, + "learning_rate": 0.0001, + "loss": 66.4269, + "step": 2620 + }, + { + "epoch": 42.422764227642276, + "grad_norm": 1.2913843393325806, + "learning_rate": 0.0001, + "loss": 67.7554, + "step": 2630 + }, + { + "epoch": 42.58536585365854, + "grad_norm": 1.3743863105773926, + "learning_rate": 0.0001, + "loss": 67.2627, + "step": 2640 + }, + { + "epoch": 42.7479674796748, + "grad_norm": 1.1267883777618408, + "learning_rate": 0.0001, + "loss": 67.4597, + "step": 2650 + }, + { + "epoch": 42.91056910569106, + "grad_norm": 2.0194661617279053, + "learning_rate": 0.0001, + "loss": 66.9295, + "step": 2660 + }, + { + "epoch": 43.0650406504065, + "grad_norm": 1.273779273033142, + "learning_rate": 0.0001, + "loss": 60.7906, + "step": 2670 + }, + { + "epoch": 43.22764227642276, + "grad_norm": 1.1498609781265259, + "learning_rate": 0.0001, + "loss": 68.0405, + "step": 2680 + }, + { + "epoch": 43.390243902439025, + "grad_norm": 1.4689130783081055, + "learning_rate": 0.0001, + "loss": 66.2358, + "step": 2690 + }, + { + "epoch": 43.552845528455286, + "grad_norm": 1.4495564699172974, + "learning_rate": 0.0001, + "loss": 67.4267, + "step": 2700 + }, + { + "epoch": 43.71544715447155, + "grad_norm": 2.5769612789154053, + "learning_rate": 0.0001, + "loss": 66.953, + "step": 2710 + }, + { + "epoch": 43.8780487804878, + "grad_norm": 2.1684205532073975, + "learning_rate": 0.0001, + "loss": 65.6393, + "step": 2720 + }, + { + "epoch": 44.03252032520325, + "grad_norm": 1.4968472719192505, + "learning_rate": 0.0001, + "loss": 60.8256, + "step": 2730 + }, + { + "epoch": 44.19512195121951, + "grad_norm": 1.1874353885650635, + "learning_rate": 0.0001, + "loss": 66.818, + "step": 2740 + }, + { + "epoch": 44.357723577235774, + "grad_norm": 2.271235227584839, + "learning_rate": 0.0001, + "loss": 66.5048, + "step": 2750 + }, + { + "epoch": 44.520325203252035, + "grad_norm": 1.20489501953125, + "learning_rate": 0.0001, + "loss": 67.0487, + "step": 2760 + }, + { + "epoch": 44.68292682926829, + "grad_norm": 1.6273691654205322, + "learning_rate": 0.0001, + "loss": 66.2228, + "step": 2770 + }, + { + "epoch": 44.84552845528455, + "grad_norm": 1.4935433864593506, + "learning_rate": 0.0001, + "loss": 67.2039, + "step": 2780 + }, + { + "epoch": 45.0, + "grad_norm": 1.2668659687042236, + "learning_rate": 0.0001, + "loss": 62.7855, + "step": 2790 + }, + { + "epoch": 45.16260162601626, + "grad_norm": 1.6439573764801025, + "learning_rate": 0.0001, + "loss": 67.919, + "step": 2800 + }, + { + "epoch": 45.32520325203252, + "grad_norm": 1.311854600906372, + "learning_rate": 0.0001, + "loss": 65.1797, + "step": 2810 + }, + { + "epoch": 45.48780487804878, + "grad_norm": 1.3596776723861694, + "learning_rate": 0.0001, + "loss": 67.469, + "step": 2820 + }, + { + "epoch": 45.65040650406504, + "grad_norm": 1.5618443489074707, + "learning_rate": 0.0001, + "loss": 66.1808, + "step": 2830 + }, + { + "epoch": 45.8130081300813, + "grad_norm": 1.7551205158233643, + "learning_rate": 0.0001, + "loss": 65.3803, + "step": 2840 + }, + { + "epoch": 45.97560975609756, + "grad_norm": 1.433794379234314, + "learning_rate": 0.0001, + "loss": 67.8314, + "step": 2850 + }, + { + "epoch": 46.13008130081301, + "grad_norm": 1.4634851217269897, + "learning_rate": 0.0001, + "loss": 63.4219, + "step": 2860 + }, + { + "epoch": 46.292682926829265, + "grad_norm": 1.083052396774292, + "learning_rate": 0.0001, + "loss": 68.0719, + "step": 2870 + }, + { + "epoch": 46.45528455284553, + "grad_norm": 1.1784703731536865, + "learning_rate": 0.0001, + "loss": 66.1391, + "step": 2880 + }, + { + "epoch": 46.61788617886179, + "grad_norm": 1.2552465200424194, + "learning_rate": 0.0001, + "loss": 65.6006, + "step": 2890 + }, + { + "epoch": 46.78048780487805, + "grad_norm": 1.4684522151947021, + "learning_rate": 0.0001, + "loss": 66.8314, + "step": 2900 + }, + { + "epoch": 46.94308943089431, + "grad_norm": 1.741514801979065, + "learning_rate": 0.0001, + "loss": 67.5056, + "step": 2910 + }, + { + "epoch": 47.09756097560975, + "grad_norm": 5.696534156799316, + "learning_rate": 0.0001, + "loss": 62.8016, + "step": 2920 + }, + { + "epoch": 47.260162601626014, + "grad_norm": 1.5000969171524048, + "learning_rate": 0.0001, + "loss": 68.0272, + "step": 2930 + }, + { + "epoch": 47.422764227642276, + "grad_norm": 1.1044933795928955, + "learning_rate": 0.0001, + "loss": 67.0905, + "step": 2940 + }, + { + "epoch": 47.58536585365854, + "grad_norm": 0.9933589100837708, + "learning_rate": 0.0001, + "loss": 67.6077, + "step": 2950 + }, + { + "epoch": 47.7479674796748, + "grad_norm": 1.2908599376678467, + "learning_rate": 0.0001, + "loss": 65.869, + "step": 2960 + }, + { + "epoch": 47.91056910569106, + "grad_norm": 1.0470529794692993, + "learning_rate": 0.0001, + "loss": 66.5353, + "step": 2970 + }, + { + "epoch": 48.0650406504065, + "grad_norm": 1.0927976369857788, + "learning_rate": 0.0001, + "loss": 62.7541, + "step": 2980 + }, + { + "epoch": 48.22764227642276, + "grad_norm": 1.0870513916015625, + "learning_rate": 0.0001, + "loss": 64.172, + "step": 2990 + }, + { + "epoch": 48.390243902439025, + "grad_norm": 1.3492090702056885, + "learning_rate": 0.0001, + "loss": 67.6642, + "step": 3000 + }, + { + "epoch": 48.552845528455286, + "grad_norm": 1.4024657011032104, + "learning_rate": 0.0001, + "loss": 65.2722, + "step": 3010 + }, + { + "epoch": 48.71544715447155, + "grad_norm": 1.1165252923965454, + "learning_rate": 0.0001, + "loss": 66.7233, + "step": 3020 + }, + { + "epoch": 48.8780487804878, + "grad_norm": 1.2147928476333618, + "learning_rate": 0.0001, + "loss": 65.7277, + "step": 3030 + }, + { + "epoch": 49.03252032520325, + "grad_norm": 0.925310492515564, + "learning_rate": 0.0001, + "loss": 62.9256, + "step": 3040 + }, + { + "epoch": 49.19512195121951, + "grad_norm": 1.2353442907333374, + "learning_rate": 0.0001, + "loss": 66.1218, + "step": 3050 + }, + { + "epoch": 49.357723577235774, + "grad_norm": 1.2177367210388184, + "learning_rate": 0.0001, + "loss": 66.7978, + "step": 3060 + }, + { + "epoch": 49.520325203252035, + "grad_norm": 1.248044729232788, + "learning_rate": 0.0001, + "loss": 66.2132, + "step": 3070 + }, + { + "epoch": 49.68292682926829, + "grad_norm": 1.366050124168396, + "learning_rate": 0.0001, + "loss": 67.0118, + "step": 3080 + }, + { + "epoch": 49.84552845528455, + "grad_norm": 1.3701151609420776, + "learning_rate": 0.0001, + "loss": 67.6099, + "step": 3090 + }, + { + "epoch": 50.0, + "grad_norm": 1.0420141220092773, + "learning_rate": 0.0001, + "loss": 61.5289, + "step": 3100 + }, + { + "epoch": 50.16260162601626, + "grad_norm": 1.0957746505737305, + "learning_rate": 0.0001, + "loss": 66.7348, + "step": 3110 + }, + { + "epoch": 50.32520325203252, + "grad_norm": 1.108628749847412, + "learning_rate": 0.0001, + "loss": 66.6199, + "step": 3120 + }, + { + "epoch": 50.48780487804878, + "grad_norm": 1.3680156469345093, + "learning_rate": 0.0001, + "loss": 67.0825, + "step": 3130 + }, + { + "epoch": 50.65040650406504, + "grad_norm": 1.169851303100586, + "learning_rate": 0.0001, + "loss": 65.4901, + "step": 3140 + }, + { + "epoch": 50.8130081300813, + "grad_norm": 2.1822028160095215, + "learning_rate": 0.0001, + "loss": 65.6841, + "step": 3150 + }, + { + "epoch": 50.97560975609756, + "grad_norm": 1.053402304649353, + "learning_rate": 0.0001, + "loss": 66.3119, + "step": 3160 + }, + { + "epoch": 51.13008130081301, + "grad_norm": 1.0627789497375488, + "learning_rate": 0.0001, + "loss": 61.3642, + "step": 3170 + }, + { + "epoch": 51.292682926829265, + "grad_norm": 1.926291823387146, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 3180 + }, + { + "epoch": 51.45528455284553, + "grad_norm": 1.5791983604431152, + "learning_rate": 0.0001, + "loss": 63.913, + "step": 3190 + }, + { + "epoch": 51.61788617886179, + "grad_norm": 1.210265040397644, + "learning_rate": 0.0001, + "loss": 64.5261, + "step": 3200 + }, + { + "epoch": 51.78048780487805, + "grad_norm": 1.0720808506011963, + "learning_rate": 0.0001, + "loss": 67.0194, + "step": 3210 + }, + { + "epoch": 51.94308943089431, + "grad_norm": 1.94563889503479, + "learning_rate": 0.0001, + "loss": 68.4662, + "step": 3220 + }, + { + "epoch": 52.09756097560975, + "grad_norm": 1.598980188369751, + "learning_rate": 0.0001, + "loss": 62.4946, + "step": 3230 + }, + { + "epoch": 52.260162601626014, + "grad_norm": 1.0564576387405396, + "learning_rate": 0.0001, + "loss": 65.7912, + "step": 3240 + }, + { + "epoch": 52.422764227642276, + "grad_norm": 1.2189645767211914, + "learning_rate": 0.0001, + "loss": 65.5772, + "step": 3250 + }, + { + "epoch": 52.58536585365854, + "grad_norm": 1.1357049942016602, + "learning_rate": 0.0001, + "loss": 65.1559, + "step": 3260 + }, + { + "epoch": 52.7479674796748, + "grad_norm": 1.316867709159851, + "learning_rate": 0.0001, + "loss": 64.7897, + "step": 3270 + }, + { + "epoch": 52.91056910569106, + "grad_norm": 1.3870197534561157, + "learning_rate": 0.0001, + "loss": 65.5195, + "step": 3280 + }, + { + "epoch": 53.0650406504065, + "grad_norm": 1.2467927932739258, + "learning_rate": 0.0001, + "loss": 61.3392, + "step": 3290 + }, + { + "epoch": 53.22764227642276, + "grad_norm": 1.1661109924316406, + "learning_rate": 0.0001, + "loss": 66.0117, + "step": 3300 + }, + { + "epoch": 53.390243902439025, + "grad_norm": 1.3509628772735596, + "learning_rate": 0.0001, + "loss": 64.7228, + "step": 3310 + }, + { + "epoch": 53.552845528455286, + "grad_norm": 1.1093642711639404, + "learning_rate": 0.0001, + "loss": 65.2534, + "step": 3320 + }, + { + "epoch": 53.71544715447155, + "grad_norm": 1.1801207065582275, + "learning_rate": 0.0001, + "loss": 65.7097, + "step": 3330 + }, + { + "epoch": 53.8780487804878, + "grad_norm": 1.175884485244751, + "learning_rate": 0.0001, + "loss": 64.4368, + "step": 3340 + }, + { + "epoch": 54.03252032520325, + "grad_norm": 1.1727889776229858, + "learning_rate": 0.0001, + "loss": 62.9373, + "step": 3350 + }, + { + "epoch": 54.19512195121951, + "grad_norm": 1.5330945253372192, + "learning_rate": 0.0001, + "loss": 66.7763, + "step": 3360 + }, + { + "epoch": 54.357723577235774, + "grad_norm": 1.2992504835128784, + "learning_rate": 0.0001, + "loss": 65.7854, + "step": 3370 + }, + { + "epoch": 54.520325203252035, + "grad_norm": 1.0185420513153076, + "learning_rate": 0.0001, + "loss": 65.7046, + "step": 3380 + }, + { + "epoch": 54.68292682926829, + "grad_norm": 1.426229476928711, + "learning_rate": 0.0001, + "loss": 64.0286, + "step": 3390 + }, + { + "epoch": 54.84552845528455, + "grad_norm": 1.2600868940353394, + "learning_rate": 0.0001, + "loss": 66.1738, + "step": 3400 + }, + { + "epoch": 55.0, + "grad_norm": 1.917455792427063, + "learning_rate": 0.0001, + "loss": 61.9381, + "step": 3410 + }, + { + "epoch": 55.16260162601626, + "grad_norm": 1.2000917196273804, + "learning_rate": 0.0001, + "loss": 65.5626, + "step": 3420 + }, + { + "epoch": 55.32520325203252, + "grad_norm": 1.362176775932312, + "learning_rate": 0.0001, + "loss": 66.8012, + "step": 3430 + }, + { + "epoch": 55.48780487804878, + "grad_norm": 1.1611402034759521, + "learning_rate": 0.0001, + "loss": 64.5227, + "step": 3440 + }, + { + "epoch": 55.65040650406504, + "grad_norm": 1.2066621780395508, + "learning_rate": 0.0001, + "loss": 63.4351, + "step": 3450 + }, + { + "epoch": 55.8130081300813, + "grad_norm": 1.005340814590454, + "learning_rate": 0.0001, + "loss": 65.9261, + "step": 3460 + }, + { + "epoch": 55.97560975609756, + "grad_norm": 1.0906637907028198, + "learning_rate": 0.0001, + "loss": 64.6635, + "step": 3470 + }, + { + "epoch": 56.13008130081301, + "grad_norm": 1.0608748197555542, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 3480 + }, + { + "epoch": 56.292682926829265, + "grad_norm": 1.025319218635559, + "learning_rate": 0.0001, + "loss": 64.7783, + "step": 3490 + }, + { + "epoch": 56.45528455284553, + "grad_norm": 1.1596784591674805, + "learning_rate": 0.0001, + "loss": 66.4629, + "step": 3500 + }, + { + "epoch": 56.61788617886179, + "grad_norm": 0.9318640232086182, + "learning_rate": 0.0001, + "loss": 65.5533, + "step": 3510 + }, + { + "epoch": 56.78048780487805, + "grad_norm": 1.0977418422698975, + "learning_rate": 0.0001, + "loss": 63.647, + "step": 3520 + }, + { + "epoch": 56.94308943089431, + "grad_norm": 1.3912436962127686, + "learning_rate": 0.0001, + "loss": 62.8273, + "step": 3530 + }, + { + "epoch": 57.09756097560975, + "grad_norm": 1.0921591520309448, + "learning_rate": 0.0001, + "loss": 61.5968, + "step": 3540 + }, + { + "epoch": 57.260162601626014, + "grad_norm": 1.17819344997406, + "learning_rate": 0.0001, + "loss": 64.9381, + "step": 3550 + }, + { + "epoch": 57.422764227642276, + "grad_norm": 1.188829779624939, + "learning_rate": 0.0001, + "loss": 64.6498, + "step": 3560 + }, + { + "epoch": 57.58536585365854, + "grad_norm": 3.664839506149292, + "learning_rate": 0.0001, + "loss": 65.3834, + "step": 3570 + }, + { + "epoch": 57.7479674796748, + "grad_norm": 1.57958984375, + "learning_rate": 0.0001, + "loss": 63.8808, + "step": 3580 + }, + { + "epoch": 57.91056910569106, + "grad_norm": 1.1837552785873413, + "learning_rate": 0.0001, + "loss": 64.1545, + "step": 3590 + }, + { + "epoch": 58.0650406504065, + "grad_norm": 1.5986013412475586, + "learning_rate": 0.0001, + "loss": 60.5742, + "step": 3600 + }, + { + "epoch": 58.22764227642276, + "grad_norm": 1.1551810503005981, + "learning_rate": 0.0001, + "loss": 66.2378, + "step": 3610 + }, + { + "epoch": 58.390243902439025, + "grad_norm": 1.0869863033294678, + "learning_rate": 0.0001, + "loss": 65.3034, + "step": 3620 + }, + { + "epoch": 58.552845528455286, + "grad_norm": 1.2263212203979492, + "learning_rate": 0.0001, + "loss": 63.976, + "step": 3630 + }, + { + "epoch": 58.71544715447155, + "grad_norm": 1.4410817623138428, + "learning_rate": 0.0001, + "loss": 63.4778, + "step": 3640 + }, + { + "epoch": 58.8780487804878, + "grad_norm": 1.9633759260177612, + "learning_rate": 0.0001, + "loss": 65.5946, + "step": 3650 + }, + { + "epoch": 59.03252032520325, + "grad_norm": 1.1729766130447388, + "learning_rate": 0.0001, + "loss": 62.2171, + "step": 3660 + }, + { + "epoch": 59.19512195121951, + "grad_norm": 1.0049556493759155, + "learning_rate": 0.0001, + "loss": 65.1854, + "step": 3670 + }, + { + "epoch": 59.357723577235774, + "grad_norm": 1.1727055311203003, + "learning_rate": 0.0001, + "loss": 65.4839, + "step": 3680 + }, + { + "epoch": 59.520325203252035, + "grad_norm": 1.5394458770751953, + "learning_rate": 0.0001, + "loss": 64.9341, + "step": 3690 + }, + { + "epoch": 59.68292682926829, + "grad_norm": 1.1583774089813232, + "learning_rate": 0.0001, + "loss": 63.9197, + "step": 3700 + }, + { + "epoch": 59.84552845528455, + "grad_norm": 1.0835607051849365, + "learning_rate": 0.0001, + "loss": 64.0494, + "step": 3710 + }, + { + "epoch": 60.0, + "grad_norm": 1.7504246234893799, + "learning_rate": 0.0001, + "loss": 59.4134, + "step": 3720 + }, + { + "epoch": 60.16260162601626, + "grad_norm": 1.1870343685150146, + "learning_rate": 0.0001, + "loss": 63.2158, + "step": 3730 + }, + { + "epoch": 60.32520325203252, + "grad_norm": 1.1983181238174438, + "learning_rate": 0.0001, + "loss": 64.7144, + "step": 3740 + }, + { + "epoch": 60.48780487804878, + "grad_norm": 1.3014581203460693, + "learning_rate": 0.0001, + "loss": 64.6219, + "step": 3750 + }, + { + "epoch": 60.65040650406504, + "grad_norm": 1.4954087734222412, + "learning_rate": 0.0001, + "loss": 63.3785, + "step": 3760 + }, + { + "epoch": 60.8130081300813, + "grad_norm": 1.2236649990081787, + "learning_rate": 0.0001, + "loss": 64.6933, + "step": 3770 + }, + { + "epoch": 60.97560975609756, + "grad_norm": 1.0796610116958618, + "learning_rate": 0.0001, + "loss": 63.2704, + "step": 3780 + }, + { + "epoch": 61.13008130081301, + "grad_norm": 0.9969968795776367, + "learning_rate": 0.0001, + "loss": 62.1838, + "step": 3790 + }, + { + "epoch": 61.292682926829265, + "grad_norm": 1.1624361276626587, + "learning_rate": 0.0001, + "loss": 63.6249, + "step": 3800 + }, + { + "epoch": 61.45528455284553, + "grad_norm": 1.3849223852157593, + "learning_rate": 0.0001, + "loss": 64.5321, + "step": 3810 + }, + { + "epoch": 61.61788617886179, + "grad_norm": 1.6607458591461182, + "learning_rate": 0.0001, + "loss": 64.651, + "step": 3820 + }, + { + "epoch": 61.78048780487805, + "grad_norm": 1.09684419631958, + "learning_rate": 0.0001, + "loss": 64.1129, + "step": 3830 + }, + { + "epoch": 61.94308943089431, + "grad_norm": 1.406246304512024, + "learning_rate": 0.0001, + "loss": 64.3642, + "step": 3840 + }, + { + "epoch": 62.09756097560975, + "grad_norm": 0.9850211143493652, + "learning_rate": 0.0001, + "loss": 60.8703, + "step": 3850 + }, + { + "epoch": 62.260162601626014, + "grad_norm": 1.001344084739685, + "learning_rate": 0.0001, + "loss": 67.5523, + "step": 3860 + }, + { + "epoch": 62.422764227642276, + "grad_norm": 2.435969591140747, + "learning_rate": 0.0001, + "loss": 62.7363, + "step": 3870 + }, + { + "epoch": 62.58536585365854, + "grad_norm": 1.0675824880599976, + "learning_rate": 0.0001, + "loss": 64.8513, + "step": 3880 + }, + { + "epoch": 62.7479674796748, + "grad_norm": 0.9793754816055298, + "learning_rate": 0.0001, + "loss": 64.9904, + "step": 3890 + }, + { + "epoch": 62.91056910569106, + "grad_norm": 1.2187775373458862, + "learning_rate": 0.0001, + "loss": 63.2821, + "step": 3900 + }, + { + "epoch": 63.0650406504065, + "grad_norm": 1.130772590637207, + "learning_rate": 0.0001, + "loss": 61.5957, + "step": 3910 + }, + { + "epoch": 63.22764227642276, + "grad_norm": 1.0573055744171143, + "learning_rate": 0.0001, + "loss": 63.6337, + "step": 3920 + }, + { + "epoch": 63.390243902439025, + "grad_norm": 2.055490255355835, + "learning_rate": 0.0001, + "loss": 63.9598, + "step": 3930 + }, + { + "epoch": 63.552845528455286, + "grad_norm": 1.0780516862869263, + "learning_rate": 0.0001, + "loss": 62.9303, + "step": 3940 + }, + { + "epoch": 63.71544715447155, + "grad_norm": 1.8136590719223022, + "learning_rate": 0.0001, + "loss": 64.4588, + "step": 3950 + }, + { + "epoch": 63.8780487804878, + "grad_norm": 1.0525016784667969, + "learning_rate": 0.0001, + "loss": 64.192, + "step": 3960 + }, + { + "epoch": 64.03252032520325, + "grad_norm": 1.12627375125885, + "learning_rate": 0.0001, + "loss": 59.1823, + "step": 3970 + }, + { + "epoch": 64.1951219512195, + "grad_norm": 1.2558730840682983, + "learning_rate": 0.0001, + "loss": 65.126, + "step": 3980 + }, + { + "epoch": 64.35772357723577, + "grad_norm": 1.2304726839065552, + "learning_rate": 0.0001, + "loss": 62.0868, + "step": 3990 + }, + { + "epoch": 64.52032520325203, + "grad_norm": 1.4179201126098633, + "learning_rate": 0.0001, + "loss": 62.3114, + "step": 4000 + }, + { + "epoch": 64.6829268292683, + "grad_norm": 1.125758409500122, + "learning_rate": 0.0001, + "loss": 63.4103, + "step": 4010 + }, + { + "epoch": 64.84552845528455, + "grad_norm": 2.321124315261841, + "learning_rate": 0.0001, + "loss": 62.9032, + "step": 4020 + }, + { + "epoch": 65.0, + "grad_norm": 1.4433993101119995, + "learning_rate": 0.0001, + "loss": 61.8766, + "step": 4030 + }, + { + "epoch": 65.16260162601625, + "grad_norm": 1.157878041267395, + "learning_rate": 0.0001, + "loss": 62.4833, + "step": 4040 + }, + { + "epoch": 65.32520325203252, + "grad_norm": 1.4127166271209717, + "learning_rate": 0.0001, + "loss": 64.768, + "step": 4050 + }, + { + "epoch": 65.48780487804878, + "grad_norm": 1.2575284242630005, + "learning_rate": 0.0001, + "loss": 62.9401, + "step": 4060 + }, + { + "epoch": 65.65040650406505, + "grad_norm": 1.0957911014556885, + "learning_rate": 0.0001, + "loss": 64.6061, + "step": 4070 + }, + { + "epoch": 65.8130081300813, + "grad_norm": 1.1773414611816406, + "learning_rate": 0.0001, + "loss": 63.9469, + "step": 4080 + }, + { + "epoch": 65.97560975609755, + "grad_norm": 2.1189942359924316, + "learning_rate": 0.0001, + "loss": 65.1422, + "step": 4090 + }, + { + "epoch": 66.130081300813, + "grad_norm": 1.1064234972000122, + "learning_rate": 0.0001, + "loss": 59.6108, + "step": 4100 + }, + { + "epoch": 66.29268292682927, + "grad_norm": 1.1135871410369873, + "learning_rate": 0.0001, + "loss": 63.4036, + "step": 4110 + }, + { + "epoch": 66.45528455284553, + "grad_norm": 4.167781829833984, + "learning_rate": 0.0001, + "loss": 63.1174, + "step": 4120 + }, + { + "epoch": 66.6178861788618, + "grad_norm": 1.1429376602172852, + "learning_rate": 0.0001, + "loss": 64.4085, + "step": 4130 + }, + { + "epoch": 66.78048780487805, + "grad_norm": 1.3188339471817017, + "learning_rate": 0.0001, + "loss": 62.5997, + "step": 4140 + }, + { + "epoch": 66.9430894308943, + "grad_norm": 1.160335659980774, + "learning_rate": 0.0001, + "loss": 64.2911, + "step": 4150 + }, + { + "epoch": 67.09756097560975, + "grad_norm": 0.9261873960494995, + "learning_rate": 0.0001, + "loss": 61.9036, + "step": 4160 + }, + { + "epoch": 67.26016260162602, + "grad_norm": 1.2254258394241333, + "learning_rate": 0.0001, + "loss": 64.1092, + "step": 4170 + }, + { + "epoch": 67.42276422764228, + "grad_norm": 0.9519097208976746, + "learning_rate": 0.0001, + "loss": 61.6428, + "step": 4180 + }, + { + "epoch": 67.58536585365853, + "grad_norm": 1.2290887832641602, + "learning_rate": 0.0001, + "loss": 64.9842, + "step": 4190 + }, + { + "epoch": 67.7479674796748, + "grad_norm": 1.0995723009109497, + "learning_rate": 0.0001, + "loss": 64.0677, + "step": 4200 + }, + { + "epoch": 67.91056910569105, + "grad_norm": 2.727957010269165, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 4210 + }, + { + "epoch": 68.0650406504065, + "grad_norm": 1.1222411394119263, + "learning_rate": 0.0001, + "loss": 61.0693, + "step": 4220 + }, + { + "epoch": 68.22764227642277, + "grad_norm": 1.159303069114685, + "learning_rate": 0.0001, + "loss": 64.2469, + "step": 4230 + }, + { + "epoch": 68.39024390243902, + "grad_norm": 1.2205990552902222, + "learning_rate": 0.0001, + "loss": 63.8831, + "step": 4240 + }, + { + "epoch": 68.55284552845528, + "grad_norm": 1.2351205348968506, + "learning_rate": 0.0001, + "loss": 63.4294, + "step": 4250 + }, + { + "epoch": 68.71544715447155, + "grad_norm": 1.4483757019042969, + "learning_rate": 0.0001, + "loss": 63.5345, + "step": 4260 + }, + { + "epoch": 68.8780487804878, + "grad_norm": 1.0808624029159546, + "learning_rate": 0.0001, + "loss": 63.6885, + "step": 4270 + }, + { + "epoch": 69.03252032520325, + "grad_norm": 1.7775999307632446, + "learning_rate": 0.0001, + "loss": 60.7266, + "step": 4280 + }, + { + "epoch": 69.1951219512195, + "grad_norm": 1.3214768171310425, + "learning_rate": 0.0001, + "loss": 63.3496, + "step": 4290 + }, + { + "epoch": 69.35772357723577, + "grad_norm": 1.1698002815246582, + "learning_rate": 0.0001, + "loss": 63.5284, + "step": 4300 + }, + { + "epoch": 69.52032520325203, + "grad_norm": 1.2050056457519531, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 4310 + }, + { + "epoch": 69.6829268292683, + "grad_norm": 1.0873243808746338, + "learning_rate": 0.0001, + "loss": 65.1709, + "step": 4320 + }, + { + "epoch": 69.84552845528455, + "grad_norm": 1.14133620262146, + "learning_rate": 0.0001, + "loss": 63.1872, + "step": 4330 + }, + { + "epoch": 70.0, + "grad_norm": 0.9684088230133057, + "learning_rate": 0.0001, + "loss": 60.1486, + "step": 4340 + }, + { + "epoch": 70.16260162601625, + "grad_norm": 1.1206653118133545, + "learning_rate": 0.0001, + "loss": 61.117, + "step": 4350 + }, + { + "epoch": 70.32520325203252, + "grad_norm": 1.2754334211349487, + "learning_rate": 0.0001, + "loss": 62.5409, + "step": 4360 + }, + { + "epoch": 70.48780487804878, + "grad_norm": 1.0154718160629272, + "learning_rate": 0.0001, + "loss": 62.9428, + "step": 4370 + }, + { + "epoch": 70.65040650406505, + "grad_norm": 1.0597270727157593, + "learning_rate": 0.0001, + "loss": 63.079, + "step": 4380 + }, + { + "epoch": 70.8130081300813, + "grad_norm": 1.254758596420288, + "learning_rate": 0.0001, + "loss": 63.3035, + "step": 4390 + }, + { + "epoch": 70.97560975609755, + "grad_norm": 1.0185120105743408, + "learning_rate": 0.0001, + "loss": 63.8922, + "step": 4400 + }, + { + "epoch": 71.130081300813, + "grad_norm": 1.007300853729248, + "learning_rate": 0.0001, + "loss": 58.7735, + "step": 4410 + }, + { + "epoch": 71.29268292682927, + "grad_norm": 1.2012463808059692, + "learning_rate": 0.0001, + "loss": 63.1782, + "step": 4420 + }, + { + "epoch": 71.45528455284553, + "grad_norm": 1.0258845090866089, + "learning_rate": 0.0001, + "loss": 63.2483, + "step": 4430 + }, + { + "epoch": 71.6178861788618, + "grad_norm": 1.0611220598220825, + "learning_rate": 0.0001, + "loss": 63.2247, + "step": 4440 + }, + { + "epoch": 71.78048780487805, + "grad_norm": 1.060114860534668, + "learning_rate": 0.0001, + "loss": 61.8352, + "step": 4450 + }, + { + "epoch": 71.9430894308943, + "grad_norm": 1.0540589094161987, + "learning_rate": 0.0001, + "loss": 62.1156, + "step": 4460 + }, + { + "epoch": 72.09756097560975, + "grad_norm": 1.4923726320266724, + "learning_rate": 0.0001, + "loss": 59.3863, + "step": 4470 + }, + { + "epoch": 72.26016260162602, + "grad_norm": 1.13791024684906, + "learning_rate": 0.0001, + "loss": 62.203, + "step": 4480 + }, + { + "epoch": 72.42276422764228, + "grad_norm": 0.9737065434455872, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 4490 + }, + { + "epoch": 72.58536585365853, + "grad_norm": 1.3980190753936768, + "learning_rate": 0.0001, + "loss": 63.5627, + "step": 4500 + }, + { + "epoch": 72.7479674796748, + "grad_norm": 1.0274118185043335, + "learning_rate": 0.0001, + "loss": 63.6129, + "step": 4510 + }, + { + "epoch": 72.91056910569105, + "grad_norm": 1.3148925304412842, + "learning_rate": 0.0001, + "loss": 63.6506, + "step": 4520 + }, + { + "epoch": 73.0650406504065, + "grad_norm": 1.4182451963424683, + "learning_rate": 0.0001, + "loss": 60.3616, + "step": 4530 + }, + { + "epoch": 73.22764227642277, + "grad_norm": 0.9235166907310486, + "learning_rate": 0.0001, + "loss": 63.6465, + "step": 4540 + }, + { + "epoch": 73.39024390243902, + "grad_norm": 0.9293469786643982, + "learning_rate": 0.0001, + "loss": 61.0984, + "step": 4550 + }, + { + "epoch": 73.55284552845528, + "grad_norm": 1.1075892448425293, + "learning_rate": 0.0001, + "loss": 62.9356, + "step": 4560 + }, + { + "epoch": 73.71544715447155, + "grad_norm": 1.4955459833145142, + "learning_rate": 0.0001, + "loss": 61.3811, + "step": 4570 + }, + { + "epoch": 73.8780487804878, + "grad_norm": 1.220595359802246, + "learning_rate": 0.0001, + "loss": 62.4364, + "step": 4580 + }, + { + "epoch": 74.03252032520325, + "grad_norm": 0.9975704550743103, + "learning_rate": 0.0001, + "loss": 58.5194, + "step": 4590 + }, + { + "epoch": 74.1951219512195, + "grad_norm": 1.0160857439041138, + "learning_rate": 0.0001, + "loss": 64.6989, + "step": 4600 + }, + { + "epoch": 74.35772357723577, + "grad_norm": 1.0052287578582764, + "learning_rate": 0.0001, + "loss": 62.526, + "step": 4610 + }, + { + "epoch": 74.52032520325203, + "grad_norm": 1.015514612197876, + "learning_rate": 0.0001, + "loss": 63.8627, + "step": 4620 + }, + { + "epoch": 74.6829268292683, + "grad_norm": 1.139051914215088, + "learning_rate": 0.0001, + "loss": 62.4525, + "step": 4630 + }, + { + "epoch": 74.84552845528455, + "grad_norm": 1.1727714538574219, + "learning_rate": 0.0001, + "loss": 64.0046, + "step": 4640 + }, + { + "epoch": 75.0, + "grad_norm": 1.1611062288284302, + "learning_rate": 0.0001, + "loss": 57.6443, + "step": 4650 + }, + { + "epoch": 75.16260162601625, + "grad_norm": 0.9893108606338501, + "learning_rate": 0.0001, + "loss": 64.1109, + "step": 4660 + }, + { + "epoch": 75.32520325203252, + "grad_norm": 1.190146803855896, + "learning_rate": 0.0001, + "loss": 62.1988, + "step": 4670 + }, + { + "epoch": 75.48780487804878, + "grad_norm": 1.1648579835891724, + "learning_rate": 0.0001, + "loss": 63.1393, + "step": 4680 + }, + { + "epoch": 75.65040650406505, + "grad_norm": 1.3303166627883911, + "learning_rate": 0.0001, + "loss": 63.7, + "step": 4690 + }, + { + "epoch": 75.8130081300813, + "grad_norm": 1.407772421836853, + "learning_rate": 0.0001, + "loss": 60.6798, + "step": 4700 + }, + { + "epoch": 75.97560975609755, + "grad_norm": 1.039741039276123, + "learning_rate": 0.0001, + "loss": 61.6126, + "step": 4710 + }, + { + "epoch": 76.130081300813, + "grad_norm": 1.013515830039978, + "learning_rate": 0.0001, + "loss": 57.8452, + "step": 4720 + }, + { + "epoch": 76.29268292682927, + "grad_norm": 1.1279789209365845, + "learning_rate": 0.0001, + "loss": 63.0592, + "step": 4730 + }, + { + "epoch": 76.45528455284553, + "grad_norm": 1.158605694770813, + "learning_rate": 0.0001, + "loss": 62.0836, + "step": 4740 + }, + { + "epoch": 76.6178861788618, + "grad_norm": 1.0200021266937256, + "learning_rate": 0.0001, + "loss": 62.1775, + "step": 4750 + }, + { + "epoch": 76.78048780487805, + "grad_norm": 1.216412901878357, + "learning_rate": 0.0001, + "loss": 62.6316, + "step": 4760 + }, + { + "epoch": 76.9430894308943, + "grad_norm": 1.0513523817062378, + "learning_rate": 0.0001, + "loss": 60.1037, + "step": 4770 + }, + { + "epoch": 77.09756097560975, + "grad_norm": 1.10075843334198, + "learning_rate": 0.0001, + "loss": 58.7599, + "step": 4780 + }, + { + "epoch": 77.26016260162602, + "grad_norm": 1.1728724241256714, + "learning_rate": 0.0001, + "loss": 63.3476, + "step": 4790 + }, + { + "epoch": 77.42276422764228, + "grad_norm": 1.3511605262756348, + "learning_rate": 0.0001, + "loss": 60.9885, + "step": 4800 + }, + { + "epoch": 77.58536585365853, + "grad_norm": 0.9733042120933533, + "learning_rate": 0.0001, + "loss": 62.4245, + "step": 4810 + }, + { + "epoch": 77.7479674796748, + "grad_norm": 1.0380535125732422, + "learning_rate": 0.0001, + "loss": 64.0755, + "step": 4820 + }, + { + "epoch": 77.91056910569105, + "grad_norm": 1.1594985723495483, + "learning_rate": 0.0001, + "loss": 62.1099, + "step": 4830 + }, + { + "epoch": 78.0650406504065, + "grad_norm": 1.0392569303512573, + "learning_rate": 0.0001, + "loss": 58.3778, + "step": 4840 + }, + { + "epoch": 78.22764227642277, + "grad_norm": 1.0032405853271484, + "learning_rate": 0.0001, + "loss": 62.1384, + "step": 4850 + }, + { + "epoch": 78.39024390243902, + "grad_norm": 1.0576013326644897, + "learning_rate": 0.0001, + "loss": 63.3308, + "step": 4860 + }, + { + "epoch": 78.55284552845528, + "grad_norm": 1.34328031539917, + "learning_rate": 0.0001, + "loss": 62.5351, + "step": 4870 + }, + { + "epoch": 78.71544715447155, + "grad_norm": 1.1040822267532349, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 4880 + }, + { + "epoch": 78.8780487804878, + "grad_norm": 1.1554768085479736, + "learning_rate": 0.0001, + "loss": 62.7308, + "step": 4890 + }, + { + "epoch": 79.03252032520325, + "grad_norm": 1.1864408254623413, + "learning_rate": 0.0001, + "loss": 60.0741, + "step": 4900 + }, + { + "epoch": 79.1951219512195, + "grad_norm": 0.9181821346282959, + "learning_rate": 0.0001, + "loss": 63.1682, + "step": 4910 + }, + { + "epoch": 79.35772357723577, + "grad_norm": 1.1400946378707886, + "learning_rate": 0.0001, + "loss": 62.0736, + "step": 4920 + }, + { + "epoch": 79.52032520325203, + "grad_norm": 1.245202660560608, + "learning_rate": 0.0001, + "loss": 60.51, + "step": 4930 + }, + { + "epoch": 79.6829268292683, + "grad_norm": 1.033347725868225, + "learning_rate": 0.0001, + "loss": 61.7845, + "step": 4940 + }, + { + "epoch": 79.84552845528455, + "grad_norm": 1.6206163167953491, + "learning_rate": 0.0001, + "loss": 61.5232, + "step": 4950 + }, + { + "epoch": 80.0, + "grad_norm": 1.0924937725067139, + "learning_rate": 0.0001, + "loss": 57.24, + "step": 4960 + }, + { + "epoch": 80.16260162601625, + "grad_norm": 1.075985074043274, + "learning_rate": 0.0001, + "loss": 61.2615, + "step": 4970 + }, + { + "epoch": 80.32520325203252, + "grad_norm": 0.9964771866798401, + "learning_rate": 0.0001, + "loss": 62.287, + "step": 4980 + }, + { + "epoch": 80.48780487804878, + "grad_norm": 1.3707735538482666, + "learning_rate": 0.0001, + "loss": 60.9383, + "step": 4990 + }, + { + "epoch": 80.65040650406505, + "grad_norm": 0.9395170211791992, + "learning_rate": 0.0001, + "loss": 61.2697, + "step": 5000 + }, + { + "epoch": 80.8130081300813, + "grad_norm": 2.0208873748779297, + "learning_rate": 0.0001, + "loss": 61.7219, + "step": 5010 + }, + { + "epoch": 80.97560975609755, + "grad_norm": 1.468702793121338, + "learning_rate": 0.0001, + "loss": 62.6967, + "step": 5020 + }, + { + "epoch": 81.130081300813, + "grad_norm": 1.0525966882705688, + "learning_rate": 0.0001, + "loss": 59.9793, + "step": 5030 + }, + { + "epoch": 81.29268292682927, + "grad_norm": 1.0951132774353027, + "learning_rate": 0.0001, + "loss": 60.9807, + "step": 5040 + }, + { + "epoch": 81.45528455284553, + "grad_norm": 1.3457857370376587, + "learning_rate": 0.0001, + "loss": 61.4137, + "step": 5050 + }, + { + "epoch": 81.6178861788618, + "grad_norm": 1.0895649194717407, + "learning_rate": 0.0001, + "loss": 61.2747, + "step": 5060 + }, + { + "epoch": 81.78048780487805, + "grad_norm": 1.0552371740341187, + "learning_rate": 0.0001, + "loss": 62.0947, + "step": 5070 + }, + { + "epoch": 81.9430894308943, + "grad_norm": 1.039038896560669, + "learning_rate": 0.0001, + "loss": 61.2571, + "step": 5080 + }, + { + "epoch": 82.09756097560975, + "grad_norm": 0.970783531665802, + "learning_rate": 0.0001, + "loss": 57.4157, + "step": 5090 + }, + { + "epoch": 82.26016260162602, + "grad_norm": 1.0276020765304565, + "learning_rate": 0.0001, + "loss": 62.1324, + "step": 5100 + }, + { + "epoch": 82.42276422764228, + "grad_norm": 1.2092448472976685, + "learning_rate": 0.0001, + "loss": 60.6448, + "step": 5110 + }, + { + "epoch": 82.58536585365853, + "grad_norm": 0.9352007508277893, + "learning_rate": 0.0001, + "loss": 61.9299, + "step": 5120 + }, + { + "epoch": 82.7479674796748, + "grad_norm": 1.2678358554840088, + "learning_rate": 0.0001, + "loss": 61.6622, + "step": 5130 + }, + { + "epoch": 82.91056910569105, + "grad_norm": 1.200494647026062, + "learning_rate": 0.0001, + "loss": 62.3169, + "step": 5140 + }, + { + "epoch": 83.0650406504065, + "grad_norm": 1.1284401416778564, + "learning_rate": 0.0001, + "loss": 58.7768, + "step": 5150 + }, + { + "epoch": 83.22764227642277, + "grad_norm": 1.1578079462051392, + "learning_rate": 0.0001, + "loss": 62.8423, + "step": 5160 + }, + { + "epoch": 83.39024390243902, + "grad_norm": 1.3426504135131836, + "learning_rate": 0.0001, + "loss": 61.5332, + "step": 5170 + }, + { + "epoch": 83.55284552845528, + "grad_norm": 0.9397301077842712, + "learning_rate": 0.0001, + "loss": 60.1043, + "step": 5180 + }, + { + "epoch": 83.71544715447155, + "grad_norm": 1.0728143453598022, + "learning_rate": 0.0001, + "loss": 60.5927, + "step": 5190 + }, + { + "epoch": 83.8780487804878, + "grad_norm": 1.2620211839675903, + "learning_rate": 0.0001, + "loss": 60.9224, + "step": 5200 + }, + { + "epoch": 84.03252032520325, + "grad_norm": 0.8232231140136719, + "learning_rate": 0.0001, + "loss": 58.8128, + "step": 5210 + }, + { + "epoch": 84.1951219512195, + "grad_norm": 1.0222564935684204, + "learning_rate": 0.0001, + "loss": 59.6995, + "step": 5220 + }, + { + "epoch": 84.35772357723577, + "grad_norm": 0.9078415632247925, + "learning_rate": 0.0001, + "loss": 61.1521, + "step": 5230 + }, + { + "epoch": 84.52032520325203, + "grad_norm": 1.0269399881362915, + "learning_rate": 0.0001, + "loss": 61.0136, + "step": 5240 + }, + { + "epoch": 84.6829268292683, + "grad_norm": 1.2108154296875, + "learning_rate": 0.0001, + "loss": 61.3564, + "step": 5250 + }, + { + "epoch": 84.84552845528455, + "grad_norm": 1.1170622110366821, + "learning_rate": 0.0001, + "loss": 61.7047, + "step": 5260 + }, + { + "epoch": 85.0, + "grad_norm": 1.467298150062561, + "learning_rate": 0.0001, + "loss": 58.1667, + "step": 5270 + }, + { + "epoch": 85.16260162601625, + "grad_norm": 0.908743143081665, + "learning_rate": 0.0001, + "loss": 61.5694, + "step": 5280 + }, + { + "epoch": 85.32520325203252, + "grad_norm": 0.9148766398429871, + "learning_rate": 0.0001, + "loss": 61.5325, + "step": 5290 + }, + { + "epoch": 85.48780487804878, + "grad_norm": 0.940021276473999, + "learning_rate": 0.0001, + "loss": 62.8807, + "step": 5300 + }, + { + "epoch": 85.65040650406505, + "grad_norm": 1.1239454746246338, + "learning_rate": 0.0001, + "loss": 62.361, + "step": 5310 + }, + { + "epoch": 85.8130081300813, + "grad_norm": 0.840437114238739, + "learning_rate": 0.0001, + "loss": 61.9057, + "step": 5320 + }, + { + "epoch": 85.97560975609755, + "grad_norm": 0.9764263033866882, + "learning_rate": 0.0001, + "loss": 61.0075, + "step": 5330 + }, + { + "epoch": 86.130081300813, + "grad_norm": 0.998540997505188, + "learning_rate": 0.0001, + "loss": 57.7146, + "step": 5340 + }, + { + "epoch": 86.29268292682927, + "grad_norm": 1.121551513671875, + "learning_rate": 0.0001, + "loss": 62.2278, + "step": 5350 + }, + { + "epoch": 86.45528455284553, + "grad_norm": 1.101842999458313, + "learning_rate": 0.0001, + "loss": 60.8837, + "step": 5360 + }, + { + "epoch": 86.6178861788618, + "grad_norm": 1.1724908351898193, + "learning_rate": 0.0001, + "loss": 61.6788, + "step": 5370 + }, + { + "epoch": 86.78048780487805, + "grad_norm": 1.3108958005905151, + "learning_rate": 0.0001, + "loss": 60.1461, + "step": 5380 + }, + { + "epoch": 86.9430894308943, + "grad_norm": 1.482298731803894, + "learning_rate": 0.0001, + "loss": 61.0457, + "step": 5390 + }, + { + "epoch": 87.09756097560975, + "grad_norm": 1.1471034288406372, + "learning_rate": 0.0001, + "loss": 57.6835, + "step": 5400 + }, + { + "epoch": 87.26016260162602, + "grad_norm": 0.9349374175071716, + "learning_rate": 0.0001, + "loss": 62.7018, + "step": 5410 + }, + { + "epoch": 87.42276422764228, + "grad_norm": 0.9917856454849243, + "learning_rate": 0.0001, + "loss": 60.2201, + "step": 5420 + }, + { + "epoch": 87.58536585365853, + "grad_norm": 1.0112310647964478, + "learning_rate": 0.0001, + "loss": 62.1185, + "step": 5430 + }, + { + "epoch": 87.7479674796748, + "grad_norm": 1.288894534111023, + "learning_rate": 0.0001, + "loss": 61.4348, + "step": 5440 + }, + { + "epoch": 87.91056910569105, + "grad_norm": 0.8577322363853455, + "learning_rate": 0.0001, + "loss": 60.7867, + "step": 5450 + }, + { + "epoch": 88.0650406504065, + "grad_norm": 0.8162860870361328, + "learning_rate": 0.0001, + "loss": 59.5224, + "step": 5460 + }, + { + "epoch": 88.22764227642277, + "grad_norm": 0.9491421580314636, + "learning_rate": 0.0001, + "loss": 60.4936, + "step": 5470 + }, + { + "epoch": 88.39024390243902, + "grad_norm": 4.164788246154785, + "learning_rate": 0.0001, + "loss": 61.8397, + "step": 5480 + }, + { + "epoch": 88.55284552845528, + "grad_norm": 0.9954016804695129, + "learning_rate": 0.0001, + "loss": 61.286, + "step": 5490 + }, + { + "epoch": 88.71544715447155, + "grad_norm": 1.4991120100021362, + "learning_rate": 0.0001, + "loss": 60.6157, + "step": 5500 + }, + { + "epoch": 88.8780487804878, + "grad_norm": 1.3678480386734009, + "learning_rate": 0.0001, + "loss": 62.5187, + "step": 5510 + }, + { + "epoch": 89.03252032520325, + "grad_norm": 1.057661771774292, + "learning_rate": 0.0001, + "loss": 57.5134, + "step": 5520 + }, + { + "epoch": 89.1951219512195, + "grad_norm": 0.8331280946731567, + "learning_rate": 0.0001, + "loss": 62.007, + "step": 5530 + }, + { + "epoch": 89.35772357723577, + "grad_norm": 0.9842398762702942, + "learning_rate": 0.0001, + "loss": 60.6178, + "step": 5540 + }, + { + "epoch": 89.52032520325203, + "grad_norm": 1.0510483980178833, + "learning_rate": 0.0001, + "loss": 59.6382, + "step": 5550 + }, + { + "epoch": 89.6829268292683, + "grad_norm": 1.013115406036377, + "learning_rate": 0.0001, + "loss": 61.015, + "step": 5560 + }, + { + "epoch": 89.84552845528455, + "grad_norm": 1.1595944166183472, + "learning_rate": 0.0001, + "loss": 62.5374, + "step": 5570 + }, + { + "epoch": 90.0, + "grad_norm": 1.1506270170211792, + "learning_rate": 0.0001, + "loss": 61.0063, + "step": 5580 + }, + { + "epoch": 90.16260162601625, + "grad_norm": 0.9940153360366821, + "learning_rate": 0.0001, + "loss": 62.0944, + "step": 5590 + }, + { + "epoch": 90.32520325203252, + "grad_norm": 1.1302493810653687, + "learning_rate": 0.0001, + "loss": 61.616, + "step": 5600 + }, + { + "epoch": 90.48780487804878, + "grad_norm": 1.2370656728744507, + "learning_rate": 0.0001, + "loss": 61.291, + "step": 5610 + }, + { + "epoch": 90.65040650406505, + "grad_norm": 1.228763461112976, + "learning_rate": 0.0001, + "loss": 60.0629, + "step": 5620 + }, + { + "epoch": 90.8130081300813, + "grad_norm": 1.2230069637298584, + "learning_rate": 0.0001, + "loss": 61.5391, + "step": 5630 + }, + { + "epoch": 90.97560975609755, + "grad_norm": 0.9849008917808533, + "learning_rate": 0.0001, + "loss": 63.1633, + "step": 5640 + }, + { + "epoch": 91.130081300813, + "grad_norm": 1.408632755279541, + "learning_rate": 0.0001, + "loss": 58.4195, + "step": 5650 + }, + { + "epoch": 91.29268292682927, + "grad_norm": 0.929027795791626, + "learning_rate": 0.0001, + "loss": 60.9862, + "step": 5660 + }, + { + "epoch": 91.45528455284553, + "grad_norm": 0.9228377342224121, + "learning_rate": 0.0001, + "loss": 60.8337, + "step": 5670 + }, + { + "epoch": 91.6178861788618, + "grad_norm": 1.0354220867156982, + "learning_rate": 0.0001, + "loss": 59.4667, + "step": 5680 + }, + { + "epoch": 91.78048780487805, + "grad_norm": 1.0338927507400513, + "learning_rate": 0.0001, + "loss": 62.0064, + "step": 5690 + }, + { + "epoch": 91.9430894308943, + "grad_norm": 1.101967215538025, + "learning_rate": 0.0001, + "loss": 61.6317, + "step": 5700 + }, + { + "epoch": 92.09756097560975, + "grad_norm": 1.370689034461975, + "learning_rate": 0.0001, + "loss": 57.2157, + "step": 5710 + }, + { + "epoch": 92.26016260162602, + "grad_norm": 0.9904115796089172, + "learning_rate": 0.0001, + "loss": 61.2786, + "step": 5720 + }, + { + "epoch": 92.42276422764228, + "grad_norm": 1.0545448064804077, + "learning_rate": 0.0001, + "loss": 63.2384, + "step": 5730 + }, + { + "epoch": 92.58536585365853, + "grad_norm": 1.0657731294631958, + "learning_rate": 0.0001, + "loss": 60.582, + "step": 5740 + }, + { + "epoch": 92.7479674796748, + "grad_norm": 0.9333183765411377, + "learning_rate": 0.0001, + "loss": 62.4627, + "step": 5750 + }, + { + "epoch": 92.91056910569105, + "grad_norm": 0.9810492396354675, + "learning_rate": 0.0001, + "loss": 60.0055, + "step": 5760 + }, + { + "epoch": 93.0650406504065, + "grad_norm": 1.6973884105682373, + "learning_rate": 0.0001, + "loss": 57.5381, + "step": 5770 + }, + { + "epoch": 93.22764227642277, + "grad_norm": 1.0075498819351196, + "learning_rate": 0.0001, + "loss": 61.2272, + "step": 5780 + }, + { + "epoch": 93.39024390243902, + "grad_norm": 0.980633556842804, + "learning_rate": 0.0001, + "loss": 60.4794, + "step": 5790 + }, + { + "epoch": 93.55284552845528, + "grad_norm": 1.128214955329895, + "learning_rate": 0.0001, + "loss": 60.4825, + "step": 5800 + }, + { + "epoch": 93.71544715447155, + "grad_norm": 1.302827000617981, + "learning_rate": 0.0001, + "loss": 62.4397, + "step": 5810 + }, + { + "epoch": 93.8780487804878, + "grad_norm": 0.826600193977356, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 5820 + }, + { + "epoch": 94.03252032520325, + "grad_norm": 11.418835639953613, + "learning_rate": 0.0001, + "loss": 58.5383, + "step": 5830 + }, + { + "epoch": 94.1951219512195, + "grad_norm": 1.0686477422714233, + "learning_rate": 0.0001, + "loss": 60.5696, + "step": 5840 + }, + { + "epoch": 94.35772357723577, + "grad_norm": 1.1232597827911377, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 5850 + }, + { + "epoch": 94.52032520325203, + "grad_norm": 1.1051944494247437, + "learning_rate": 0.0001, + "loss": 60.5387, + "step": 5860 + }, + { + "epoch": 94.6829268292683, + "grad_norm": 0.9954739809036255, + "learning_rate": 0.0001, + "loss": 61.3899, + "step": 5870 + }, + { + "epoch": 94.84552845528455, + "grad_norm": 1.9687999486923218, + "learning_rate": 0.0001, + "loss": 60.7711, + "step": 5880 + }, + { + "epoch": 95.0, + "grad_norm": 1.3172541856765747, + "learning_rate": 0.0001, + "loss": 58.5394, + "step": 5890 + }, + { + "epoch": 95.16260162601625, + "grad_norm": 1.2251427173614502, + "learning_rate": 0.0001, + "loss": 61.8319, + "step": 5900 + }, + { + "epoch": 95.32520325203252, + "grad_norm": 1.1701544523239136, + "learning_rate": 0.0001, + "loss": 60.1928, + "step": 5910 + }, + { + "epoch": 95.48780487804878, + "grad_norm": 1.209375262260437, + "learning_rate": 0.0001, + "loss": 61.1347, + "step": 5920 + }, + { + "epoch": 95.65040650406505, + "grad_norm": 0.9307044148445129, + "learning_rate": 0.0001, + "loss": 62.6676, + "step": 5930 + }, + { + "epoch": 95.8130081300813, + "grad_norm": 0.9679144024848938, + "learning_rate": 0.0001, + "loss": 60.7623, + "step": 5940 + }, + { + "epoch": 95.97560975609755, + "grad_norm": 1.398711085319519, + "learning_rate": 0.0001, + "loss": 59.0407, + "step": 5950 + }, + { + "epoch": 96.130081300813, + "grad_norm": 1.9118247032165527, + "learning_rate": 0.0001, + "loss": 59.9417, + "step": 5960 + }, + { + "epoch": 96.29268292682927, + "grad_norm": 1.036737084388733, + "learning_rate": 0.0001, + "loss": 60.6891, + "step": 5970 + }, + { + "epoch": 96.45528455284553, + "grad_norm": 1.5916132926940918, + "learning_rate": 0.0001, + "loss": 59.44, + "step": 5980 + }, + { + "epoch": 96.6178861788618, + "grad_norm": 1.526591181755066, + "learning_rate": 0.0001, + "loss": 58.9081, + "step": 5990 + }, + { + "epoch": 96.78048780487805, + "grad_norm": 0.9556435942649841, + "learning_rate": 0.0001, + "loss": 61.5743, + "step": 6000 + }, + { + "epoch": 96.9430894308943, + "grad_norm": 1.0041887760162354, + "learning_rate": 0.0001, + "loss": 60.0724, + "step": 6010 + }, + { + "epoch": 97.09756097560975, + "grad_norm": 0.9792081117630005, + "learning_rate": 0.0001, + "loss": 60.0613, + "step": 6020 + }, + { + "epoch": 97.26016260162602, + "grad_norm": 1.029634714126587, + "learning_rate": 0.0001, + "loss": 60.3526, + "step": 6030 + }, + { + "epoch": 97.42276422764228, + "grad_norm": 0.9812865257263184, + "learning_rate": 0.0001, + "loss": 60.9818, + "step": 6040 + }, + { + "epoch": 97.58536585365853, + "grad_norm": 0.8826327323913574, + "learning_rate": 0.0001, + "loss": 61.6375, + "step": 6050 + }, + { + "epoch": 97.7479674796748, + "grad_norm": 1.2197599411010742, + "learning_rate": 0.0001, + "loss": 60.2698, + "step": 6060 + }, + { + "epoch": 97.91056910569105, + "grad_norm": 0.9068922400474548, + "learning_rate": 0.0001, + "loss": 58.8698, + "step": 6070 + }, + { + "epoch": 98.0650406504065, + "grad_norm": 0.9008221626281738, + "learning_rate": 0.0001, + "loss": 56.4687, + "step": 6080 + }, + { + "epoch": 98.22764227642277, + "grad_norm": 1.8165878057479858, + "learning_rate": 0.0001, + "loss": 61.4213, + "step": 6090 + }, + { + "epoch": 98.39024390243902, + "grad_norm": 0.9541304111480713, + "learning_rate": 0.0001, + "loss": 62.8381, + "step": 6100 + }, + { + "epoch": 98.55284552845528, + "grad_norm": 1.293872594833374, + "learning_rate": 0.0001, + "loss": 61.3292, + "step": 6110 + }, + { + "epoch": 98.71544715447155, + "grad_norm": 0.9569422006607056, + "learning_rate": 0.0001, + "loss": 61.436, + "step": 6120 + }, + { + "epoch": 98.8780487804878, + "grad_norm": 1.160468578338623, + "learning_rate": 0.0001, + "loss": 60.8419, + "step": 6130 + }, + { + "epoch": 99.03252032520325, + "grad_norm": 1.3102303743362427, + "learning_rate": 0.0001, + "loss": 59.0064, + "step": 6140 + }, + { + "epoch": 99.1951219512195, + "grad_norm": 0.9553463459014893, + "learning_rate": 0.0001, + "loss": 59.925, + "step": 6150 + }, + { + "epoch": 99.35772357723577, + "grad_norm": 1.1993688344955444, + "learning_rate": 0.0001, + "loss": 62.3683, + "step": 6160 + }, + { + "epoch": 99.52032520325203, + "grad_norm": 1.0867350101470947, + "learning_rate": 0.0001, + "loss": 61.8848, + "step": 6170 + }, + { + "epoch": 99.6829268292683, + "grad_norm": 0.9924171566963196, + "learning_rate": 0.0001, + "loss": 60.9243, + "step": 6180 + }, + { + "epoch": 99.84552845528455, + "grad_norm": 0.9548614025115967, + "learning_rate": 0.0001, + "loss": 61.0234, + "step": 6190 + }, + { + "epoch": 100.0, + "grad_norm": 1.0070925951004028, + "learning_rate": 0.0001, + "loss": 56.6554, + "step": 6200 + }, + { + "epoch": 100.16260162601625, + "grad_norm": 1.202168583869934, + "learning_rate": 0.0001, + "loss": 61.3235, + "step": 6210 + }, + { + "epoch": 100.32520325203252, + "grad_norm": 0.9931091070175171, + "learning_rate": 0.0001, + "loss": 59.651, + "step": 6220 + }, + { + "epoch": 100.48780487804878, + "grad_norm": 0.9679157137870789, + "learning_rate": 0.0001, + "loss": 61.5785, + "step": 6230 + }, + { + "epoch": 100.65040650406505, + "grad_norm": 1.049842119216919, + "learning_rate": 0.0001, + "loss": 58.9646, + "step": 6240 + }, + { + "epoch": 100.8130081300813, + "grad_norm": 1.0772044658660889, + "learning_rate": 0.0001, + "loss": 62.3309, + "step": 6250 + }, + { + "epoch": 100.97560975609755, + "grad_norm": 0.9494168162345886, + "learning_rate": 0.0001, + "loss": 61.8663, + "step": 6260 + }, + { + "epoch": 101.130081300813, + "grad_norm": 1.0491962432861328, + "learning_rate": 0.0001, + "loss": 56.5872, + "step": 6270 + }, + { + "epoch": 101.29268292682927, + "grad_norm": 1.118611216545105, + "learning_rate": 0.0001, + "loss": 58.1338, + "step": 6280 + }, + { + "epoch": 101.45528455284553, + "grad_norm": 1.0419825315475464, + "learning_rate": 0.0001, + "loss": 59.6661, + "step": 6290 + }, + { + "epoch": 101.6178861788618, + "grad_norm": 1.2351056337356567, + "learning_rate": 0.0001, + "loss": 61.1697, + "step": 6300 + }, + { + "epoch": 101.78048780487805, + "grad_norm": 1.0556049346923828, + "learning_rate": 0.0001, + "loss": 60.463, + "step": 6310 + }, + { + "epoch": 101.9430894308943, + "grad_norm": 1.090761661529541, + "learning_rate": 0.0001, + "loss": 61.983, + "step": 6320 + }, + { + "epoch": 102.09756097560975, + "grad_norm": 1.4305297136306763, + "learning_rate": 0.0001, + "loss": 55.559, + "step": 6330 + }, + { + "epoch": 102.26016260162602, + "grad_norm": 1.1773794889450073, + "learning_rate": 0.0001, + "loss": 59.5931, + "step": 6340 + }, + { + "epoch": 102.42276422764228, + "grad_norm": 1.0028730630874634, + "learning_rate": 0.0001, + "loss": 60.2714, + "step": 6350 + }, + { + "epoch": 102.58536585365853, + "grad_norm": 0.9494019150733948, + "learning_rate": 0.0001, + "loss": 59.7757, + "step": 6360 + }, + { + "epoch": 102.7479674796748, + "grad_norm": 1.0415751934051514, + "learning_rate": 0.0001, + "loss": 60.7123, + "step": 6370 + }, + { + "epoch": 102.91056910569105, + "grad_norm": 1.2036449909210205, + "learning_rate": 0.0001, + "loss": 60.1201, + "step": 6380 + }, + { + "epoch": 103.0650406504065, + "grad_norm": 0.8801053166389465, + "learning_rate": 0.0001, + "loss": 56.4943, + "step": 6390 + }, + { + "epoch": 103.22764227642277, + "grad_norm": 0.9018735289573669, + "learning_rate": 0.0001, + "loss": 62.4522, + "step": 6400 + }, + { + "epoch": 103.39024390243902, + "grad_norm": 0.971685528755188, + "learning_rate": 0.0001, + "loss": 62.2367, + "step": 6410 + }, + { + "epoch": 103.55284552845528, + "grad_norm": 0.8531002402305603, + "learning_rate": 0.0001, + "loss": 59.5436, + "step": 6420 + }, + { + "epoch": 103.71544715447155, + "grad_norm": 2.123033285140991, + "learning_rate": 0.0001, + "loss": 59.0421, + "step": 6430 + }, + { + "epoch": 103.8780487804878, + "grad_norm": 1.209804654121399, + "learning_rate": 0.0001, + "loss": 60.7791, + "step": 6440 + }, + { + "epoch": 104.03252032520325, + "grad_norm": 0.8219161033630371, + "learning_rate": 0.0001, + "loss": 55.7522, + "step": 6450 + }, + { + "epoch": 104.1951219512195, + "grad_norm": 1.8217495679855347, + "learning_rate": 0.0001, + "loss": 60.6916, + "step": 6460 + }, + { + "epoch": 104.35772357723577, + "grad_norm": 1.060073971748352, + "learning_rate": 0.0001, + "loss": 59.971, + "step": 6470 + }, + { + "epoch": 104.52032520325203, + "grad_norm": 0.9892823696136475, + "learning_rate": 0.0001, + "loss": 61.467, + "step": 6480 + }, + { + "epoch": 104.6829268292683, + "grad_norm": 1.128934621810913, + "learning_rate": 0.0001, + "loss": 62.0451, + "step": 6490 + }, + { + "epoch": 104.84552845528455, + "grad_norm": 0.9627387523651123, + "learning_rate": 0.0001, + "loss": 61.532, + "step": 6500 + }, + { + "epoch": 105.0, + "grad_norm": 1.5344375371932983, + "learning_rate": 0.0001, + "loss": 56.4323, + "step": 6510 + }, + { + "epoch": 105.16260162601625, + "grad_norm": 0.9524087309837341, + "learning_rate": 0.0001, + "loss": 59.6628, + "step": 6520 + }, + { + "epoch": 105.32520325203252, + "grad_norm": 1.0469943284988403, + "learning_rate": 0.0001, + "loss": 59.0605, + "step": 6530 + }, + { + "epoch": 105.48780487804878, + "grad_norm": 2.7462990283966064, + "learning_rate": 0.0001, + "loss": 61.1429, + "step": 6540 + }, + { + "epoch": 105.65040650406505, + "grad_norm": 1.3140382766723633, + "learning_rate": 0.0001, + "loss": 58.2711, + "step": 6550 + }, + { + "epoch": 105.8130081300813, + "grad_norm": 1.1528750658035278, + "learning_rate": 0.0001, + "loss": 59.9937, + "step": 6560 + }, + { + "epoch": 105.97560975609755, + "grad_norm": 1.4449951648712158, + "learning_rate": 0.0001, + "loss": 59.1526, + "step": 6570 + }, + { + "epoch": 106.130081300813, + "grad_norm": 1.1350126266479492, + "learning_rate": 0.0001, + "loss": 55.1142, + "step": 6580 + }, + { + "epoch": 106.29268292682927, + "grad_norm": 0.9332845211029053, + "learning_rate": 0.0001, + "loss": 59.8049, + "step": 6590 + }, + { + "epoch": 106.45528455284553, + "grad_norm": 0.9678266644477844, + "learning_rate": 0.0001, + "loss": 60.9627, + "step": 6600 + }, + { + "epoch": 106.6178861788618, + "grad_norm": 1.1794427633285522, + "learning_rate": 0.0001, + "loss": 59.5372, + "step": 6610 + }, + { + "epoch": 106.78048780487805, + "grad_norm": 1.0404181480407715, + "learning_rate": 0.0001, + "loss": 59.2715, + "step": 6620 + }, + { + "epoch": 106.9430894308943, + "grad_norm": 0.8667073845863342, + "learning_rate": 0.0001, + "loss": 60.2625, + "step": 6630 + }, + { + "epoch": 107.09756097560975, + "grad_norm": 0.9940314888954163, + "learning_rate": 0.0001, + "loss": 55.6906, + "step": 6640 + }, + { + "epoch": 107.26016260162602, + "grad_norm": 1.3880919218063354, + "learning_rate": 0.0001, + "loss": 59.8103, + "step": 6650 + }, + { + "epoch": 107.42276422764228, + "grad_norm": 0.900931179523468, + "learning_rate": 0.0001, + "loss": 59.1929, + "step": 6660 + }, + { + "epoch": 107.58536585365853, + "grad_norm": 0.9374089241027832, + "learning_rate": 0.0001, + "loss": 60.1047, + "step": 6670 + }, + { + "epoch": 107.7479674796748, + "grad_norm": 1.0847140550613403, + "learning_rate": 0.0001, + "loss": 58.6943, + "step": 6680 + }, + { + "epoch": 107.91056910569105, + "grad_norm": 1.086506962776184, + "learning_rate": 0.0001, + "loss": 60.0341, + "step": 6690 + }, + { + "epoch": 108.0650406504065, + "grad_norm": 0.8984777331352234, + "learning_rate": 0.0001, + "loss": 57.0422, + "step": 6700 + }, + { + "epoch": 108.22764227642277, + "grad_norm": 1.187198519706726, + "learning_rate": 0.0001, + "loss": 60.7935, + "step": 6710 + }, + { + "epoch": 108.39024390243902, + "grad_norm": 0.8393696546554565, + "learning_rate": 0.0001, + "loss": 59.7896, + "step": 6720 + }, + { + "epoch": 108.55284552845528, + "grad_norm": 1.0898207426071167, + "learning_rate": 0.0001, + "loss": 60.2579, + "step": 6730 + }, + { + "epoch": 108.71544715447155, + "grad_norm": 1.016926646232605, + "learning_rate": 0.0001, + "loss": 60.9018, + "step": 6740 + }, + { + "epoch": 108.8780487804878, + "grad_norm": 1.3991293907165527, + "learning_rate": 0.0001, + "loss": 60.889, + "step": 6750 + }, + { + "epoch": 109.03252032520325, + "grad_norm": 0.9950843453407288, + "learning_rate": 0.0001, + "loss": 55.1098, + "step": 6760 + }, + { + "epoch": 109.1951219512195, + "grad_norm": 1.0843164920806885, + "learning_rate": 0.0001, + "loss": 60.3208, + "step": 6770 + }, + { + "epoch": 109.35772357723577, + "grad_norm": 0.9067014455795288, + "learning_rate": 0.0001, + "loss": 59.1254, + "step": 6780 + }, + { + "epoch": 109.52032520325203, + "grad_norm": 1.2166807651519775, + "learning_rate": 0.0001, + "loss": 59.9978, + "step": 6790 + }, + { + "epoch": 109.6829268292683, + "grad_norm": 1.256101131439209, + "learning_rate": 0.0001, + "loss": 60.2544, + "step": 6800 + }, + { + "epoch": 109.84552845528455, + "grad_norm": 1.312761664390564, + "learning_rate": 0.0001, + "loss": 59.1564, + "step": 6810 + }, + { + "epoch": 110.0, + "grad_norm": 0.9668724536895752, + "learning_rate": 0.0001, + "loss": 58.067, + "step": 6820 + }, + { + "epoch": 110.16260162601625, + "grad_norm": 1.1048885583877563, + "learning_rate": 0.0001, + "loss": 59.1506, + "step": 6830 + }, + { + "epoch": 110.32520325203252, + "grad_norm": 1.498469352722168, + "learning_rate": 0.0001, + "loss": 59.1366, + "step": 6840 + }, + { + "epoch": 110.48780487804878, + "grad_norm": 1.1747376918792725, + "learning_rate": 0.0001, + "loss": 59.8449, + "step": 6850 + }, + { + "epoch": 110.65040650406505, + "grad_norm": 1.1041772365570068, + "learning_rate": 0.0001, + "loss": 61.0879, + "step": 6860 + }, + { + "epoch": 110.8130081300813, + "grad_norm": 0.9452431797981262, + "learning_rate": 0.0001, + "loss": 60.8911, + "step": 6870 + }, + { + "epoch": 110.97560975609755, + "grad_norm": 1.0108182430267334, + "learning_rate": 0.0001, + "loss": 58.7924, + "step": 6880 + }, + { + "epoch": 111.130081300813, + "grad_norm": 0.9914487600326538, + "learning_rate": 0.0001, + "loss": 57.4749, + "step": 6890 + }, + { + "epoch": 111.29268292682927, + "grad_norm": 1.0404850244522095, + "learning_rate": 0.0001, + "loss": 59.5329, + "step": 6900 + }, + { + "epoch": 111.45528455284553, + "grad_norm": 1.0363966226577759, + "learning_rate": 0.0001, + "loss": 58.8972, + "step": 6910 + }, + { + "epoch": 111.6178861788618, + "grad_norm": 1.0189651250839233, + "learning_rate": 0.0001, + "loss": 59.3613, + "step": 6920 + }, + { + "epoch": 111.78048780487805, + "grad_norm": 0.9696914553642273, + "learning_rate": 0.0001, + "loss": 59.8001, + "step": 6930 + }, + { + "epoch": 111.9430894308943, + "grad_norm": 1.406821370124817, + "learning_rate": 0.0001, + "loss": 58.6405, + "step": 6940 + }, + { + "epoch": 112.09756097560975, + "grad_norm": 1.0969383716583252, + "learning_rate": 0.0001, + "loss": 57.4458, + "step": 6950 + }, + { + "epoch": 112.26016260162602, + "grad_norm": 1.117593765258789, + "learning_rate": 0.0001, + "loss": 62.599, + "step": 6960 + }, + { + "epoch": 112.42276422764228, + "grad_norm": 0.9327741861343384, + "learning_rate": 0.0001, + "loss": 60.632, + "step": 6970 + }, + { + "epoch": 112.58536585365853, + "grad_norm": 1.1198201179504395, + "learning_rate": 0.0001, + "loss": 59.6119, + "step": 6980 + }, + { + "epoch": 112.7479674796748, + "grad_norm": 0.8709354400634766, + "learning_rate": 0.0001, + "loss": 60.831, + "step": 6990 + }, + { + "epoch": 112.91056910569105, + "grad_norm": 0.9589444994926453, + "learning_rate": 0.0001, + "loss": 58.0002, + "step": 7000 + }, + { + "epoch": 113.0650406504065, + "grad_norm": 1.0971899032592773, + "learning_rate": 0.0001, + "loss": 56.8572, + "step": 7010 + }, + { + "epoch": 113.22764227642277, + "grad_norm": 0.9764513969421387, + "learning_rate": 0.0001, + "loss": 59.3038, + "step": 7020 + }, + { + "epoch": 113.39024390243902, + "grad_norm": 0.9050093293190002, + "learning_rate": 0.0001, + "loss": 59.9146, + "step": 7030 + }, + { + "epoch": 113.55284552845528, + "grad_norm": 0.9506774544715881, + "learning_rate": 0.0001, + "loss": 59.7523, + "step": 7040 + }, + { + "epoch": 113.71544715447155, + "grad_norm": 0.9678346514701843, + "learning_rate": 0.0001, + "loss": 61.3103, + "step": 7050 + }, + { + "epoch": 113.8780487804878, + "grad_norm": 0.9859515428543091, + "learning_rate": 0.0001, + "loss": 56.3866, + "step": 7060 + }, + { + "epoch": 114.03252032520325, + "grad_norm": 1.2696164846420288, + "learning_rate": 0.0001, + "loss": 58.16, + "step": 7070 + }, + { + "epoch": 114.1951219512195, + "grad_norm": 0.9323938488960266, + "learning_rate": 0.0001, + "loss": 58.9483, + "step": 7080 + }, + { + "epoch": 114.35772357723577, + "grad_norm": 1.4123785495758057, + "learning_rate": 0.0001, + "loss": 60.4666, + "step": 7090 + }, + { + "epoch": 114.52032520325203, + "grad_norm": 1.009020447731018, + "learning_rate": 0.0001, + "loss": 58.7152, + "step": 7100 + }, + { + "epoch": 114.6829268292683, + "grad_norm": 1.219383955001831, + "learning_rate": 0.0001, + "loss": 59.1494, + "step": 7110 + }, + { + "epoch": 114.84552845528455, + "grad_norm": 0.9579033255577087, + "learning_rate": 0.0001, + "loss": 58.9879, + "step": 7120 + }, + { + "epoch": 115.0, + "grad_norm": 0.899074912071228, + "learning_rate": 0.0001, + "loss": 55.2031, + "step": 7130 + }, + { + "epoch": 115.16260162601625, + "grad_norm": 0.9924830794334412, + "learning_rate": 0.0001, + "loss": 58.583, + "step": 7140 + }, + { + "epoch": 115.32520325203252, + "grad_norm": 1.0280295610427856, + "learning_rate": 0.0001, + "loss": 60.0948, + "step": 7150 + }, + { + "epoch": 115.48780487804878, + "grad_norm": 0.8108797669410706, + "learning_rate": 0.0001, + "loss": 60.9433, + "step": 7160 + }, + { + "epoch": 115.65040650406505, + "grad_norm": 1.2597944736480713, + "learning_rate": 0.0001, + "loss": 59.1905, + "step": 7170 + }, + { + "epoch": 115.8130081300813, + "grad_norm": 1.2565767765045166, + "learning_rate": 0.0001, + "loss": 59.3545, + "step": 7180 + }, + { + "epoch": 115.97560975609755, + "grad_norm": 1.2012240886688232, + "learning_rate": 0.0001, + "loss": 58.348, + "step": 7190 + }, + { + "epoch": 116.130081300813, + "grad_norm": 1.2883665561676025, + "learning_rate": 0.0001, + "loss": 57.462, + "step": 7200 + }, + { + "epoch": 116.29268292682927, + "grad_norm": 0.904152512550354, + "learning_rate": 0.0001, + "loss": 58.958, + "step": 7210 + }, + { + "epoch": 116.45528455284553, + "grad_norm": 1.1928468942642212, + "learning_rate": 0.0001, + "loss": 58.6106, + "step": 7220 + }, + { + "epoch": 116.6178861788618, + "grad_norm": 1.0655683279037476, + "learning_rate": 0.0001, + "loss": 58.4948, + "step": 7230 + }, + { + "epoch": 116.78048780487805, + "grad_norm": 1.0527434349060059, + "learning_rate": 0.0001, + "loss": 60.1771, + "step": 7240 + }, + { + "epoch": 116.9430894308943, + "grad_norm": 1.1439871788024902, + "learning_rate": 0.0001, + "loss": 58.6737, + "step": 7250 + }, + { + "epoch": 117.09756097560975, + "grad_norm": 1.506571650505066, + "learning_rate": 0.0001, + "loss": 58.0901, + "step": 7260 + }, + { + "epoch": 117.26016260162602, + "grad_norm": 1.1419955492019653, + "learning_rate": 0.0001, + "loss": 59.0705, + "step": 7270 + }, + { + "epoch": 117.42276422764228, + "grad_norm": 1.1211196184158325, + "learning_rate": 0.0001, + "loss": 57.801, + "step": 7280 + }, + { + "epoch": 117.58536585365853, + "grad_norm": 1.1424155235290527, + "learning_rate": 0.0001, + "loss": 59.7449, + "step": 7290 + }, + { + "epoch": 117.7479674796748, + "grad_norm": 1.0593281984329224, + "learning_rate": 0.0001, + "loss": 60.1789, + "step": 7300 + }, + { + "epoch": 117.91056910569105, + "grad_norm": 1.3680177927017212, + "learning_rate": 0.0001, + "loss": 58.4225, + "step": 7310 + }, + { + "epoch": 118.0650406504065, + "grad_norm": 0.8391700983047485, + "learning_rate": 0.0001, + "loss": 56.2102, + "step": 7320 + }, + { + "epoch": 118.22764227642277, + "grad_norm": 1.3166592121124268, + "learning_rate": 0.0001, + "loss": 61.0399, + "step": 7330 + }, + { + "epoch": 118.39024390243902, + "grad_norm": 1.929949402809143, + "learning_rate": 0.0001, + "loss": 59.6514, + "step": 7340 + }, + { + "epoch": 118.55284552845528, + "grad_norm": 0.9464254975318909, + "learning_rate": 0.0001, + "loss": 58.6488, + "step": 7350 + }, + { + "epoch": 118.71544715447155, + "grad_norm": 1.1066193580627441, + "learning_rate": 0.0001, + "loss": 61.5959, + "step": 7360 + }, + { + "epoch": 118.8780487804878, + "grad_norm": 1.216125726699829, + "learning_rate": 0.0001, + "loss": 58.8231, + "step": 7370 + }, + { + "epoch": 119.03252032520325, + "grad_norm": 1.1182583570480347, + "learning_rate": 0.0001, + "loss": 56.9496, + "step": 7380 + }, + { + "epoch": 119.1951219512195, + "grad_norm": 1.105729341506958, + "learning_rate": 0.0001, + "loss": 58.4291, + "step": 7390 + }, + { + "epoch": 119.35772357723577, + "grad_norm": 1.017879843711853, + "learning_rate": 0.0001, + "loss": 59.4963, + "step": 7400 + }, + { + "epoch": 119.52032520325203, + "grad_norm": 0.7882587909698486, + "learning_rate": 0.0001, + "loss": 57.7243, + "step": 7410 + }, + { + "epoch": 119.6829268292683, + "grad_norm": 0.8356695175170898, + "learning_rate": 0.0001, + "loss": 60.4962, + "step": 7420 + }, + { + "epoch": 119.84552845528455, + "grad_norm": 1.1358702182769775, + "learning_rate": 0.0001, + "loss": 60.7173, + "step": 7430 + }, + { + "epoch": 120.0, + "grad_norm": 1.116322636604309, + "learning_rate": 0.0001, + "loss": 53.8387, + "step": 7440 + }, + { + "epoch": 120.16260162601625, + "grad_norm": 0.8522642254829407, + "learning_rate": 0.0001, + "loss": 57.9095, + "step": 7450 + }, + { + "epoch": 120.32520325203252, + "grad_norm": 0.890731692314148, + "learning_rate": 0.0001, + "loss": 57.2296, + "step": 7460 + }, + { + "epoch": 120.48780487804878, + "grad_norm": 1.6366336345672607, + "learning_rate": 0.0001, + "loss": 58.9417, + "step": 7470 + }, + { + "epoch": 120.65040650406505, + "grad_norm": 0.9774733781814575, + "learning_rate": 0.0001, + "loss": 58.8446, + "step": 7480 + }, + { + "epoch": 120.8130081300813, + "grad_norm": 1.6188803911209106, + "learning_rate": 0.0001, + "loss": 59.9794, + "step": 7490 + }, + { + "epoch": 120.97560975609755, + "grad_norm": 1.146599292755127, + "learning_rate": 0.0001, + "loss": 59.8794, + "step": 7500 + }, + { + "epoch": 121.130081300813, + "grad_norm": 0.9472863078117371, + "learning_rate": 0.0001, + "loss": 55.2454, + "step": 7510 + }, + { + "epoch": 121.29268292682927, + "grad_norm": 0.8604205250740051, + "learning_rate": 0.0001, + "loss": 57.6314, + "step": 7520 + }, + { + "epoch": 121.45528455284553, + "grad_norm": 1.0755794048309326, + "learning_rate": 0.0001, + "loss": 60.909, + "step": 7530 + }, + { + "epoch": 121.6178861788618, + "grad_norm": 0.7967071533203125, + "learning_rate": 0.0001, + "loss": 60.1755, + "step": 7540 + }, + { + "epoch": 121.78048780487805, + "grad_norm": 1.037655234336853, + "learning_rate": 0.0001, + "loss": 59.283, + "step": 7550 + }, + { + "epoch": 121.9430894308943, + "grad_norm": 1.070142388343811, + "learning_rate": 0.0001, + "loss": 57.0941, + "step": 7560 + }, + { + "epoch": 122.09756097560975, + "grad_norm": 1.4201022386550903, + "learning_rate": 0.0001, + "loss": 56.2891, + "step": 7570 + }, + { + "epoch": 122.26016260162602, + "grad_norm": 1.0909277200698853, + "learning_rate": 0.0001, + "loss": 57.5213, + "step": 7580 + }, + { + "epoch": 122.42276422764228, + "grad_norm": 0.7573383450508118, + "learning_rate": 0.0001, + "loss": 59.0847, + "step": 7590 + }, + { + "epoch": 122.58536585365853, + "grad_norm": 0.8931725025177002, + "learning_rate": 0.0001, + "loss": 58.9706, + "step": 7600 + }, + { + "epoch": 122.7479674796748, + "grad_norm": 0.9459935426712036, + "learning_rate": 0.0001, + "loss": 58.6267, + "step": 7610 + }, + { + "epoch": 122.91056910569105, + "grad_norm": 0.9463319778442383, + "learning_rate": 0.0001, + "loss": 60.8536, + "step": 7620 + }, + { + "epoch": 123.0650406504065, + "grad_norm": 0.9286382794380188, + "learning_rate": 0.0001, + "loss": 55.2711, + "step": 7630 + }, + { + "epoch": 123.22764227642277, + "grad_norm": 0.9317494630813599, + "learning_rate": 0.0001, + "loss": 60.1578, + "step": 7640 + }, + { + "epoch": 123.39024390243902, + "grad_norm": 0.9834237098693848, + "learning_rate": 0.0001, + "loss": 58.8121, + "step": 7650 + }, + { + "epoch": 123.55284552845528, + "grad_norm": 0.9992889761924744, + "learning_rate": 0.0001, + "loss": 59.8061, + "step": 7660 + }, + { + "epoch": 123.71544715447155, + "grad_norm": 1.0354453325271606, + "learning_rate": 0.0001, + "loss": 59.5853, + "step": 7670 + }, + { + "epoch": 123.8780487804878, + "grad_norm": 1.0609121322631836, + "learning_rate": 0.0001, + "loss": 59.6556, + "step": 7680 + }, + { + "epoch": 124.03252032520325, + "grad_norm": 0.9742912650108337, + "learning_rate": 0.0001, + "loss": 57.2294, + "step": 7690 + }, + { + "epoch": 124.1951219512195, + "grad_norm": 0.9758710265159607, + "learning_rate": 0.0001, + "loss": 59.8581, + "step": 7700 + }, + { + "epoch": 124.35772357723577, + "grad_norm": 0.9492778182029724, + "learning_rate": 0.0001, + "loss": 60.0624, + "step": 7710 + }, + { + "epoch": 124.52032520325203, + "grad_norm": 0.8311835527420044, + "learning_rate": 0.0001, + "loss": 60.0849, + "step": 7720 + }, + { + "epoch": 124.6829268292683, + "grad_norm": 0.8652944564819336, + "learning_rate": 0.0001, + "loss": 58.2321, + "step": 7730 + }, + { + "epoch": 124.84552845528455, + "grad_norm": 1.2980835437774658, + "learning_rate": 0.0001, + "loss": 57.5315, + "step": 7740 + }, + { + "epoch": 125.0, + "grad_norm": 1.0550237894058228, + "learning_rate": 0.0001, + "loss": 56.9615, + "step": 7750 + }, + { + "epoch": 125.16260162601625, + "grad_norm": 1.1248353719711304, + "learning_rate": 0.0001, + "loss": 58.4853, + "step": 7760 + }, + { + "epoch": 125.32520325203252, + "grad_norm": 1.338152289390564, + "learning_rate": 0.0001, + "loss": 57.8604, + "step": 7770 + }, + { + "epoch": 125.48780487804878, + "grad_norm": 0.8162221908569336, + "learning_rate": 0.0001, + "loss": 56.6396, + "step": 7780 + }, + { + "epoch": 125.65040650406505, + "grad_norm": 1.0431848764419556, + "learning_rate": 0.0001, + "loss": 58.9896, + "step": 7790 + }, + { + "epoch": 125.8130081300813, + "grad_norm": 0.848052978515625, + "learning_rate": 0.0001, + "loss": 57.7459, + "step": 7800 + }, + { + "epoch": 125.97560975609755, + "grad_norm": 0.8441061973571777, + "learning_rate": 0.0001, + "loss": 57.9488, + "step": 7810 + }, + { + "epoch": 126.130081300813, + "grad_norm": 0.774566650390625, + "learning_rate": 0.0001, + "loss": 55.9949, + "step": 7820 + }, + { + "epoch": 126.29268292682927, + "grad_norm": 1.178572416305542, + "learning_rate": 0.0001, + "loss": 58.3554, + "step": 7830 + }, + { + "epoch": 126.45528455284553, + "grad_norm": 0.9363889098167419, + "learning_rate": 0.0001, + "loss": 60.6901, + "step": 7840 + }, + { + "epoch": 126.6178861788618, + "grad_norm": 1.0010961294174194, + "learning_rate": 0.0001, + "loss": 57.0078, + "step": 7850 + }, + { + "epoch": 126.78048780487805, + "grad_norm": 0.9573602080345154, + "learning_rate": 0.0001, + "loss": 58.0567, + "step": 7860 + }, + { + "epoch": 126.9430894308943, + "grad_norm": 1.1903672218322754, + "learning_rate": 0.0001, + "loss": 58.3699, + "step": 7870 + }, + { + "epoch": 127.09756097560975, + "grad_norm": 1.2158106565475464, + "learning_rate": 0.0001, + "loss": 57.1154, + "step": 7880 + }, + { + "epoch": 127.26016260162602, + "grad_norm": 1.198789119720459, + "learning_rate": 0.0001, + "loss": 59.1004, + "step": 7890 + }, + { + "epoch": 127.42276422764228, + "grad_norm": 0.8893532752990723, + "learning_rate": 0.0001, + "loss": 59.6481, + "step": 7900 + }, + { + "epoch": 127.58536585365853, + "grad_norm": 0.9834772348403931, + "learning_rate": 0.0001, + "loss": 57.4394, + "step": 7910 + }, + { + "epoch": 127.7479674796748, + "grad_norm": 1.1957768201828003, + "learning_rate": 0.0001, + "loss": 59.1622, + "step": 7920 + }, + { + "epoch": 127.91056910569105, + "grad_norm": 0.822515070438385, + "learning_rate": 0.0001, + "loss": 58.4967, + "step": 7930 + }, + { + "epoch": 128.0650406504065, + "grad_norm": 1.0374864339828491, + "learning_rate": 0.0001, + "loss": 53.9391, + "step": 7940 + }, + { + "epoch": 128.22764227642276, + "grad_norm": 1.1767654418945312, + "learning_rate": 0.0001, + "loss": 59.4305, + "step": 7950 + }, + { + "epoch": 128.390243902439, + "grad_norm": 1.0048503875732422, + "learning_rate": 0.0001, + "loss": 59.7954, + "step": 7960 + }, + { + "epoch": 128.5528455284553, + "grad_norm": 0.9039503931999207, + "learning_rate": 0.0001, + "loss": 58.6945, + "step": 7970 + }, + { + "epoch": 128.71544715447155, + "grad_norm": 1.047418236732483, + "learning_rate": 0.0001, + "loss": 58.6324, + "step": 7980 + }, + { + "epoch": 128.8780487804878, + "grad_norm": 1.7829573154449463, + "learning_rate": 0.0001, + "loss": 57.3699, + "step": 7990 + }, + { + "epoch": 129.03252032520325, + "grad_norm": 0.8874698281288147, + "learning_rate": 0.0001, + "loss": 57.8226, + "step": 8000 + }, + { + "epoch": 129.1951219512195, + "grad_norm": 0.8585357069969177, + "learning_rate": 0.0001, + "loss": 59.1828, + "step": 8010 + }, + { + "epoch": 129.35772357723576, + "grad_norm": 0.7905164957046509, + "learning_rate": 0.0001, + "loss": 58.4257, + "step": 8020 + }, + { + "epoch": 129.52032520325204, + "grad_norm": 1.15513014793396, + "learning_rate": 0.0001, + "loss": 57.712, + "step": 8030 + }, + { + "epoch": 129.6829268292683, + "grad_norm": 1.4509782791137695, + "learning_rate": 0.0001, + "loss": 59.965, + "step": 8040 + }, + { + "epoch": 129.84552845528455, + "grad_norm": 0.8998057246208191, + "learning_rate": 0.0001, + "loss": 58.6614, + "step": 8050 + }, + { + "epoch": 130.0, + "grad_norm": 1.3381586074829102, + "learning_rate": 0.0001, + "loss": 55.2756, + "step": 8060 + }, + { + "epoch": 130.16260162601625, + "grad_norm": 0.9927449226379395, + "learning_rate": 0.0001, + "loss": 59.6809, + "step": 8070 + }, + { + "epoch": 130.3252032520325, + "grad_norm": 0.9900999665260315, + "learning_rate": 0.0001, + "loss": 60.0017, + "step": 8080 + }, + { + "epoch": 130.4878048780488, + "grad_norm": 0.8317362070083618, + "learning_rate": 0.0001, + "loss": 58.2334, + "step": 8090 + }, + { + "epoch": 130.65040650406505, + "grad_norm": 1.0413036346435547, + "learning_rate": 0.0001, + "loss": 58.987, + "step": 8100 + }, + { + "epoch": 130.8130081300813, + "grad_norm": 1.0915170907974243, + "learning_rate": 0.0001, + "loss": 58.9502, + "step": 8110 + }, + { + "epoch": 130.97560975609755, + "grad_norm": 0.9717564582824707, + "learning_rate": 0.0001, + "loss": 58.6578, + "step": 8120 + }, + { + "epoch": 131.130081300813, + "grad_norm": 0.9847933650016785, + "learning_rate": 0.0001, + "loss": 55.0067, + "step": 8130 + }, + { + "epoch": 131.29268292682926, + "grad_norm": 1.2512580156326294, + "learning_rate": 0.0001, + "loss": 56.0223, + "step": 8140 + }, + { + "epoch": 131.45528455284554, + "grad_norm": 0.8859831094741821, + "learning_rate": 0.0001, + "loss": 57.8123, + "step": 8150 + }, + { + "epoch": 131.6178861788618, + "grad_norm": 1.1545639038085938, + "learning_rate": 0.0001, + "loss": 60.2219, + "step": 8160 + }, + { + "epoch": 131.78048780487805, + "grad_norm": 1.3861064910888672, + "learning_rate": 0.0001, + "loss": 60.2156, + "step": 8170 + }, + { + "epoch": 131.9430894308943, + "grad_norm": 1.2621673345565796, + "learning_rate": 0.0001, + "loss": 58.6422, + "step": 8180 + }, + { + "epoch": 132.09756097560975, + "grad_norm": 1.2075496912002563, + "learning_rate": 0.0001, + "loss": 55.4202, + "step": 8190 + }, + { + "epoch": 132.260162601626, + "grad_norm": 1.1960432529449463, + "learning_rate": 0.0001, + "loss": 59.6405, + "step": 8200 + }, + { + "epoch": 132.4227642276423, + "grad_norm": 1.683369755744934, + "learning_rate": 0.0001, + "loss": 60.1361, + "step": 8210 + }, + { + "epoch": 132.58536585365854, + "grad_norm": 0.9033818244934082, + "learning_rate": 0.0001, + "loss": 58.2383, + "step": 8220 + }, + { + "epoch": 132.7479674796748, + "grad_norm": 1.0529452562332153, + "learning_rate": 0.0001, + "loss": 58.9644, + "step": 8230 + }, + { + "epoch": 132.91056910569105, + "grad_norm": 1.0898828506469727, + "learning_rate": 0.0001, + "loss": 57.0898, + "step": 8240 + }, + { + "epoch": 133.0650406504065, + "grad_norm": 1.0203288793563843, + "learning_rate": 0.0001, + "loss": 56.8206, + "step": 8250 + }, + { + "epoch": 133.22764227642276, + "grad_norm": 0.9973819255828857, + "learning_rate": 0.0001, + "loss": 59.2097, + "step": 8260 + }, + { + "epoch": 133.390243902439, + "grad_norm": 0.9728502035140991, + "learning_rate": 0.0001, + "loss": 57.9959, + "step": 8270 + }, + { + "epoch": 133.5528455284553, + "grad_norm": 0.8645833134651184, + "learning_rate": 0.0001, + "loss": 58.9452, + "step": 8280 + }, + { + "epoch": 133.71544715447155, + "grad_norm": 1.0179742574691772, + "learning_rate": 0.0001, + "loss": 57.8577, + "step": 8290 + }, + { + "epoch": 133.8780487804878, + "grad_norm": 0.8542449474334717, + "learning_rate": 0.0001, + "loss": 57.5798, + "step": 8300 + }, + { + "epoch": 134.03252032520325, + "grad_norm": 0.9958596229553223, + "learning_rate": 0.0001, + "loss": 56.6277, + "step": 8310 + }, + { + "epoch": 134.1951219512195, + "grad_norm": 1.1476788520812988, + "learning_rate": 0.0001, + "loss": 60.8397, + "step": 8320 + }, + { + "epoch": 134.35772357723576, + "grad_norm": 0.9098069071769714, + "learning_rate": 0.0001, + "loss": 59.6647, + "step": 8330 + }, + { + "epoch": 134.52032520325204, + "grad_norm": 0.8126556277275085, + "learning_rate": 0.0001, + "loss": 57.147, + "step": 8340 + }, + { + "epoch": 134.6829268292683, + "grad_norm": 1.4846469163894653, + "learning_rate": 0.0001, + "loss": 58.1245, + "step": 8350 + }, + { + "epoch": 134.84552845528455, + "grad_norm": 1.0117210149765015, + "learning_rate": 0.0001, + "loss": 58.7183, + "step": 8360 + }, + { + "epoch": 135.0, + "grad_norm": 0.9517931938171387, + "learning_rate": 0.0001, + "loss": 57.0423, + "step": 8370 + }, + { + "epoch": 135.16260162601625, + "grad_norm": 1.0269144773483276, + "learning_rate": 0.0001, + "loss": 59.6931, + "step": 8380 + }, + { + "epoch": 135.3252032520325, + "grad_norm": 1.1544915437698364, + "learning_rate": 0.0001, + "loss": 57.4079, + "step": 8390 + }, + { + "epoch": 135.4878048780488, + "grad_norm": 1.1180702447891235, + "learning_rate": 0.0001, + "loss": 57.855, + "step": 8400 + }, + { + "epoch": 135.65040650406505, + "grad_norm": 1.2864829301834106, + "learning_rate": 0.0001, + "loss": 57.677, + "step": 8410 + }, + { + "epoch": 135.8130081300813, + "grad_norm": 1.1786772012710571, + "learning_rate": 0.0001, + "loss": 59.5238, + "step": 8420 + }, + { + "epoch": 135.97560975609755, + "grad_norm": 1.8596229553222656, + "learning_rate": 0.0001, + "loss": 57.1596, + "step": 8430 + }, + { + "epoch": 136.130081300813, + "grad_norm": 0.8087555170059204, + "learning_rate": 0.0001, + "loss": 55.8342, + "step": 8440 + }, + { + "epoch": 136.29268292682926, + "grad_norm": 0.8107356429100037, + "learning_rate": 0.0001, + "loss": 58.5965, + "step": 8450 + }, + { + "epoch": 136.45528455284554, + "grad_norm": 1.2416011095046997, + "learning_rate": 0.0001, + "loss": 57.5909, + "step": 8460 + }, + { + "epoch": 136.6178861788618, + "grad_norm": 0.9763500690460205, + "learning_rate": 0.0001, + "loss": 58.1822, + "step": 8470 + }, + { + "epoch": 136.78048780487805, + "grad_norm": 1.1418349742889404, + "learning_rate": 0.0001, + "loss": 59.3942, + "step": 8480 + }, + { + "epoch": 136.9430894308943, + "grad_norm": 1.206878900527954, + "learning_rate": 0.0001, + "loss": 58.4128, + "step": 8490 + }, + { + "epoch": 137.09756097560975, + "grad_norm": 0.861142098903656, + "learning_rate": 0.0001, + "loss": 55.8831, + "step": 8500 + }, + { + "epoch": 137.260162601626, + "grad_norm": 1.0507313013076782, + "learning_rate": 0.0001, + "loss": 59.755, + "step": 8510 + }, + { + "epoch": 137.4227642276423, + "grad_norm": 0.7070153951644897, + "learning_rate": 0.0001, + "loss": 55.6448, + "step": 8520 + }, + { + "epoch": 137.58536585365854, + "grad_norm": 0.9338926672935486, + "learning_rate": 0.0001, + "loss": 58.4242, + "step": 8530 + }, + { + "epoch": 137.7479674796748, + "grad_norm": 1.0600488185882568, + "learning_rate": 0.0001, + "loss": 59.4947, + "step": 8540 + }, + { + "epoch": 137.91056910569105, + "grad_norm": 1.069604754447937, + "learning_rate": 0.0001, + "loss": 57.5164, + "step": 8550 + }, + { + "epoch": 138.0650406504065, + "grad_norm": 0.872332751750946, + "learning_rate": 0.0001, + "loss": 56.453, + "step": 8560 + }, + { + "epoch": 138.22764227642276, + "grad_norm": 0.9670246243476868, + "learning_rate": 0.0001, + "loss": 59.6467, + "step": 8570 + }, + { + "epoch": 138.390243902439, + "grad_norm": 1.1045281887054443, + "learning_rate": 0.0001, + "loss": 59.3334, + "step": 8580 + }, + { + "epoch": 138.5528455284553, + "grad_norm": 1.0553908348083496, + "learning_rate": 0.0001, + "loss": 59.9081, + "step": 8590 + }, + { + "epoch": 138.71544715447155, + "grad_norm": 0.7342197895050049, + "learning_rate": 0.0001, + "loss": 60.2319, + "step": 8600 + }, + { + "epoch": 138.8780487804878, + "grad_norm": 0.9574354290962219, + "learning_rate": 0.0001, + "loss": 56.7984, + "step": 8610 + }, + { + "epoch": 139.03252032520325, + "grad_norm": 0.9238156676292419, + "learning_rate": 0.0001, + "loss": 56.0007, + "step": 8620 + }, + { + "epoch": 139.1951219512195, + "grad_norm": 1.0704458951950073, + "learning_rate": 0.0001, + "loss": 59.4573, + "step": 8630 + }, + { + "epoch": 139.35772357723576, + "grad_norm": 0.8392401337623596, + "learning_rate": 0.0001, + "loss": 57.7589, + "step": 8640 + }, + { + "epoch": 139.52032520325204, + "grad_norm": 0.9042978286743164, + "learning_rate": 0.0001, + "loss": 59.0548, + "step": 8650 + }, + { + "epoch": 139.6829268292683, + "grad_norm": 2.024853467941284, + "learning_rate": 0.0001, + "loss": 58.0019, + "step": 8660 + }, + { + "epoch": 139.84552845528455, + "grad_norm": 1.9823323488235474, + "learning_rate": 0.0001, + "loss": 57.4398, + "step": 8670 + }, + { + "epoch": 140.0, + "grad_norm": 1.626451849937439, + "learning_rate": 0.0001, + "loss": 54.969, + "step": 8680 + }, + { + "epoch": 140.16260162601625, + "grad_norm": 2.435973644256592, + "learning_rate": 0.0001, + "loss": 57.8945, + "step": 8690 + }, + { + "epoch": 140.3252032520325, + "grad_norm": 1.0417332649230957, + "learning_rate": 0.0001, + "loss": 59.9172, + "step": 8700 + }, + { + "epoch": 140.4878048780488, + "grad_norm": 1.0246574878692627, + "learning_rate": 0.0001, + "loss": 57.7337, + "step": 8710 + }, + { + "epoch": 140.65040650406505, + "grad_norm": 0.9492414593696594, + "learning_rate": 0.0001, + "loss": 57.6049, + "step": 8720 + }, + { + "epoch": 140.8130081300813, + "grad_norm": 1.054689645767212, + "learning_rate": 0.0001, + "loss": 56.4394, + "step": 8730 + }, + { + "epoch": 140.97560975609755, + "grad_norm": 0.7142108082771301, + "learning_rate": 0.0001, + "loss": 58.0006, + "step": 8740 + }, + { + "epoch": 141.130081300813, + "grad_norm": 1.1387385129928589, + "learning_rate": 0.0001, + "loss": 55.2639, + "step": 8750 + }, + { + "epoch": 141.29268292682926, + "grad_norm": 2.105823040008545, + "learning_rate": 0.0001, + "loss": 59.3938, + "step": 8760 + }, + { + "epoch": 141.45528455284554, + "grad_norm": 0.9188179969787598, + "learning_rate": 0.0001, + "loss": 58.731, + "step": 8770 + }, + { + "epoch": 141.6178861788618, + "grad_norm": 1.043778896331787, + "learning_rate": 0.0001, + "loss": 56.2486, + "step": 8780 + }, + { + "epoch": 141.78048780487805, + "grad_norm": 1.438591718673706, + "learning_rate": 0.0001, + "loss": 58.4228, + "step": 8790 + }, + { + "epoch": 141.9430894308943, + "grad_norm": 1.116957187652588, + "learning_rate": 0.0001, + "loss": 59.2317, + "step": 8800 + }, + { + "epoch": 142.09756097560975, + "grad_norm": 0.8644933104515076, + "learning_rate": 0.0001, + "loss": 55.3905, + "step": 8810 + }, + { + "epoch": 142.260162601626, + "grad_norm": 1.0249485969543457, + "learning_rate": 0.0001, + "loss": 59.0372, + "step": 8820 + }, + { + "epoch": 142.4227642276423, + "grad_norm": 0.9939082264900208, + "learning_rate": 0.0001, + "loss": 57.3947, + "step": 8830 + }, + { + "epoch": 142.58536585365854, + "grad_norm": 0.9836716055870056, + "learning_rate": 0.0001, + "loss": 57.9229, + "step": 8840 + }, + { + "epoch": 142.7479674796748, + "grad_norm": 0.9596876502037048, + "learning_rate": 0.0001, + "loss": 57.0285, + "step": 8850 + }, + { + "epoch": 142.91056910569105, + "grad_norm": 1.052899956703186, + "learning_rate": 0.0001, + "loss": 60.1384, + "step": 8860 + }, + { + "epoch": 143.0650406504065, + "grad_norm": 1.1220297813415527, + "learning_rate": 0.0001, + "loss": 55.3928, + "step": 8870 + }, + { + "epoch": 143.22764227642276, + "grad_norm": 0.8395596742630005, + "learning_rate": 0.0001, + "loss": 57.8185, + "step": 8880 + }, + { + "epoch": 143.390243902439, + "grad_norm": 1.1940062046051025, + "learning_rate": 0.0001, + "loss": 58.0322, + "step": 8890 + }, + { + "epoch": 143.5528455284553, + "grad_norm": 1.1662484407424927, + "learning_rate": 0.0001, + "loss": 58.2746, + "step": 8900 + }, + { + "epoch": 143.71544715447155, + "grad_norm": 0.9028235077857971, + "learning_rate": 0.0001, + "loss": 56.5817, + "step": 8910 + }, + { + "epoch": 143.8780487804878, + "grad_norm": 1.0597296953201294, + "learning_rate": 0.0001, + "loss": 56.8426, + "step": 8920 + }, + { + "epoch": 144.03252032520325, + "grad_norm": 0.9968701004981995, + "learning_rate": 0.0001, + "loss": 54.9727, + "step": 8930 + }, + { + "epoch": 144.1951219512195, + "grad_norm": 0.9306491613388062, + "learning_rate": 0.0001, + "loss": 56.5321, + "step": 8940 + }, + { + "epoch": 144.35772357723576, + "grad_norm": 1.1079609394073486, + "learning_rate": 0.0001, + "loss": 56.9714, + "step": 8950 + }, + { + "epoch": 144.52032520325204, + "grad_norm": 0.9644935131072998, + "learning_rate": 0.0001, + "loss": 58.613, + "step": 8960 + }, + { + "epoch": 144.6829268292683, + "grad_norm": 1.2037341594696045, + "learning_rate": 0.0001, + "loss": 56.7768, + "step": 8970 + }, + { + "epoch": 144.84552845528455, + "grad_norm": 0.7347294092178345, + "learning_rate": 0.0001, + "loss": 58.5653, + "step": 8980 + }, + { + "epoch": 145.0, + "grad_norm": 0.8877835273742676, + "learning_rate": 0.0001, + "loss": 53.7768, + "step": 8990 + }, + { + "epoch": 145.16260162601625, + "grad_norm": 0.9649038910865784, + "learning_rate": 0.0001, + "loss": 57.7391, + "step": 9000 + }, + { + "epoch": 145.3252032520325, + "grad_norm": 1.1804879903793335, + "learning_rate": 0.0001, + "loss": 60.3727, + "step": 9010 + }, + { + "epoch": 145.4878048780488, + "grad_norm": 0.8406301140785217, + "learning_rate": 0.0001, + "loss": 57.9417, + "step": 9020 + }, + { + "epoch": 145.65040650406505, + "grad_norm": 0.9555912613868713, + "learning_rate": 0.0001, + "loss": 58.0936, + "step": 9030 + }, + { + "epoch": 145.8130081300813, + "grad_norm": 1.20796537399292, + "learning_rate": 0.0001, + "loss": 57.0784, + "step": 9040 + }, + { + "epoch": 145.97560975609755, + "grad_norm": 1.0527915954589844, + "learning_rate": 0.0001, + "loss": 56.752, + "step": 9050 + }, + { + "epoch": 146.130081300813, + "grad_norm": 1.0104299783706665, + "learning_rate": 0.0001, + "loss": 56.2158, + "step": 9060 + }, + { + "epoch": 146.29268292682926, + "grad_norm": 1.4884357452392578, + "learning_rate": 0.0001, + "loss": 57.8631, + "step": 9070 + }, + { + "epoch": 146.45528455284554, + "grad_norm": 0.9095257520675659, + "learning_rate": 0.0001, + "loss": 57.6492, + "step": 9080 + }, + { + "epoch": 146.6178861788618, + "grad_norm": 0.8465835452079773, + "learning_rate": 0.0001, + "loss": 57.78, + "step": 9090 + }, + { + "epoch": 146.78048780487805, + "grad_norm": 1.0254603624343872, + "learning_rate": 0.0001, + "loss": 57.7678, + "step": 9100 + }, + { + "epoch": 146.9430894308943, + "grad_norm": 1.0137311220169067, + "learning_rate": 0.0001, + "loss": 58.2813, + "step": 9110 + }, + { + "epoch": 147.09756097560975, + "grad_norm": 0.9816886186599731, + "learning_rate": 0.0001, + "loss": 54.4789, + "step": 9120 + }, + { + "epoch": 147.260162601626, + "grad_norm": 0.9038514494895935, + "learning_rate": 0.0001, + "loss": 58.5366, + "step": 9130 + }, + { + "epoch": 147.4227642276423, + "grad_norm": 0.7649261355400085, + "learning_rate": 0.0001, + "loss": 57.2898, + "step": 9140 + }, + { + "epoch": 147.58536585365854, + "grad_norm": 0.8841894865036011, + "learning_rate": 0.0001, + "loss": 57.9109, + "step": 9150 + }, + { + "epoch": 147.7479674796748, + "grad_norm": 1.0480542182922363, + "learning_rate": 0.0001, + "loss": 56.9625, + "step": 9160 + }, + { + "epoch": 147.91056910569105, + "grad_norm": 0.8710455894470215, + "learning_rate": 0.0001, + "loss": 57.1721, + "step": 9170 + }, + { + "epoch": 148.0650406504065, + "grad_norm": 1.374118447303772, + "learning_rate": 0.0001, + "loss": 55.3243, + "step": 9180 + }, + { + "epoch": 148.22764227642276, + "grad_norm": 1.0879178047180176, + "learning_rate": 0.0001, + "loss": 57.1631, + "step": 9190 + }, + { + "epoch": 148.390243902439, + "grad_norm": 0.8242596387863159, + "learning_rate": 0.0001, + "loss": 56.9993, + "step": 9200 + }, + { + "epoch": 148.5528455284553, + "grad_norm": 1.02505362033844, + "learning_rate": 0.0001, + "loss": 57.9201, + "step": 9210 + }, + { + "epoch": 148.71544715447155, + "grad_norm": 0.7904821038246155, + "learning_rate": 0.0001, + "loss": 59.2553, + "step": 9220 + }, + { + "epoch": 148.8780487804878, + "grad_norm": 1.4904934167861938, + "learning_rate": 0.0001, + "loss": 59.498, + "step": 9230 + }, + { + "epoch": 149.03252032520325, + "grad_norm": 1.4203144311904907, + "learning_rate": 0.0001, + "loss": 54.6357, + "step": 9240 + }, + { + "epoch": 149.1951219512195, + "grad_norm": 0.8256629705429077, + "learning_rate": 0.0001, + "loss": 57.5996, + "step": 9250 + }, + { + "epoch": 149.35772357723576, + "grad_norm": 0.8736788630485535, + "learning_rate": 0.0001, + "loss": 56.8112, + "step": 9260 + }, + { + "epoch": 149.52032520325204, + "grad_norm": 3.20312762260437, + "learning_rate": 0.0001, + "loss": 55.8763, + "step": 9270 + }, + { + "epoch": 149.6829268292683, + "grad_norm": 0.7431173920631409, + "learning_rate": 0.0001, + "loss": 56.7385, + "step": 9280 + }, + { + "epoch": 149.84552845528455, + "grad_norm": 0.8165793418884277, + "learning_rate": 0.0001, + "loss": 57.8441, + "step": 9290 + }, + { + "epoch": 150.0, + "grad_norm": 1.2387259006500244, + "learning_rate": 0.0001, + "loss": 53.47, + "step": 9300 + }, + { + "epoch": 150.16260162601625, + "grad_norm": 0.8952508568763733, + "learning_rate": 0.0001, + "loss": 57.7642, + "step": 9310 + }, + { + "epoch": 150.3252032520325, + "grad_norm": 0.9837163686752319, + "learning_rate": 0.0001, + "loss": 57.2288, + "step": 9320 + }, + { + "epoch": 150.4878048780488, + "grad_norm": 0.9019481539726257, + "learning_rate": 0.0001, + "loss": 57.4239, + "step": 9330 + }, + { + "epoch": 150.65040650406505, + "grad_norm": 1.0404818058013916, + "learning_rate": 0.0001, + "loss": 57.7678, + "step": 9340 + }, + { + "epoch": 150.8130081300813, + "grad_norm": 1.0074576139450073, + "learning_rate": 0.0001, + "loss": 58.1508, + "step": 9350 + }, + { + "epoch": 150.97560975609755, + "grad_norm": 0.9951609969139099, + "learning_rate": 0.0001, + "loss": 56.1586, + "step": 9360 + }, + { + "epoch": 151.130081300813, + "grad_norm": 1.0799294710159302, + "learning_rate": 0.0001, + "loss": 52.2597, + "step": 9370 + }, + { + "epoch": 151.29268292682926, + "grad_norm": 0.9001291394233704, + "learning_rate": 0.0001, + "loss": 58.1814, + "step": 9380 + }, + { + "epoch": 151.45528455284554, + "grad_norm": 1.0388239622116089, + "learning_rate": 0.0001, + "loss": 57.1812, + "step": 9390 + }, + { + "epoch": 151.6178861788618, + "grad_norm": 0.7662557363510132, + "learning_rate": 0.0001, + "loss": 56.087, + "step": 9400 + }, + { + "epoch": 151.78048780487805, + "grad_norm": 1.0790860652923584, + "learning_rate": 0.0001, + "loss": 55.9247, + "step": 9410 + }, + { + "epoch": 151.9430894308943, + "grad_norm": 0.8173624277114868, + "learning_rate": 0.0001, + "loss": 57.5506, + "step": 9420 + }, + { + "epoch": 152.09756097560975, + "grad_norm": 1.0954891443252563, + "learning_rate": 0.0001, + "loss": 54.6799, + "step": 9430 + }, + { + "epoch": 152.260162601626, + "grad_norm": 1.0103733539581299, + "learning_rate": 0.0001, + "loss": 58.5454, + "step": 9440 + }, + { + "epoch": 152.4227642276423, + "grad_norm": 0.6966420412063599, + "learning_rate": 0.0001, + "loss": 57.561, + "step": 9450 + }, + { + "epoch": 152.58536585365854, + "grad_norm": 0.9085037708282471, + "learning_rate": 0.0001, + "loss": 57.1746, + "step": 9460 + }, + { + "epoch": 152.7479674796748, + "grad_norm": 1.0009453296661377, + "learning_rate": 0.0001, + "loss": 56.8, + "step": 9470 + }, + { + "epoch": 152.91056910569105, + "grad_norm": 0.9139633178710938, + "learning_rate": 0.0001, + "loss": 56.8229, + "step": 9480 + }, + { + "epoch": 153.0650406504065, + "grad_norm": 1.4209275245666504, + "learning_rate": 0.0001, + "loss": 55.3074, + "step": 9490 + }, + { + "epoch": 153.22764227642276, + "grad_norm": 1.3912067413330078, + "learning_rate": 0.0001, + "loss": 58.1401, + "step": 9500 + }, + { + "epoch": 153.390243902439, + "grad_norm": 0.8208648562431335, + "learning_rate": 0.0001, + "loss": 55.2981, + "step": 9510 + }, + { + "epoch": 153.5528455284553, + "grad_norm": 1.1563907861709595, + "learning_rate": 0.0001, + "loss": 59.0262, + "step": 9520 + }, + { + "epoch": 153.71544715447155, + "grad_norm": 1.1142042875289917, + "learning_rate": 0.0001, + "loss": 55.5979, + "step": 9530 + }, + { + "epoch": 153.8780487804878, + "grad_norm": 0.8886147141456604, + "learning_rate": 0.0001, + "loss": 58.5592, + "step": 9540 + }, + { + "epoch": 154.03252032520325, + "grad_norm": 0.9479674696922302, + "learning_rate": 0.0001, + "loss": 53.956, + "step": 9550 + }, + { + "epoch": 154.1951219512195, + "grad_norm": 1.0572675466537476, + "learning_rate": 0.0001, + "loss": 59.5914, + "step": 9560 + }, + { + "epoch": 154.35772357723576, + "grad_norm": 0.8284531831741333, + "learning_rate": 0.0001, + "loss": 56.4313, + "step": 9570 + }, + { + "epoch": 154.52032520325204, + "grad_norm": 0.8882488012313843, + "learning_rate": 0.0001, + "loss": 57.707, + "step": 9580 + }, + { + "epoch": 154.6829268292683, + "grad_norm": 0.9115352630615234, + "learning_rate": 0.0001, + "loss": 58.0544, + "step": 9590 + }, + { + "epoch": 154.84552845528455, + "grad_norm": 1.0006799697875977, + "learning_rate": 0.0001, + "loss": 59.4528, + "step": 9600 + }, + { + "epoch": 155.0, + "grad_norm": 0.9925333261489868, + "learning_rate": 0.0001, + "loss": 52.977, + "step": 9610 + }, + { + "epoch": 155.16260162601625, + "grad_norm": 1.423391580581665, + "learning_rate": 0.0001, + "loss": 57.0083, + "step": 9620 + }, + { + "epoch": 155.3252032520325, + "grad_norm": 1.0822738409042358, + "learning_rate": 0.0001, + "loss": 58.187, + "step": 9630 + }, + { + "epoch": 155.4878048780488, + "grad_norm": 0.932689905166626, + "learning_rate": 0.0001, + "loss": 55.6032, + "step": 9640 + }, + { + "epoch": 155.65040650406505, + "grad_norm": 0.895504355430603, + "learning_rate": 0.0001, + "loss": 57.1324, + "step": 9650 + }, + { + "epoch": 155.8130081300813, + "grad_norm": 1.0883748531341553, + "learning_rate": 0.0001, + "loss": 58.8422, + "step": 9660 + }, + { + "epoch": 155.97560975609755, + "grad_norm": 1.0161819458007812, + "learning_rate": 0.0001, + "loss": 56.8333, + "step": 9670 + }, + { + "epoch": 156.130081300813, + "grad_norm": 0.9483168721199036, + "learning_rate": 0.0001, + "loss": 52.8434, + "step": 9680 + }, + { + "epoch": 156.29268292682926, + "grad_norm": 1.0677295923233032, + "learning_rate": 0.0001, + "loss": 57.749, + "step": 9690 + }, + { + "epoch": 156.45528455284554, + "grad_norm": 0.8123661875724792, + "learning_rate": 0.0001, + "loss": 56.0896, + "step": 9700 + }, + { + "epoch": 156.6178861788618, + "grad_norm": 0.7926080822944641, + "learning_rate": 0.0001, + "loss": 57.8899, + "step": 9710 + }, + { + "epoch": 156.78048780487805, + "grad_norm": 1.0433725118637085, + "learning_rate": 0.0001, + "loss": 59.6728, + "step": 9720 + }, + { + "epoch": 156.9430894308943, + "grad_norm": 0.8284909129142761, + "learning_rate": 0.0001, + "loss": 55.6093, + "step": 9730 + }, + { + "epoch": 157.09756097560975, + "grad_norm": 0.9702232480049133, + "learning_rate": 0.0001, + "loss": 55.3595, + "step": 9740 + }, + { + "epoch": 157.260162601626, + "grad_norm": 0.8909348249435425, + "learning_rate": 0.0001, + "loss": 57.6344, + "step": 9750 + }, + { + "epoch": 157.4227642276423, + "grad_norm": 0.9539676308631897, + "learning_rate": 0.0001, + "loss": 55.8826, + "step": 9760 + }, + { + "epoch": 157.58536585365854, + "grad_norm": 1.1801178455352783, + "learning_rate": 0.0001, + "loss": 55.7141, + "step": 9770 + }, + { + "epoch": 157.7479674796748, + "grad_norm": 0.8674126863479614, + "learning_rate": 0.0001, + "loss": 56.0571, + "step": 9780 + }, + { + "epoch": 157.91056910569105, + "grad_norm": 1.0690264701843262, + "learning_rate": 0.0001, + "loss": 56.1485, + "step": 9790 + }, + { + "epoch": 158.0650406504065, + "grad_norm": 0.8807869553565979, + "learning_rate": 0.0001, + "loss": 56.9874, + "step": 9800 + }, + { + "epoch": 158.22764227642276, + "grad_norm": 0.9477714896202087, + "learning_rate": 0.0001, + "loss": 58.2035, + "step": 9810 + }, + { + "epoch": 158.390243902439, + "grad_norm": 1.4875513315200806, + "learning_rate": 0.0001, + "loss": 59.0302, + "step": 9820 + }, + { + "epoch": 158.5528455284553, + "grad_norm": 0.7979354858398438, + "learning_rate": 0.0001, + "loss": 57.2801, + "step": 9830 + }, + { + "epoch": 158.71544715447155, + "grad_norm": 0.8941402435302734, + "learning_rate": 0.0001, + "loss": 56.3615, + "step": 9840 + }, + { + "epoch": 158.8780487804878, + "grad_norm": 0.8475911021232605, + "learning_rate": 0.0001, + "loss": 57.266, + "step": 9850 + }, + { + "epoch": 159.03252032520325, + "grad_norm": 0.8971277475357056, + "learning_rate": 0.0001, + "loss": 52.792, + "step": 9860 + }, + { + "epoch": 159.1951219512195, + "grad_norm": 0.930777907371521, + "learning_rate": 0.0001, + "loss": 57.4098, + "step": 9870 + }, + { + "epoch": 159.35772357723576, + "grad_norm": 0.9232887625694275, + "learning_rate": 0.0001, + "loss": 56.0267, + "step": 9880 + }, + { + "epoch": 159.52032520325204, + "grad_norm": 0.8904425501823425, + "learning_rate": 0.0001, + "loss": 57.6255, + "step": 9890 + }, + { + "epoch": 159.6829268292683, + "grad_norm": 0.9250755906105042, + "learning_rate": 0.0001, + "loss": 56.5248, + "step": 9900 + }, + { + "epoch": 159.84552845528455, + "grad_norm": 1.3746178150177002, + "learning_rate": 0.0001, + "loss": 56.7843, + "step": 9910 + }, + { + "epoch": 160.0, + "grad_norm": 0.9464873671531677, + "learning_rate": 0.0001, + "loss": 54.1325, + "step": 9920 + }, + { + "epoch": 160.16260162601625, + "grad_norm": 0.9019768238067627, + "learning_rate": 0.0001, + "loss": 58.3641, + "step": 9930 + }, + { + "epoch": 160.3252032520325, + "grad_norm": 0.9282474517822266, + "learning_rate": 0.0001, + "loss": 57.9474, + "step": 9940 + }, + { + "epoch": 160.4878048780488, + "grad_norm": 1.1150450706481934, + "learning_rate": 0.0001, + "loss": 58.2999, + "step": 9950 + }, + { + "epoch": 160.65040650406505, + "grad_norm": 0.9780807495117188, + "learning_rate": 0.0001, + "loss": 55.6409, + "step": 9960 + }, + { + "epoch": 160.8130081300813, + "grad_norm": 0.9708426594734192, + "learning_rate": 0.0001, + "loss": 58.651, + "step": 9970 + }, + { + "epoch": 160.97560975609755, + "grad_norm": 0.9891226887702942, + "learning_rate": 0.0001, + "loss": 57.3798, + "step": 9980 + }, + { + "epoch": 161.130081300813, + "grad_norm": 0.8949639797210693, + "learning_rate": 0.0001, + "loss": 53.4708, + "step": 9990 + }, + { + "epoch": 161.29268292682926, + "grad_norm": 0.831688404083252, + "learning_rate": 0.0001, + "loss": 57.1608, + "step": 10000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 323, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2146439475571143e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c7f229ebe1034caaf3e92006c8c7528c21af93d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517586849d044483051c7a59cec7c6ce389951878a7357ef837edb1cbc9983c8 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-10000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/latest new file mode 100644 index 0000000000000000000000000000000000000000..b8f4d8b87537b41ea73a42841f5fb1704b16b234 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/latest @@ -0,0 +1 @@ +global_step12000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7cbcc86c979ba7827acd06ab1ed5065cad98e2fd --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d176eecf990400b537accbe740a194d5a0f9d91b41a5a3f5b55f8946546d2dc +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..3c02975ed68964ce8670d0ac29e9e6aeea69142c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbaadde507048aaa4a60bf8ffc94e8154ffa9247069726c1907042a473e4b016 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..de8b6667bd7d5530a5ce4dec584484e890eb942e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb87e0866cf3f7f98409b3961ff4b6aa050796616357454e51946de25db8d180 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..403a7b900dbcd541a1d5c4caea9260593957feaf --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a612bfcdecc3bb660f0fbab667dd23d5502ffb94f1d678e0c9398ce834620318 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..4457ff03620a7f37f7b8fda00cdf1c9db4e6791a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac5e5852236c67551a1d2027fe382298525bf382283b45785b4627df3826e607 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..9b46a11952daa2d5e99de6f65e13c9817ef97a45 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d678f374f0495926a4d552a70b7ce8f51dfcf5ed56141f9c1ed2d1ef2c0e5c2 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..dfc1afa8e3f24c4e7558262d63ee4af6375ad0a8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91b1d9c9d9925603b20b2d7c759df6c406caf42029feb7c2469b460b09314777 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..ee71031dff9e21f0038790e1155602740daa18ad --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b21658c832b5e6f9cd0cb1603ec5fcd8e338f3ee2d063a9e0c62aa2a35092cf +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..db404c5165a108ac4a2c114da212a69caeafc649 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb7a5202b85965551035f8bf834edd7e4be5be9b1b324ef36607910decdb2277 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf54a839aaa564cb41c7b758f7e7846a4a7a40d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b65dd4f7c4351bc70e2ca42eddd7e6f1c0fdc194263d9f4061210a8ec7559c0e +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..231b0a943d51c0de28ce2118c9b2e082d0e1e21b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/trainer_state.json @@ -0,0 +1,8434 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 193.5528455284553, + "eval_steps": 2000, + "global_step": 12000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.16260162601626016, + "grad_norm": 23.72986602783203, + "learning_rate": 0.0001, + "loss": 271.6598, + "step": 10 + }, + { + "epoch": 0.3252032520325203, + "grad_norm": 18.99613380432129, + "learning_rate": 0.0001, + "loss": 233.029, + "step": 20 + }, + { + "epoch": 0.4878048780487805, + "grad_norm": 3.9815304279327393, + "learning_rate": 0.0001, + "loss": 231.0776, + "step": 30 + }, + { + "epoch": 0.6504065040650406, + "grad_norm": 4.641753196716309, + "learning_rate": 0.0001, + "loss": 228.6894, + "step": 40 + }, + { + "epoch": 0.8130081300813008, + "grad_norm": 3.000852108001709, + "learning_rate": 0.0001, + "loss": 228.4694, + "step": 50 + }, + { + "epoch": 0.975609756097561, + "grad_norm": 12.213018417358398, + "learning_rate": 0.0001, + "loss": 226.4258, + "step": 60 + }, + { + "epoch": 1.1300813008130082, + "grad_norm": 2.535187005996704, + "learning_rate": 0.0001, + "loss": 209.9772, + "step": 70 + }, + { + "epoch": 1.2926829268292683, + "grad_norm": 2.0984582901000977, + "learning_rate": 0.0001, + "loss": 217.2421, + "step": 80 + }, + { + "epoch": 1.4552845528455285, + "grad_norm": 3.4802305698394775, + "learning_rate": 0.0001, + "loss": 210.9241, + "step": 90 + }, + { + "epoch": 1.6178861788617886, + "grad_norm": 3.813626527786255, + "learning_rate": 0.0001, + "loss": 205.7566, + "step": 100 + }, + { + "epoch": 1.7804878048780488, + "grad_norm": 2.6378512382507324, + "learning_rate": 0.0001, + "loss": 201.8789, + "step": 110 + }, + { + "epoch": 1.943089430894309, + "grad_norm": 2.7502875328063965, + "learning_rate": 0.0001, + "loss": 196.373, + "step": 120 + }, + { + "epoch": 2.097560975609756, + "grad_norm": 2.659212112426758, + "learning_rate": 0.0001, + "loss": 183.1456, + "step": 130 + }, + { + "epoch": 2.2601626016260163, + "grad_norm": 2.513044595718384, + "learning_rate": 0.0001, + "loss": 190.1661, + "step": 140 + }, + { + "epoch": 2.4227642276422765, + "grad_norm": 2.3192145824432373, + "learning_rate": 0.0001, + "loss": 184.9997, + "step": 150 + }, + { + "epoch": 2.5853658536585367, + "grad_norm": 3.056708574295044, + "learning_rate": 0.0001, + "loss": 182.0164, + "step": 160 + }, + { + "epoch": 2.747967479674797, + "grad_norm": 2.9687108993530273, + "learning_rate": 0.0001, + "loss": 179.6539, + "step": 170 + }, + { + "epoch": 2.910569105691057, + "grad_norm": 2.1498093605041504, + "learning_rate": 0.0001, + "loss": 176.7159, + "step": 180 + }, + { + "epoch": 3.065040650406504, + "grad_norm": 3.0103771686553955, + "learning_rate": 0.0001, + "loss": 166.1703, + "step": 190 + }, + { + "epoch": 3.227642276422764, + "grad_norm": 2.992462635040283, + "learning_rate": 0.0001, + "loss": 171.124, + "step": 200 + }, + { + "epoch": 3.3902439024390243, + "grad_norm": 4.8617119789123535, + "learning_rate": 0.0001, + "loss": 168.2646, + "step": 210 + }, + { + "epoch": 3.5528455284552845, + "grad_norm": 3.1047444343566895, + "learning_rate": 0.0001, + "loss": 163.0889, + "step": 220 + }, + { + "epoch": 3.7154471544715446, + "grad_norm": 6.473227500915527, + "learning_rate": 0.0001, + "loss": 151.3667, + "step": 230 + }, + { + "epoch": 3.8780487804878048, + "grad_norm": 6.16747522354126, + "learning_rate": 0.0001, + "loss": 137.5915, + "step": 240 + }, + { + "epoch": 4.032520325203252, + "grad_norm": 7.994515419006348, + "learning_rate": 0.0001, + "loss": 118.8875, + "step": 250 + }, + { + "epoch": 4.195121951219512, + "grad_norm": 5.882101058959961, + "learning_rate": 0.0001, + "loss": 119.4793, + "step": 260 + }, + { + "epoch": 4.357723577235772, + "grad_norm": 12.32968521118164, + "learning_rate": 0.0001, + "loss": 113.1209, + "step": 270 + }, + { + "epoch": 4.520325203252033, + "grad_norm": 5.697225570678711, + "learning_rate": 0.0001, + "loss": 107.51, + "step": 280 + }, + { + "epoch": 4.682926829268292, + "grad_norm": 4.654167175292969, + "learning_rate": 0.0001, + "loss": 107.0172, + "step": 290 + }, + { + "epoch": 4.845528455284553, + "grad_norm": 2.9595484733581543, + "learning_rate": 0.0001, + "loss": 100.9164, + "step": 300 + }, + { + "epoch": 5.0, + "grad_norm": 21.242591857910156, + "learning_rate": 0.0001, + "loss": 94.9884, + "step": 310 + }, + { + "epoch": 5.16260162601626, + "grad_norm": 5.760314464569092, + "learning_rate": 0.0001, + "loss": 97.5901, + "step": 320 + }, + { + "epoch": 5.32520325203252, + "grad_norm": 3.1373329162597656, + "learning_rate": 0.0001, + "loss": 94.8485, + "step": 330 + }, + { + "epoch": 5.487804878048781, + "grad_norm": 2.3777568340301514, + "learning_rate": 0.0001, + "loss": 96.9982, + "step": 340 + }, + { + "epoch": 5.650406504065041, + "grad_norm": 3.0607948303222656, + "learning_rate": 0.0001, + "loss": 96.1586, + "step": 350 + }, + { + "epoch": 5.8130081300813, + "grad_norm": 4.168269634246826, + "learning_rate": 0.0001, + "loss": 94.5743, + "step": 360 + }, + { + "epoch": 5.975609756097561, + "grad_norm": 2.4383699893951416, + "learning_rate": 0.0001, + "loss": 95.5197, + "step": 370 + }, + { + "epoch": 6.130081300813008, + "grad_norm": 2.302259683609009, + "learning_rate": 0.0001, + "loss": 85.6599, + "step": 380 + }, + { + "epoch": 6.2926829268292686, + "grad_norm": 2.552753210067749, + "learning_rate": 0.0001, + "loss": 91.2999, + "step": 390 + }, + { + "epoch": 6.455284552845528, + "grad_norm": 2.2092080116271973, + "learning_rate": 0.0001, + "loss": 90.7041, + "step": 400 + }, + { + "epoch": 6.617886178861789, + "grad_norm": 2.2935690879821777, + "learning_rate": 0.0001, + "loss": 89.9949, + "step": 410 + }, + { + "epoch": 6.780487804878049, + "grad_norm": 2.487510919570923, + "learning_rate": 0.0001, + "loss": 86.9056, + "step": 420 + }, + { + "epoch": 6.943089430894309, + "grad_norm": 2.8630545139312744, + "learning_rate": 0.0001, + "loss": 88.3347, + "step": 430 + }, + { + "epoch": 7.097560975609756, + "grad_norm": 5.87056303024292, + "learning_rate": 0.0001, + "loss": 84.9632, + "step": 440 + }, + { + "epoch": 7.260162601626016, + "grad_norm": 2.587661027908325, + "learning_rate": 0.0001, + "loss": 86.2024, + "step": 450 + }, + { + "epoch": 7.4227642276422765, + "grad_norm": 3.5671255588531494, + "learning_rate": 0.0001, + "loss": 86.218, + "step": 460 + }, + { + "epoch": 7.585365853658536, + "grad_norm": 2.5036680698394775, + "learning_rate": 0.0001, + "loss": 86.5914, + "step": 470 + }, + { + "epoch": 7.747967479674797, + "grad_norm": 3.2656471729278564, + "learning_rate": 0.0001, + "loss": 82.6081, + "step": 480 + }, + { + "epoch": 7.9105691056910565, + "grad_norm": 2.5053915977478027, + "learning_rate": 0.0001, + "loss": 84.6116, + "step": 490 + }, + { + "epoch": 8.065040650406504, + "grad_norm": 2.1669957637786865, + "learning_rate": 0.0001, + "loss": 80.4354, + "step": 500 + }, + { + "epoch": 8.227642276422765, + "grad_norm": 4.2003703117370605, + "learning_rate": 0.0001, + "loss": 85.2286, + "step": 510 + }, + { + "epoch": 8.390243902439025, + "grad_norm": 3.1909608840942383, + "learning_rate": 0.0001, + "loss": 85.0358, + "step": 520 + }, + { + "epoch": 8.552845528455284, + "grad_norm": 1.8420121669769287, + "learning_rate": 0.0001, + "loss": 85.4647, + "step": 530 + }, + { + "epoch": 8.715447154471544, + "grad_norm": 1.9822801351547241, + "learning_rate": 0.0001, + "loss": 84.7134, + "step": 540 + }, + { + "epoch": 8.878048780487806, + "grad_norm": 2.936758518218994, + "learning_rate": 0.0001, + "loss": 83.8068, + "step": 550 + }, + { + "epoch": 9.032520325203253, + "grad_norm": 1.5713881254196167, + "learning_rate": 0.0001, + "loss": 80.0695, + "step": 560 + }, + { + "epoch": 9.195121951219512, + "grad_norm": 2.0584616661071777, + "learning_rate": 0.0001, + "loss": 84.5158, + "step": 570 + }, + { + "epoch": 9.357723577235772, + "grad_norm": 2.101391553878784, + "learning_rate": 0.0001, + "loss": 82.4719, + "step": 580 + }, + { + "epoch": 9.520325203252032, + "grad_norm": 1.9956543445587158, + "learning_rate": 0.0001, + "loss": 81.6715, + "step": 590 + }, + { + "epoch": 9.682926829268293, + "grad_norm": 2.7622063159942627, + "learning_rate": 0.0001, + "loss": 85.4178, + "step": 600 + }, + { + "epoch": 9.845528455284553, + "grad_norm": 2.2084789276123047, + "learning_rate": 0.0001, + "loss": 84.5702, + "step": 610 + }, + { + "epoch": 10.0, + "grad_norm": 1.9635663032531738, + "learning_rate": 0.0001, + "loss": 77.3838, + "step": 620 + }, + { + "epoch": 10.16260162601626, + "grad_norm": 2.5895581245422363, + "learning_rate": 0.0001, + "loss": 81.723, + "step": 630 + }, + { + "epoch": 10.32520325203252, + "grad_norm": 2.743727445602417, + "learning_rate": 0.0001, + "loss": 83.1332, + "step": 640 + }, + { + "epoch": 10.487804878048781, + "grad_norm": 2.024585008621216, + "learning_rate": 0.0001, + "loss": 81.5968, + "step": 650 + }, + { + "epoch": 10.65040650406504, + "grad_norm": 1.8254153728485107, + "learning_rate": 0.0001, + "loss": 78.9589, + "step": 660 + }, + { + "epoch": 10.8130081300813, + "grad_norm": 3.2015740871429443, + "learning_rate": 0.0001, + "loss": 79.1829, + "step": 670 + }, + { + "epoch": 10.975609756097562, + "grad_norm": 1.788453459739685, + "learning_rate": 0.0001, + "loss": 81.527, + "step": 680 + }, + { + "epoch": 11.130081300813009, + "grad_norm": 1.5744423866271973, + "learning_rate": 0.0001, + "loss": 74.9869, + "step": 690 + }, + { + "epoch": 11.292682926829269, + "grad_norm": 2.006002187728882, + "learning_rate": 0.0001, + "loss": 80.2677, + "step": 700 + }, + { + "epoch": 11.455284552845528, + "grad_norm": 1.8847148418426514, + "learning_rate": 0.0001, + "loss": 78.8709, + "step": 710 + }, + { + "epoch": 11.617886178861788, + "grad_norm": 1.8384478092193604, + "learning_rate": 0.0001, + "loss": 79.008, + "step": 720 + }, + { + "epoch": 11.78048780487805, + "grad_norm": 1.9486687183380127, + "learning_rate": 0.0001, + "loss": 76.1525, + "step": 730 + }, + { + "epoch": 11.94308943089431, + "grad_norm": 1.8644076585769653, + "learning_rate": 0.0001, + "loss": 77.5802, + "step": 740 + }, + { + "epoch": 12.097560975609756, + "grad_norm": 1.6035499572753906, + "learning_rate": 0.0001, + "loss": 75.2422, + "step": 750 + }, + { + "epoch": 12.260162601626016, + "grad_norm": 1.6935622692108154, + "learning_rate": 0.0001, + "loss": 77.5838, + "step": 760 + }, + { + "epoch": 12.422764227642276, + "grad_norm": 1.9327987432479858, + "learning_rate": 0.0001, + "loss": 78.3237, + "step": 770 + }, + { + "epoch": 12.585365853658537, + "grad_norm": 1.7903852462768555, + "learning_rate": 0.0001, + "loss": 78.7459, + "step": 780 + }, + { + "epoch": 12.747967479674797, + "grad_norm": 1.4962575435638428, + "learning_rate": 0.0001, + "loss": 78.9193, + "step": 790 + }, + { + "epoch": 12.910569105691057, + "grad_norm": 42.10435104370117, + "learning_rate": 0.0001, + "loss": 80.1429, + "step": 800 + }, + { + "epoch": 13.065040650406504, + "grad_norm": 1.904065728187561, + "learning_rate": 0.0001, + "loss": 73.7798, + "step": 810 + }, + { + "epoch": 13.227642276422765, + "grad_norm": 1.8768481016159058, + "learning_rate": 0.0001, + "loss": 76.8692, + "step": 820 + }, + { + "epoch": 13.390243902439025, + "grad_norm": 16.140565872192383, + "learning_rate": 0.0001, + "loss": 78.122, + "step": 830 + }, + { + "epoch": 13.552845528455284, + "grad_norm": 2.160288095474243, + "learning_rate": 0.0001, + "loss": 76.2169, + "step": 840 + }, + { + "epoch": 13.715447154471544, + "grad_norm": 27.682231903076172, + "learning_rate": 0.0001, + "loss": 76.6266, + "step": 850 + }, + { + "epoch": 13.878048780487806, + "grad_norm": 1.779772162437439, + "learning_rate": 0.0001, + "loss": 80.0434, + "step": 860 + }, + { + "epoch": 14.032520325203253, + "grad_norm": 1.6516485214233398, + "learning_rate": 0.0001, + "loss": 73.3943, + "step": 870 + }, + { + "epoch": 14.195121951219512, + "grad_norm": 1.846498966217041, + "learning_rate": 0.0001, + "loss": 77.5157, + "step": 880 + }, + { + "epoch": 14.357723577235772, + "grad_norm": 1.942946434020996, + "learning_rate": 0.0001, + "loss": 77.6944, + "step": 890 + }, + { + "epoch": 14.520325203252032, + "grad_norm": 2.0929224491119385, + "learning_rate": 0.0001, + "loss": 76.9411, + "step": 900 + }, + { + "epoch": 14.682926829268293, + "grad_norm": 3.4498767852783203, + "learning_rate": 0.0001, + "loss": 74.9234, + "step": 910 + }, + { + "epoch": 14.845528455284553, + "grad_norm": 1.6880728006362915, + "learning_rate": 0.0001, + "loss": 75.3617, + "step": 920 + }, + { + "epoch": 15.0, + "grad_norm": 1.8403468132019043, + "learning_rate": 0.0001, + "loss": 73.2543, + "step": 930 + }, + { + "epoch": 15.16260162601626, + "grad_norm": 1.4368852376937866, + "learning_rate": 0.0001, + "loss": 76.6271, + "step": 940 + }, + { + "epoch": 15.32520325203252, + "grad_norm": 2.9467780590057373, + "learning_rate": 0.0001, + "loss": 75.8744, + "step": 950 + }, + { + "epoch": 15.487804878048781, + "grad_norm": 1.567134141921997, + "learning_rate": 0.0001, + "loss": 76.6629, + "step": 960 + }, + { + "epoch": 15.65040650406504, + "grad_norm": 1.4503871202468872, + "learning_rate": 0.0001, + "loss": 76.3525, + "step": 970 + }, + { + "epoch": 15.8130081300813, + "grad_norm": 1.8871147632598877, + "learning_rate": 0.0001, + "loss": 77.4684, + "step": 980 + }, + { + "epoch": 15.975609756097562, + "grad_norm": 1.6707026958465576, + "learning_rate": 0.0001, + "loss": 76.3063, + "step": 990 + }, + { + "epoch": 16.130081300813007, + "grad_norm": 3.6300716400146484, + "learning_rate": 0.0001, + "loss": 73.0586, + "step": 1000 + }, + { + "epoch": 16.29268292682927, + "grad_norm": 1.6669234037399292, + "learning_rate": 0.0001, + "loss": 75.8027, + "step": 1010 + }, + { + "epoch": 16.45528455284553, + "grad_norm": 1.6920552253723145, + "learning_rate": 0.0001, + "loss": 74.7252, + "step": 1020 + }, + { + "epoch": 16.617886178861788, + "grad_norm": 1.8184431791305542, + "learning_rate": 0.0001, + "loss": 75.3625, + "step": 1030 + }, + { + "epoch": 16.78048780487805, + "grad_norm": 1.5828496217727661, + "learning_rate": 0.0001, + "loss": 78.0302, + "step": 1040 + }, + { + "epoch": 16.943089430894307, + "grad_norm": 1.7372915744781494, + "learning_rate": 0.0001, + "loss": 78.6949, + "step": 1050 + }, + { + "epoch": 17.097560975609756, + "grad_norm": 1.837161898612976, + "learning_rate": 0.0001, + "loss": 74.4726, + "step": 1060 + }, + { + "epoch": 17.260162601626018, + "grad_norm": 3.8733808994293213, + "learning_rate": 0.0001, + "loss": 76.1711, + "step": 1070 + }, + { + "epoch": 17.422764227642276, + "grad_norm": 1.5907851457595825, + "learning_rate": 0.0001, + "loss": 75.8292, + "step": 1080 + }, + { + "epoch": 17.585365853658537, + "grad_norm": 1.8271747827529907, + "learning_rate": 0.0001, + "loss": 74.5316, + "step": 1090 + }, + { + "epoch": 17.747967479674795, + "grad_norm": 1.827980875968933, + "learning_rate": 0.0001, + "loss": 74.7281, + "step": 1100 + }, + { + "epoch": 17.910569105691057, + "grad_norm": 1.4184074401855469, + "learning_rate": 0.0001, + "loss": 75.736, + "step": 1110 + }, + { + "epoch": 18.065040650406505, + "grad_norm": 2.3459699153900146, + "learning_rate": 0.0001, + "loss": 71.9586, + "step": 1120 + }, + { + "epoch": 18.227642276422763, + "grad_norm": 47.673641204833984, + "learning_rate": 0.0001, + "loss": 75.6649, + "step": 1130 + }, + { + "epoch": 18.390243902439025, + "grad_norm": 1.677398443222046, + "learning_rate": 0.0001, + "loss": 76.0574, + "step": 1140 + }, + { + "epoch": 18.552845528455286, + "grad_norm": 1.4180850982666016, + "learning_rate": 0.0001, + "loss": 74.3077, + "step": 1150 + }, + { + "epoch": 18.715447154471544, + "grad_norm": 2.0041465759277344, + "learning_rate": 0.0001, + "loss": 73.5916, + "step": 1160 + }, + { + "epoch": 18.878048780487806, + "grad_norm": 1.9098238945007324, + "learning_rate": 0.0001, + "loss": 74.5002, + "step": 1170 + }, + { + "epoch": 19.03252032520325, + "grad_norm": 1.405555248260498, + "learning_rate": 0.0001, + "loss": 69.8115, + "step": 1180 + }, + { + "epoch": 19.195121951219512, + "grad_norm": 1.424198865890503, + "learning_rate": 0.0001, + "loss": 73.7289, + "step": 1190 + }, + { + "epoch": 19.357723577235774, + "grad_norm": 7.542550563812256, + "learning_rate": 0.0001, + "loss": 73.7696, + "step": 1200 + }, + { + "epoch": 19.520325203252032, + "grad_norm": 1.425329566001892, + "learning_rate": 0.0001, + "loss": 72.1261, + "step": 1210 + }, + { + "epoch": 19.682926829268293, + "grad_norm": 1.525724172592163, + "learning_rate": 0.0001, + "loss": 73.6578, + "step": 1220 + }, + { + "epoch": 19.84552845528455, + "grad_norm": 1.6387193202972412, + "learning_rate": 0.0001, + "loss": 74.3027, + "step": 1230 + }, + { + "epoch": 20.0, + "grad_norm": 1.8429840803146362, + "learning_rate": 0.0001, + "loss": 69.5056, + "step": 1240 + }, + { + "epoch": 20.16260162601626, + "grad_norm": 2.5166006088256836, + "learning_rate": 0.0001, + "loss": 73.5965, + "step": 1250 + }, + { + "epoch": 20.32520325203252, + "grad_norm": 1.6006672382354736, + "learning_rate": 0.0001, + "loss": 72.4581, + "step": 1260 + }, + { + "epoch": 20.48780487804878, + "grad_norm": 1.6722878217697144, + "learning_rate": 0.0001, + "loss": 72.3097, + "step": 1270 + }, + { + "epoch": 20.65040650406504, + "grad_norm": 1.8715728521347046, + "learning_rate": 0.0001, + "loss": 73.4752, + "step": 1280 + }, + { + "epoch": 20.8130081300813, + "grad_norm": 1.3280900716781616, + "learning_rate": 0.0001, + "loss": 73.2145, + "step": 1290 + }, + { + "epoch": 20.975609756097562, + "grad_norm": 1.941451907157898, + "learning_rate": 0.0001, + "loss": 73.2085, + "step": 1300 + }, + { + "epoch": 21.130081300813007, + "grad_norm": 4.277497291564941, + "learning_rate": 0.0001, + "loss": 68.8305, + "step": 1310 + }, + { + "epoch": 21.29268292682927, + "grad_norm": 1.9949846267700195, + "learning_rate": 0.0001, + "loss": 72.6689, + "step": 1320 + }, + { + "epoch": 21.45528455284553, + "grad_norm": 1.524044394493103, + "learning_rate": 0.0001, + "loss": 72.9705, + "step": 1330 + }, + { + "epoch": 21.617886178861788, + "grad_norm": 1.384194016456604, + "learning_rate": 0.0001, + "loss": 72.5465, + "step": 1340 + }, + { + "epoch": 21.78048780487805, + "grad_norm": 1.724225640296936, + "learning_rate": 0.0001, + "loss": 72.9203, + "step": 1350 + }, + { + "epoch": 21.943089430894307, + "grad_norm": 2.0856704711914062, + "learning_rate": 0.0001, + "loss": 72.4034, + "step": 1360 + }, + { + "epoch": 22.097560975609756, + "grad_norm": 5.128044128417969, + "learning_rate": 0.0001, + "loss": 68.7754, + "step": 1370 + }, + { + "epoch": 22.260162601626018, + "grad_norm": 1.7385634183883667, + "learning_rate": 0.0001, + "loss": 73.9349, + "step": 1380 + }, + { + "epoch": 22.422764227642276, + "grad_norm": 1.3946012258529663, + "learning_rate": 0.0001, + "loss": 70.9627, + "step": 1390 + }, + { + "epoch": 22.585365853658537, + "grad_norm": 1.285414457321167, + "learning_rate": 0.0001, + "loss": 72.4578, + "step": 1400 + }, + { + "epoch": 22.747967479674795, + "grad_norm": 1.4197710752487183, + "learning_rate": 0.0001, + "loss": 72.5252, + "step": 1410 + }, + { + "epoch": 22.910569105691057, + "grad_norm": 3.048942804336548, + "learning_rate": 0.0001, + "loss": 71.1107, + "step": 1420 + }, + { + "epoch": 23.065040650406505, + "grad_norm": 1.3114765882492065, + "learning_rate": 0.0001, + "loss": 68.2725, + "step": 1430 + }, + { + "epoch": 23.227642276422763, + "grad_norm": 1.2763880491256714, + "learning_rate": 0.0001, + "loss": 71.5783, + "step": 1440 + }, + { + "epoch": 23.390243902439025, + "grad_norm": 1.3913089036941528, + "learning_rate": 0.0001, + "loss": 72.5986, + "step": 1450 + }, + { + "epoch": 23.552845528455286, + "grad_norm": 1.628751516342163, + "learning_rate": 0.0001, + "loss": 71.8383, + "step": 1460 + }, + { + "epoch": 23.715447154471544, + "grad_norm": 1.4331274032592773, + "learning_rate": 0.0001, + "loss": 70.8051, + "step": 1470 + }, + { + "epoch": 23.878048780487806, + "grad_norm": 1.4154727458953857, + "learning_rate": 0.0001, + "loss": 71.3283, + "step": 1480 + }, + { + "epoch": 24.03252032520325, + "grad_norm": 1.7208784818649292, + "learning_rate": 0.0001, + "loss": 68.4719, + "step": 1490 + }, + { + "epoch": 24.195121951219512, + "grad_norm": 1.466123342514038, + "learning_rate": 0.0001, + "loss": 71.9597, + "step": 1500 + }, + { + "epoch": 24.357723577235774, + "grad_norm": 1.4823886156082153, + "learning_rate": 0.0001, + "loss": 72.4533, + "step": 1510 + }, + { + "epoch": 24.520325203252032, + "grad_norm": 1.8407355546951294, + "learning_rate": 0.0001, + "loss": 71.1468, + "step": 1520 + }, + { + "epoch": 24.682926829268293, + "grad_norm": 1.630565881729126, + "learning_rate": 0.0001, + "loss": 71.5192, + "step": 1530 + }, + { + "epoch": 24.84552845528455, + "grad_norm": 1.5248785018920898, + "learning_rate": 0.0001, + "loss": 68.4804, + "step": 1540 + }, + { + "epoch": 25.0, + "grad_norm": 1.4690219163894653, + "learning_rate": 0.0001, + "loss": 67.5716, + "step": 1550 + }, + { + "epoch": 25.16260162601626, + "grad_norm": 2.0246646404266357, + "learning_rate": 0.0001, + "loss": 71.2554, + "step": 1560 + }, + { + "epoch": 25.32520325203252, + "grad_norm": 1.3611458539962769, + "learning_rate": 0.0001, + "loss": 69.8558, + "step": 1570 + }, + { + "epoch": 25.48780487804878, + "grad_norm": 2.0006039142608643, + "learning_rate": 0.0001, + "loss": 72.5344, + "step": 1580 + }, + { + "epoch": 25.65040650406504, + "grad_norm": 1.7373167276382446, + "learning_rate": 0.0001, + "loss": 70.6253, + "step": 1590 + }, + { + "epoch": 25.8130081300813, + "grad_norm": 1.2878378629684448, + "learning_rate": 0.0001, + "loss": 72.165, + "step": 1600 + }, + { + "epoch": 25.975609756097562, + "grad_norm": 1.5337013006210327, + "learning_rate": 0.0001, + "loss": 70.6989, + "step": 1610 + }, + { + "epoch": 26.130081300813007, + "grad_norm": 1.364615559577942, + "learning_rate": 0.0001, + "loss": 67.3898, + "step": 1620 + }, + { + "epoch": 26.29268292682927, + "grad_norm": 1.3249316215515137, + "learning_rate": 0.0001, + "loss": 72.7102, + "step": 1630 + }, + { + "epoch": 26.45528455284553, + "grad_norm": 4.209898471832275, + "learning_rate": 0.0001, + "loss": 70.3185, + "step": 1640 + }, + { + "epoch": 26.617886178861788, + "grad_norm": 1.5488810539245605, + "learning_rate": 0.0001, + "loss": 72.3678, + "step": 1650 + }, + { + "epoch": 26.78048780487805, + "grad_norm": 1.7743597030639648, + "learning_rate": 0.0001, + "loss": 71.0917, + "step": 1660 + }, + { + "epoch": 26.943089430894307, + "grad_norm": 1.5433533191680908, + "learning_rate": 0.0001, + "loss": 71.4001, + "step": 1670 + }, + { + "epoch": 27.097560975609756, + "grad_norm": 1.6320385932922363, + "learning_rate": 0.0001, + "loss": 66.7441, + "step": 1680 + }, + { + "epoch": 27.260162601626018, + "grad_norm": 1.3137969970703125, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 1690 + }, + { + "epoch": 27.422764227642276, + "grad_norm": 1.9150484800338745, + "learning_rate": 0.0001, + "loss": 69.8956, + "step": 1700 + }, + { + "epoch": 27.585365853658537, + "grad_norm": 1.4442802667617798, + "learning_rate": 0.0001, + "loss": 70.7185, + "step": 1710 + }, + { + "epoch": 27.747967479674795, + "grad_norm": 2.572497606277466, + "learning_rate": 0.0001, + "loss": 70.8934, + "step": 1720 + }, + { + "epoch": 27.910569105691057, + "grad_norm": 1.6669617891311646, + "learning_rate": 0.0001, + "loss": 71.8221, + "step": 1730 + }, + { + "epoch": 28.065040650406505, + "grad_norm": 1.8449876308441162, + "learning_rate": 0.0001, + "loss": 66.3923, + "step": 1740 + }, + { + "epoch": 28.227642276422763, + "grad_norm": 1.6600004434585571, + "learning_rate": 0.0001, + "loss": 69.5301, + "step": 1750 + }, + { + "epoch": 28.390243902439025, + "grad_norm": 1.4655731916427612, + "learning_rate": 0.0001, + "loss": 72.1924, + "step": 1760 + }, + { + "epoch": 28.552845528455286, + "grad_norm": 1.134505033493042, + "learning_rate": 0.0001, + "loss": 69.5855, + "step": 1770 + }, + { + "epoch": 28.715447154471544, + "grad_norm": 1.301984190940857, + "learning_rate": 0.0001, + "loss": 70.7999, + "step": 1780 + }, + { + "epoch": 28.878048780487806, + "grad_norm": 1.5641013383865356, + "learning_rate": 0.0001, + "loss": 69.9091, + "step": 1790 + }, + { + "epoch": 29.03252032520325, + "grad_norm": 1.4850797653198242, + "learning_rate": 0.0001, + "loss": 66.6266, + "step": 1800 + }, + { + "epoch": 29.195121951219512, + "grad_norm": 1.2069118022918701, + "learning_rate": 0.0001, + "loss": 69.3802, + "step": 1810 + }, + { + "epoch": 29.357723577235774, + "grad_norm": 1.2890352010726929, + "learning_rate": 0.0001, + "loss": 68.5044, + "step": 1820 + }, + { + "epoch": 29.520325203252032, + "grad_norm": 1.4150917530059814, + "learning_rate": 0.0001, + "loss": 69.807, + "step": 1830 + }, + { + "epoch": 29.682926829268293, + "grad_norm": 1.514807939529419, + "learning_rate": 0.0001, + "loss": 71.3858, + "step": 1840 + }, + { + "epoch": 29.84552845528455, + "grad_norm": 1.4246330261230469, + "learning_rate": 0.0001, + "loss": 70.494, + "step": 1850 + }, + { + "epoch": 30.0, + "grad_norm": 1.1789597272872925, + "learning_rate": 0.0001, + "loss": 67.1056, + "step": 1860 + }, + { + "epoch": 30.16260162601626, + "grad_norm": 1.23678719997406, + "learning_rate": 0.0001, + "loss": 70.336, + "step": 1870 + }, + { + "epoch": 30.32520325203252, + "grad_norm": 1.1756274700164795, + "learning_rate": 0.0001, + "loss": 67.9909, + "step": 1880 + }, + { + "epoch": 30.48780487804878, + "grad_norm": 1.4175690412521362, + "learning_rate": 0.0001, + "loss": 69.2617, + "step": 1890 + }, + { + "epoch": 30.65040650406504, + "grad_norm": 1.3084465265274048, + "learning_rate": 0.0001, + "loss": 70.2963, + "step": 1900 + }, + { + "epoch": 30.8130081300813, + "grad_norm": 1.486464500427246, + "learning_rate": 0.0001, + "loss": 68.6317, + "step": 1910 + }, + { + "epoch": 30.975609756097562, + "grad_norm": 10.415762901306152, + "learning_rate": 0.0001, + "loss": 69.7767, + "step": 1920 + }, + { + "epoch": 31.130081300813007, + "grad_norm": 1.4484038352966309, + "learning_rate": 0.0001, + "loss": 65.6345, + "step": 1930 + }, + { + "epoch": 31.29268292682927, + "grad_norm": 2.297306537628174, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 1940 + }, + { + "epoch": 31.45528455284553, + "grad_norm": 1.223490595817566, + "learning_rate": 0.0001, + "loss": 69.2139, + "step": 1950 + }, + { + "epoch": 31.617886178861788, + "grad_norm": 1.2144125699996948, + "learning_rate": 0.0001, + "loss": 70.3061, + "step": 1960 + }, + { + "epoch": 31.78048780487805, + "grad_norm": 2.633528470993042, + "learning_rate": 0.0001, + "loss": 68.7096, + "step": 1970 + }, + { + "epoch": 31.943089430894307, + "grad_norm": 1.5048350095748901, + "learning_rate": 0.0001, + "loss": 68.6711, + "step": 1980 + }, + { + "epoch": 32.09756097560975, + "grad_norm": 1.5154467821121216, + "learning_rate": 0.0001, + "loss": 66.7989, + "step": 1990 + }, + { + "epoch": 32.260162601626014, + "grad_norm": 2.721277952194214, + "learning_rate": 0.0001, + "loss": 70.4987, + "step": 2000 + }, + { + "epoch": 32.422764227642276, + "grad_norm": 1.44464910030365, + "learning_rate": 0.0001, + "loss": 69.9712, + "step": 2010 + }, + { + "epoch": 32.58536585365854, + "grad_norm": 1.3462835550308228, + "learning_rate": 0.0001, + "loss": 68.9381, + "step": 2020 + }, + { + "epoch": 32.7479674796748, + "grad_norm": 1.3283029794692993, + "learning_rate": 0.0001, + "loss": 68.5301, + "step": 2030 + }, + { + "epoch": 32.91056910569106, + "grad_norm": 1.5172960758209229, + "learning_rate": 0.0001, + "loss": 70.1894, + "step": 2040 + }, + { + "epoch": 33.0650406504065, + "grad_norm": 1.1872576475143433, + "learning_rate": 0.0001, + "loss": 65.0158, + "step": 2050 + }, + { + "epoch": 33.22764227642276, + "grad_norm": 1.5414419174194336, + "learning_rate": 0.0001, + "loss": 69.1537, + "step": 2060 + }, + { + "epoch": 33.390243902439025, + "grad_norm": 1.3143430948257446, + "learning_rate": 0.0001, + "loss": 68.6792, + "step": 2070 + }, + { + "epoch": 33.552845528455286, + "grad_norm": 1.3863723278045654, + "learning_rate": 0.0001, + "loss": 70.3867, + "step": 2080 + }, + { + "epoch": 33.71544715447155, + "grad_norm": 1.4060479402542114, + "learning_rate": 0.0001, + "loss": 68.4981, + "step": 2090 + }, + { + "epoch": 33.8780487804878, + "grad_norm": 1.34132981300354, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 2100 + }, + { + "epoch": 34.03252032520325, + "grad_norm": 1.8303673267364502, + "learning_rate": 0.0001, + "loss": 65.819, + "step": 2110 + }, + { + "epoch": 34.19512195121951, + "grad_norm": 1.387339472770691, + "learning_rate": 0.0001, + "loss": 69.9825, + "step": 2120 + }, + { + "epoch": 34.357723577235774, + "grad_norm": 1.4125195741653442, + "learning_rate": 0.0001, + "loss": 67.5149, + "step": 2130 + }, + { + "epoch": 34.520325203252035, + "grad_norm": 1.2909412384033203, + "learning_rate": 0.0001, + "loss": 69.7848, + "step": 2140 + }, + { + "epoch": 34.68292682926829, + "grad_norm": 1.8860795497894287, + "learning_rate": 0.0001, + "loss": 68.6145, + "step": 2150 + }, + { + "epoch": 34.84552845528455, + "grad_norm": 1.5111079216003418, + "learning_rate": 0.0001, + "loss": 68.2612, + "step": 2160 + }, + { + "epoch": 35.0, + "grad_norm": 1.3275264501571655, + "learning_rate": 0.0001, + "loss": 64.0298, + "step": 2170 + }, + { + "epoch": 35.16260162601626, + "grad_norm": 1.174864411354065, + "learning_rate": 0.0001, + "loss": 66.6153, + "step": 2180 + }, + { + "epoch": 35.32520325203252, + "grad_norm": 1.6873915195465088, + "learning_rate": 0.0001, + "loss": 69.6278, + "step": 2190 + }, + { + "epoch": 35.48780487804878, + "grad_norm": 2.4457507133483887, + "learning_rate": 0.0001, + "loss": 68.3611, + "step": 2200 + }, + { + "epoch": 35.65040650406504, + "grad_norm": 1.4779958724975586, + "learning_rate": 0.0001, + "loss": 69.2609, + "step": 2210 + }, + { + "epoch": 35.8130081300813, + "grad_norm": 2.273709774017334, + "learning_rate": 0.0001, + "loss": 69.1312, + "step": 2220 + }, + { + "epoch": 35.97560975609756, + "grad_norm": 1.5741862058639526, + "learning_rate": 0.0001, + "loss": 69.1191, + "step": 2230 + }, + { + "epoch": 36.13008130081301, + "grad_norm": 1.2730036973953247, + "learning_rate": 0.0001, + "loss": 65.692, + "step": 2240 + }, + { + "epoch": 36.292682926829265, + "grad_norm": 1.2936429977416992, + "learning_rate": 0.0001, + "loss": 68.2962, + "step": 2250 + }, + { + "epoch": 36.45528455284553, + "grad_norm": 1.354279637336731, + "learning_rate": 0.0001, + "loss": 70.5507, + "step": 2260 + }, + { + "epoch": 36.61788617886179, + "grad_norm": 1.4165853261947632, + "learning_rate": 0.0001, + "loss": 68.0048, + "step": 2270 + }, + { + "epoch": 36.78048780487805, + "grad_norm": 1.2992445230484009, + "learning_rate": 0.0001, + "loss": 67.7072, + "step": 2280 + }, + { + "epoch": 36.94308943089431, + "grad_norm": 1.5053461790084839, + "learning_rate": 0.0001, + "loss": 66.7587, + "step": 2290 + }, + { + "epoch": 37.09756097560975, + "grad_norm": 1.337390661239624, + "learning_rate": 0.0001, + "loss": 63.786, + "step": 2300 + }, + { + "epoch": 37.260162601626014, + "grad_norm": 1.0407774448394775, + "learning_rate": 0.0001, + "loss": 66.6253, + "step": 2310 + }, + { + "epoch": 37.422764227642276, + "grad_norm": 1.564152479171753, + "learning_rate": 0.0001, + "loss": 68.3606, + "step": 2320 + }, + { + "epoch": 37.58536585365854, + "grad_norm": 1.2028454542160034, + "learning_rate": 0.0001, + "loss": 66.2841, + "step": 2330 + }, + { + "epoch": 37.7479674796748, + "grad_norm": 1.4528603553771973, + "learning_rate": 0.0001, + "loss": 67.4277, + "step": 2340 + }, + { + "epoch": 37.91056910569106, + "grad_norm": 1.4954335689544678, + "learning_rate": 0.0001, + "loss": 68.9954, + "step": 2350 + }, + { + "epoch": 38.0650406504065, + "grad_norm": 1.2820416688919067, + "learning_rate": 0.0001, + "loss": 65.2449, + "step": 2360 + }, + { + "epoch": 38.22764227642276, + "grad_norm": 1.2392579317092896, + "learning_rate": 0.0001, + "loss": 67.5006, + "step": 2370 + }, + { + "epoch": 38.390243902439025, + "grad_norm": 1.2135238647460938, + "learning_rate": 0.0001, + "loss": 69.2214, + "step": 2380 + }, + { + "epoch": 38.552845528455286, + "grad_norm": 1.5964744091033936, + "learning_rate": 0.0001, + "loss": 67.934, + "step": 2390 + }, + { + "epoch": 38.71544715447155, + "grad_norm": 1.2841702699661255, + "learning_rate": 0.0001, + "loss": 67.0848, + "step": 2400 + }, + { + "epoch": 38.8780487804878, + "grad_norm": 1.8026081323623657, + "learning_rate": 0.0001, + "loss": 67.5358, + "step": 2410 + }, + { + "epoch": 39.03252032520325, + "grad_norm": 1.4178807735443115, + "learning_rate": 0.0001, + "loss": 64.771, + "step": 2420 + }, + { + "epoch": 39.19512195121951, + "grad_norm": 1.3671854734420776, + "learning_rate": 0.0001, + "loss": 68.0531, + "step": 2430 + }, + { + "epoch": 39.357723577235774, + "grad_norm": 1.43596613407135, + "learning_rate": 0.0001, + "loss": 67.7735, + "step": 2440 + }, + { + "epoch": 39.520325203252035, + "grad_norm": 1.464250922203064, + "learning_rate": 0.0001, + "loss": 67.8331, + "step": 2450 + }, + { + "epoch": 39.68292682926829, + "grad_norm": 3.101461410522461, + "learning_rate": 0.0001, + "loss": 65.8956, + "step": 2460 + }, + { + "epoch": 39.84552845528455, + "grad_norm": 1.8332289457321167, + "learning_rate": 0.0001, + "loss": 66.8218, + "step": 2470 + }, + { + "epoch": 40.0, + "grad_norm": 1.2660984992980957, + "learning_rate": 0.0001, + "loss": 63.9969, + "step": 2480 + }, + { + "epoch": 40.16260162601626, + "grad_norm": 2.0309898853302, + "learning_rate": 0.0001, + "loss": 67.3538, + "step": 2490 + }, + { + "epoch": 40.32520325203252, + "grad_norm": 1.207000494003296, + "learning_rate": 0.0001, + "loss": 68.0898, + "step": 2500 + }, + { + "epoch": 40.48780487804878, + "grad_norm": 1.3609018325805664, + "learning_rate": 0.0001, + "loss": 67.0381, + "step": 2510 + }, + { + "epoch": 40.65040650406504, + "grad_norm": 1.310826301574707, + "learning_rate": 0.0001, + "loss": 69.0267, + "step": 2520 + }, + { + "epoch": 40.8130081300813, + "grad_norm": 1.4150960445404053, + "learning_rate": 0.0001, + "loss": 69.1934, + "step": 2530 + }, + { + "epoch": 40.97560975609756, + "grad_norm": 1.5414319038391113, + "learning_rate": 0.0001, + "loss": 68.0604, + "step": 2540 + }, + { + "epoch": 41.13008130081301, + "grad_norm": 2.3246140480041504, + "learning_rate": 0.0001, + "loss": 61.5698, + "step": 2550 + }, + { + "epoch": 41.292682926829265, + "grad_norm": 1.206296443939209, + "learning_rate": 0.0001, + "loss": 64.9898, + "step": 2560 + }, + { + "epoch": 41.45528455284553, + "grad_norm": 1.6080626249313354, + "learning_rate": 0.0001, + "loss": 68.2685, + "step": 2570 + }, + { + "epoch": 41.61788617886179, + "grad_norm": 1.2404112815856934, + "learning_rate": 0.0001, + "loss": 67.8457, + "step": 2580 + }, + { + "epoch": 41.78048780487805, + "grad_norm": 1.2304514646530151, + "learning_rate": 0.0001, + "loss": 67.9052, + "step": 2590 + }, + { + "epoch": 41.94308943089431, + "grad_norm": 1.5806187391281128, + "learning_rate": 0.0001, + "loss": 66.0663, + "step": 2600 + }, + { + "epoch": 42.09756097560975, + "grad_norm": 1.2564151287078857, + "learning_rate": 0.0001, + "loss": 65.7763, + "step": 2610 + }, + { + "epoch": 42.260162601626014, + "grad_norm": 1.4972114562988281, + "learning_rate": 0.0001, + "loss": 66.4269, + "step": 2620 + }, + { + "epoch": 42.422764227642276, + "grad_norm": 1.2913843393325806, + "learning_rate": 0.0001, + "loss": 67.7554, + "step": 2630 + }, + { + "epoch": 42.58536585365854, + "grad_norm": 1.3743863105773926, + "learning_rate": 0.0001, + "loss": 67.2627, + "step": 2640 + }, + { + "epoch": 42.7479674796748, + "grad_norm": 1.1267883777618408, + "learning_rate": 0.0001, + "loss": 67.4597, + "step": 2650 + }, + { + "epoch": 42.91056910569106, + "grad_norm": 2.0194661617279053, + "learning_rate": 0.0001, + "loss": 66.9295, + "step": 2660 + }, + { + "epoch": 43.0650406504065, + "grad_norm": 1.273779273033142, + "learning_rate": 0.0001, + "loss": 60.7906, + "step": 2670 + }, + { + "epoch": 43.22764227642276, + "grad_norm": 1.1498609781265259, + "learning_rate": 0.0001, + "loss": 68.0405, + "step": 2680 + }, + { + "epoch": 43.390243902439025, + "grad_norm": 1.4689130783081055, + "learning_rate": 0.0001, + "loss": 66.2358, + "step": 2690 + }, + { + "epoch": 43.552845528455286, + "grad_norm": 1.4495564699172974, + "learning_rate": 0.0001, + "loss": 67.4267, + "step": 2700 + }, + { + "epoch": 43.71544715447155, + "grad_norm": 2.5769612789154053, + "learning_rate": 0.0001, + "loss": 66.953, + "step": 2710 + }, + { + "epoch": 43.8780487804878, + "grad_norm": 2.1684205532073975, + "learning_rate": 0.0001, + "loss": 65.6393, + "step": 2720 + }, + { + "epoch": 44.03252032520325, + "grad_norm": 1.4968472719192505, + "learning_rate": 0.0001, + "loss": 60.8256, + "step": 2730 + }, + { + "epoch": 44.19512195121951, + "grad_norm": 1.1874353885650635, + "learning_rate": 0.0001, + "loss": 66.818, + "step": 2740 + }, + { + "epoch": 44.357723577235774, + "grad_norm": 2.271235227584839, + "learning_rate": 0.0001, + "loss": 66.5048, + "step": 2750 + }, + { + "epoch": 44.520325203252035, + "grad_norm": 1.20489501953125, + "learning_rate": 0.0001, + "loss": 67.0487, + "step": 2760 + }, + { + "epoch": 44.68292682926829, + "grad_norm": 1.6273691654205322, + "learning_rate": 0.0001, + "loss": 66.2228, + "step": 2770 + }, + { + "epoch": 44.84552845528455, + "grad_norm": 1.4935433864593506, + "learning_rate": 0.0001, + "loss": 67.2039, + "step": 2780 + }, + { + "epoch": 45.0, + "grad_norm": 1.2668659687042236, + "learning_rate": 0.0001, + "loss": 62.7855, + "step": 2790 + }, + { + "epoch": 45.16260162601626, + "grad_norm": 1.6439573764801025, + "learning_rate": 0.0001, + "loss": 67.919, + "step": 2800 + }, + { + "epoch": 45.32520325203252, + "grad_norm": 1.311854600906372, + "learning_rate": 0.0001, + "loss": 65.1797, + "step": 2810 + }, + { + "epoch": 45.48780487804878, + "grad_norm": 1.3596776723861694, + "learning_rate": 0.0001, + "loss": 67.469, + "step": 2820 + }, + { + "epoch": 45.65040650406504, + "grad_norm": 1.5618443489074707, + "learning_rate": 0.0001, + "loss": 66.1808, + "step": 2830 + }, + { + "epoch": 45.8130081300813, + "grad_norm": 1.7551205158233643, + "learning_rate": 0.0001, + "loss": 65.3803, + "step": 2840 + }, + { + "epoch": 45.97560975609756, + "grad_norm": 1.433794379234314, + "learning_rate": 0.0001, + "loss": 67.8314, + "step": 2850 + }, + { + "epoch": 46.13008130081301, + "grad_norm": 1.4634851217269897, + "learning_rate": 0.0001, + "loss": 63.4219, + "step": 2860 + }, + { + "epoch": 46.292682926829265, + "grad_norm": 1.083052396774292, + "learning_rate": 0.0001, + "loss": 68.0719, + "step": 2870 + }, + { + "epoch": 46.45528455284553, + "grad_norm": 1.1784703731536865, + "learning_rate": 0.0001, + "loss": 66.1391, + "step": 2880 + }, + { + "epoch": 46.61788617886179, + "grad_norm": 1.2552465200424194, + "learning_rate": 0.0001, + "loss": 65.6006, + "step": 2890 + }, + { + "epoch": 46.78048780487805, + "grad_norm": 1.4684522151947021, + "learning_rate": 0.0001, + "loss": 66.8314, + "step": 2900 + }, + { + "epoch": 46.94308943089431, + "grad_norm": 1.741514801979065, + "learning_rate": 0.0001, + "loss": 67.5056, + "step": 2910 + }, + { + "epoch": 47.09756097560975, + "grad_norm": 5.696534156799316, + "learning_rate": 0.0001, + "loss": 62.8016, + "step": 2920 + }, + { + "epoch": 47.260162601626014, + "grad_norm": 1.5000969171524048, + "learning_rate": 0.0001, + "loss": 68.0272, + "step": 2930 + }, + { + "epoch": 47.422764227642276, + "grad_norm": 1.1044933795928955, + "learning_rate": 0.0001, + "loss": 67.0905, + "step": 2940 + }, + { + "epoch": 47.58536585365854, + "grad_norm": 0.9933589100837708, + "learning_rate": 0.0001, + "loss": 67.6077, + "step": 2950 + }, + { + "epoch": 47.7479674796748, + "grad_norm": 1.2908599376678467, + "learning_rate": 0.0001, + "loss": 65.869, + "step": 2960 + }, + { + "epoch": 47.91056910569106, + "grad_norm": 1.0470529794692993, + "learning_rate": 0.0001, + "loss": 66.5353, + "step": 2970 + }, + { + "epoch": 48.0650406504065, + "grad_norm": 1.0927976369857788, + "learning_rate": 0.0001, + "loss": 62.7541, + "step": 2980 + }, + { + "epoch": 48.22764227642276, + "grad_norm": 1.0870513916015625, + "learning_rate": 0.0001, + "loss": 64.172, + "step": 2990 + }, + { + "epoch": 48.390243902439025, + "grad_norm": 1.3492090702056885, + "learning_rate": 0.0001, + "loss": 67.6642, + "step": 3000 + }, + { + "epoch": 48.552845528455286, + "grad_norm": 1.4024657011032104, + "learning_rate": 0.0001, + "loss": 65.2722, + "step": 3010 + }, + { + "epoch": 48.71544715447155, + "grad_norm": 1.1165252923965454, + "learning_rate": 0.0001, + "loss": 66.7233, + "step": 3020 + }, + { + "epoch": 48.8780487804878, + "grad_norm": 1.2147928476333618, + "learning_rate": 0.0001, + "loss": 65.7277, + "step": 3030 + }, + { + "epoch": 49.03252032520325, + "grad_norm": 0.925310492515564, + "learning_rate": 0.0001, + "loss": 62.9256, + "step": 3040 + }, + { + "epoch": 49.19512195121951, + "grad_norm": 1.2353442907333374, + "learning_rate": 0.0001, + "loss": 66.1218, + "step": 3050 + }, + { + "epoch": 49.357723577235774, + "grad_norm": 1.2177367210388184, + "learning_rate": 0.0001, + "loss": 66.7978, + "step": 3060 + }, + { + "epoch": 49.520325203252035, + "grad_norm": 1.248044729232788, + "learning_rate": 0.0001, + "loss": 66.2132, + "step": 3070 + }, + { + "epoch": 49.68292682926829, + "grad_norm": 1.366050124168396, + "learning_rate": 0.0001, + "loss": 67.0118, + "step": 3080 + }, + { + "epoch": 49.84552845528455, + "grad_norm": 1.3701151609420776, + "learning_rate": 0.0001, + "loss": 67.6099, + "step": 3090 + }, + { + "epoch": 50.0, + "grad_norm": 1.0420141220092773, + "learning_rate": 0.0001, + "loss": 61.5289, + "step": 3100 + }, + { + "epoch": 50.16260162601626, + "grad_norm": 1.0957746505737305, + "learning_rate": 0.0001, + "loss": 66.7348, + "step": 3110 + }, + { + "epoch": 50.32520325203252, + "grad_norm": 1.108628749847412, + "learning_rate": 0.0001, + "loss": 66.6199, + "step": 3120 + }, + { + "epoch": 50.48780487804878, + "grad_norm": 1.3680156469345093, + "learning_rate": 0.0001, + "loss": 67.0825, + "step": 3130 + }, + { + "epoch": 50.65040650406504, + "grad_norm": 1.169851303100586, + "learning_rate": 0.0001, + "loss": 65.4901, + "step": 3140 + }, + { + "epoch": 50.8130081300813, + "grad_norm": 2.1822028160095215, + "learning_rate": 0.0001, + "loss": 65.6841, + "step": 3150 + }, + { + "epoch": 50.97560975609756, + "grad_norm": 1.053402304649353, + "learning_rate": 0.0001, + "loss": 66.3119, + "step": 3160 + }, + { + "epoch": 51.13008130081301, + "grad_norm": 1.0627789497375488, + "learning_rate": 0.0001, + "loss": 61.3642, + "step": 3170 + }, + { + "epoch": 51.292682926829265, + "grad_norm": 1.926291823387146, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 3180 + }, + { + "epoch": 51.45528455284553, + "grad_norm": 1.5791983604431152, + "learning_rate": 0.0001, + "loss": 63.913, + "step": 3190 + }, + { + "epoch": 51.61788617886179, + "grad_norm": 1.210265040397644, + "learning_rate": 0.0001, + "loss": 64.5261, + "step": 3200 + }, + { + "epoch": 51.78048780487805, + "grad_norm": 1.0720808506011963, + "learning_rate": 0.0001, + "loss": 67.0194, + "step": 3210 + }, + { + "epoch": 51.94308943089431, + "grad_norm": 1.94563889503479, + "learning_rate": 0.0001, + "loss": 68.4662, + "step": 3220 + }, + { + "epoch": 52.09756097560975, + "grad_norm": 1.598980188369751, + "learning_rate": 0.0001, + "loss": 62.4946, + "step": 3230 + }, + { + "epoch": 52.260162601626014, + "grad_norm": 1.0564576387405396, + "learning_rate": 0.0001, + "loss": 65.7912, + "step": 3240 + }, + { + "epoch": 52.422764227642276, + "grad_norm": 1.2189645767211914, + "learning_rate": 0.0001, + "loss": 65.5772, + "step": 3250 + }, + { + "epoch": 52.58536585365854, + "grad_norm": 1.1357049942016602, + "learning_rate": 0.0001, + "loss": 65.1559, + "step": 3260 + }, + { + "epoch": 52.7479674796748, + "grad_norm": 1.316867709159851, + "learning_rate": 0.0001, + "loss": 64.7897, + "step": 3270 + }, + { + "epoch": 52.91056910569106, + "grad_norm": 1.3870197534561157, + "learning_rate": 0.0001, + "loss": 65.5195, + "step": 3280 + }, + { + "epoch": 53.0650406504065, + "grad_norm": 1.2467927932739258, + "learning_rate": 0.0001, + "loss": 61.3392, + "step": 3290 + }, + { + "epoch": 53.22764227642276, + "grad_norm": 1.1661109924316406, + "learning_rate": 0.0001, + "loss": 66.0117, + "step": 3300 + }, + { + "epoch": 53.390243902439025, + "grad_norm": 1.3509628772735596, + "learning_rate": 0.0001, + "loss": 64.7228, + "step": 3310 + }, + { + "epoch": 53.552845528455286, + "grad_norm": 1.1093642711639404, + "learning_rate": 0.0001, + "loss": 65.2534, + "step": 3320 + }, + { + "epoch": 53.71544715447155, + "grad_norm": 1.1801207065582275, + "learning_rate": 0.0001, + "loss": 65.7097, + "step": 3330 + }, + { + "epoch": 53.8780487804878, + "grad_norm": 1.175884485244751, + "learning_rate": 0.0001, + "loss": 64.4368, + "step": 3340 + }, + { + "epoch": 54.03252032520325, + "grad_norm": 1.1727889776229858, + "learning_rate": 0.0001, + "loss": 62.9373, + "step": 3350 + }, + { + "epoch": 54.19512195121951, + "grad_norm": 1.5330945253372192, + "learning_rate": 0.0001, + "loss": 66.7763, + "step": 3360 + }, + { + "epoch": 54.357723577235774, + "grad_norm": 1.2992504835128784, + "learning_rate": 0.0001, + "loss": 65.7854, + "step": 3370 + }, + { + "epoch": 54.520325203252035, + "grad_norm": 1.0185420513153076, + "learning_rate": 0.0001, + "loss": 65.7046, + "step": 3380 + }, + { + "epoch": 54.68292682926829, + "grad_norm": 1.426229476928711, + "learning_rate": 0.0001, + "loss": 64.0286, + "step": 3390 + }, + { + "epoch": 54.84552845528455, + "grad_norm": 1.2600868940353394, + "learning_rate": 0.0001, + "loss": 66.1738, + "step": 3400 + }, + { + "epoch": 55.0, + "grad_norm": 1.917455792427063, + "learning_rate": 0.0001, + "loss": 61.9381, + "step": 3410 + }, + { + "epoch": 55.16260162601626, + "grad_norm": 1.2000917196273804, + "learning_rate": 0.0001, + "loss": 65.5626, + "step": 3420 + }, + { + "epoch": 55.32520325203252, + "grad_norm": 1.362176775932312, + "learning_rate": 0.0001, + "loss": 66.8012, + "step": 3430 + }, + { + "epoch": 55.48780487804878, + "grad_norm": 1.1611402034759521, + "learning_rate": 0.0001, + "loss": 64.5227, + "step": 3440 + }, + { + "epoch": 55.65040650406504, + "grad_norm": 1.2066621780395508, + "learning_rate": 0.0001, + "loss": 63.4351, + "step": 3450 + }, + { + "epoch": 55.8130081300813, + "grad_norm": 1.005340814590454, + "learning_rate": 0.0001, + "loss": 65.9261, + "step": 3460 + }, + { + "epoch": 55.97560975609756, + "grad_norm": 1.0906637907028198, + "learning_rate": 0.0001, + "loss": 64.6635, + "step": 3470 + }, + { + "epoch": 56.13008130081301, + "grad_norm": 1.0608748197555542, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 3480 + }, + { + "epoch": 56.292682926829265, + "grad_norm": 1.025319218635559, + "learning_rate": 0.0001, + "loss": 64.7783, + "step": 3490 + }, + { + "epoch": 56.45528455284553, + "grad_norm": 1.1596784591674805, + "learning_rate": 0.0001, + "loss": 66.4629, + "step": 3500 + }, + { + "epoch": 56.61788617886179, + "grad_norm": 0.9318640232086182, + "learning_rate": 0.0001, + "loss": 65.5533, + "step": 3510 + }, + { + "epoch": 56.78048780487805, + "grad_norm": 1.0977418422698975, + "learning_rate": 0.0001, + "loss": 63.647, + "step": 3520 + }, + { + "epoch": 56.94308943089431, + "grad_norm": 1.3912436962127686, + "learning_rate": 0.0001, + "loss": 62.8273, + "step": 3530 + }, + { + "epoch": 57.09756097560975, + "grad_norm": 1.0921591520309448, + "learning_rate": 0.0001, + "loss": 61.5968, + "step": 3540 + }, + { + "epoch": 57.260162601626014, + "grad_norm": 1.17819344997406, + "learning_rate": 0.0001, + "loss": 64.9381, + "step": 3550 + }, + { + "epoch": 57.422764227642276, + "grad_norm": 1.188829779624939, + "learning_rate": 0.0001, + "loss": 64.6498, + "step": 3560 + }, + { + "epoch": 57.58536585365854, + "grad_norm": 3.664839506149292, + "learning_rate": 0.0001, + "loss": 65.3834, + "step": 3570 + }, + { + "epoch": 57.7479674796748, + "grad_norm": 1.57958984375, + "learning_rate": 0.0001, + "loss": 63.8808, + "step": 3580 + }, + { + "epoch": 57.91056910569106, + "grad_norm": 1.1837552785873413, + "learning_rate": 0.0001, + "loss": 64.1545, + "step": 3590 + }, + { + "epoch": 58.0650406504065, + "grad_norm": 1.5986013412475586, + "learning_rate": 0.0001, + "loss": 60.5742, + "step": 3600 + }, + { + "epoch": 58.22764227642276, + "grad_norm": 1.1551810503005981, + "learning_rate": 0.0001, + "loss": 66.2378, + "step": 3610 + }, + { + "epoch": 58.390243902439025, + "grad_norm": 1.0869863033294678, + "learning_rate": 0.0001, + "loss": 65.3034, + "step": 3620 + }, + { + "epoch": 58.552845528455286, + "grad_norm": 1.2263212203979492, + "learning_rate": 0.0001, + "loss": 63.976, + "step": 3630 + }, + { + "epoch": 58.71544715447155, + "grad_norm": 1.4410817623138428, + "learning_rate": 0.0001, + "loss": 63.4778, + "step": 3640 + }, + { + "epoch": 58.8780487804878, + "grad_norm": 1.9633759260177612, + "learning_rate": 0.0001, + "loss": 65.5946, + "step": 3650 + }, + { + "epoch": 59.03252032520325, + "grad_norm": 1.1729766130447388, + "learning_rate": 0.0001, + "loss": 62.2171, + "step": 3660 + }, + { + "epoch": 59.19512195121951, + "grad_norm": 1.0049556493759155, + "learning_rate": 0.0001, + "loss": 65.1854, + "step": 3670 + }, + { + "epoch": 59.357723577235774, + "grad_norm": 1.1727055311203003, + "learning_rate": 0.0001, + "loss": 65.4839, + "step": 3680 + }, + { + "epoch": 59.520325203252035, + "grad_norm": 1.5394458770751953, + "learning_rate": 0.0001, + "loss": 64.9341, + "step": 3690 + }, + { + "epoch": 59.68292682926829, + "grad_norm": 1.1583774089813232, + "learning_rate": 0.0001, + "loss": 63.9197, + "step": 3700 + }, + { + "epoch": 59.84552845528455, + "grad_norm": 1.0835607051849365, + "learning_rate": 0.0001, + "loss": 64.0494, + "step": 3710 + }, + { + "epoch": 60.0, + "grad_norm": 1.7504246234893799, + "learning_rate": 0.0001, + "loss": 59.4134, + "step": 3720 + }, + { + "epoch": 60.16260162601626, + "grad_norm": 1.1870343685150146, + "learning_rate": 0.0001, + "loss": 63.2158, + "step": 3730 + }, + { + "epoch": 60.32520325203252, + "grad_norm": 1.1983181238174438, + "learning_rate": 0.0001, + "loss": 64.7144, + "step": 3740 + }, + { + "epoch": 60.48780487804878, + "grad_norm": 1.3014581203460693, + "learning_rate": 0.0001, + "loss": 64.6219, + "step": 3750 + }, + { + "epoch": 60.65040650406504, + "grad_norm": 1.4954087734222412, + "learning_rate": 0.0001, + "loss": 63.3785, + "step": 3760 + }, + { + "epoch": 60.8130081300813, + "grad_norm": 1.2236649990081787, + "learning_rate": 0.0001, + "loss": 64.6933, + "step": 3770 + }, + { + "epoch": 60.97560975609756, + "grad_norm": 1.0796610116958618, + "learning_rate": 0.0001, + "loss": 63.2704, + "step": 3780 + }, + { + "epoch": 61.13008130081301, + "grad_norm": 0.9969968795776367, + "learning_rate": 0.0001, + "loss": 62.1838, + "step": 3790 + }, + { + "epoch": 61.292682926829265, + "grad_norm": 1.1624361276626587, + "learning_rate": 0.0001, + "loss": 63.6249, + "step": 3800 + }, + { + "epoch": 61.45528455284553, + "grad_norm": 1.3849223852157593, + "learning_rate": 0.0001, + "loss": 64.5321, + "step": 3810 + }, + { + "epoch": 61.61788617886179, + "grad_norm": 1.6607458591461182, + "learning_rate": 0.0001, + "loss": 64.651, + "step": 3820 + }, + { + "epoch": 61.78048780487805, + "grad_norm": 1.09684419631958, + "learning_rate": 0.0001, + "loss": 64.1129, + "step": 3830 + }, + { + "epoch": 61.94308943089431, + "grad_norm": 1.406246304512024, + "learning_rate": 0.0001, + "loss": 64.3642, + "step": 3840 + }, + { + "epoch": 62.09756097560975, + "grad_norm": 0.9850211143493652, + "learning_rate": 0.0001, + "loss": 60.8703, + "step": 3850 + }, + { + "epoch": 62.260162601626014, + "grad_norm": 1.001344084739685, + "learning_rate": 0.0001, + "loss": 67.5523, + "step": 3860 + }, + { + "epoch": 62.422764227642276, + "grad_norm": 2.435969591140747, + "learning_rate": 0.0001, + "loss": 62.7363, + "step": 3870 + }, + { + "epoch": 62.58536585365854, + "grad_norm": 1.0675824880599976, + "learning_rate": 0.0001, + "loss": 64.8513, + "step": 3880 + }, + { + "epoch": 62.7479674796748, + "grad_norm": 0.9793754816055298, + "learning_rate": 0.0001, + "loss": 64.9904, + "step": 3890 + }, + { + "epoch": 62.91056910569106, + "grad_norm": 1.2187775373458862, + "learning_rate": 0.0001, + "loss": 63.2821, + "step": 3900 + }, + { + "epoch": 63.0650406504065, + "grad_norm": 1.130772590637207, + "learning_rate": 0.0001, + "loss": 61.5957, + "step": 3910 + }, + { + "epoch": 63.22764227642276, + "grad_norm": 1.0573055744171143, + "learning_rate": 0.0001, + "loss": 63.6337, + "step": 3920 + }, + { + "epoch": 63.390243902439025, + "grad_norm": 2.055490255355835, + "learning_rate": 0.0001, + "loss": 63.9598, + "step": 3930 + }, + { + "epoch": 63.552845528455286, + "grad_norm": 1.0780516862869263, + "learning_rate": 0.0001, + "loss": 62.9303, + "step": 3940 + }, + { + "epoch": 63.71544715447155, + "grad_norm": 1.8136590719223022, + "learning_rate": 0.0001, + "loss": 64.4588, + "step": 3950 + }, + { + "epoch": 63.8780487804878, + "grad_norm": 1.0525016784667969, + "learning_rate": 0.0001, + "loss": 64.192, + "step": 3960 + }, + { + "epoch": 64.03252032520325, + "grad_norm": 1.12627375125885, + "learning_rate": 0.0001, + "loss": 59.1823, + "step": 3970 + }, + { + "epoch": 64.1951219512195, + "grad_norm": 1.2558730840682983, + "learning_rate": 0.0001, + "loss": 65.126, + "step": 3980 + }, + { + "epoch": 64.35772357723577, + "grad_norm": 1.2304726839065552, + "learning_rate": 0.0001, + "loss": 62.0868, + "step": 3990 + }, + { + "epoch": 64.52032520325203, + "grad_norm": 1.4179201126098633, + "learning_rate": 0.0001, + "loss": 62.3114, + "step": 4000 + }, + { + "epoch": 64.6829268292683, + "grad_norm": 1.125758409500122, + "learning_rate": 0.0001, + "loss": 63.4103, + "step": 4010 + }, + { + "epoch": 64.84552845528455, + "grad_norm": 2.321124315261841, + "learning_rate": 0.0001, + "loss": 62.9032, + "step": 4020 + }, + { + "epoch": 65.0, + "grad_norm": 1.4433993101119995, + "learning_rate": 0.0001, + "loss": 61.8766, + "step": 4030 + }, + { + "epoch": 65.16260162601625, + "grad_norm": 1.157878041267395, + "learning_rate": 0.0001, + "loss": 62.4833, + "step": 4040 + }, + { + "epoch": 65.32520325203252, + "grad_norm": 1.4127166271209717, + "learning_rate": 0.0001, + "loss": 64.768, + "step": 4050 + }, + { + "epoch": 65.48780487804878, + "grad_norm": 1.2575284242630005, + "learning_rate": 0.0001, + "loss": 62.9401, + "step": 4060 + }, + { + "epoch": 65.65040650406505, + "grad_norm": 1.0957911014556885, + "learning_rate": 0.0001, + "loss": 64.6061, + "step": 4070 + }, + { + "epoch": 65.8130081300813, + "grad_norm": 1.1773414611816406, + "learning_rate": 0.0001, + "loss": 63.9469, + "step": 4080 + }, + { + "epoch": 65.97560975609755, + "grad_norm": 2.1189942359924316, + "learning_rate": 0.0001, + "loss": 65.1422, + "step": 4090 + }, + { + "epoch": 66.130081300813, + "grad_norm": 1.1064234972000122, + "learning_rate": 0.0001, + "loss": 59.6108, + "step": 4100 + }, + { + "epoch": 66.29268292682927, + "grad_norm": 1.1135871410369873, + "learning_rate": 0.0001, + "loss": 63.4036, + "step": 4110 + }, + { + "epoch": 66.45528455284553, + "grad_norm": 4.167781829833984, + "learning_rate": 0.0001, + "loss": 63.1174, + "step": 4120 + }, + { + "epoch": 66.6178861788618, + "grad_norm": 1.1429376602172852, + "learning_rate": 0.0001, + "loss": 64.4085, + "step": 4130 + }, + { + "epoch": 66.78048780487805, + "grad_norm": 1.3188339471817017, + "learning_rate": 0.0001, + "loss": 62.5997, + "step": 4140 + }, + { + "epoch": 66.9430894308943, + "grad_norm": 1.160335659980774, + "learning_rate": 0.0001, + "loss": 64.2911, + "step": 4150 + }, + { + "epoch": 67.09756097560975, + "grad_norm": 0.9261873960494995, + "learning_rate": 0.0001, + "loss": 61.9036, + "step": 4160 + }, + { + "epoch": 67.26016260162602, + "grad_norm": 1.2254258394241333, + "learning_rate": 0.0001, + "loss": 64.1092, + "step": 4170 + }, + { + "epoch": 67.42276422764228, + "grad_norm": 0.9519097208976746, + "learning_rate": 0.0001, + "loss": 61.6428, + "step": 4180 + }, + { + "epoch": 67.58536585365853, + "grad_norm": 1.2290887832641602, + "learning_rate": 0.0001, + "loss": 64.9842, + "step": 4190 + }, + { + "epoch": 67.7479674796748, + "grad_norm": 1.0995723009109497, + "learning_rate": 0.0001, + "loss": 64.0677, + "step": 4200 + }, + { + "epoch": 67.91056910569105, + "grad_norm": 2.727957010269165, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 4210 + }, + { + "epoch": 68.0650406504065, + "grad_norm": 1.1222411394119263, + "learning_rate": 0.0001, + "loss": 61.0693, + "step": 4220 + }, + { + "epoch": 68.22764227642277, + "grad_norm": 1.159303069114685, + "learning_rate": 0.0001, + "loss": 64.2469, + "step": 4230 + }, + { + "epoch": 68.39024390243902, + "grad_norm": 1.2205990552902222, + "learning_rate": 0.0001, + "loss": 63.8831, + "step": 4240 + }, + { + "epoch": 68.55284552845528, + "grad_norm": 1.2351205348968506, + "learning_rate": 0.0001, + "loss": 63.4294, + "step": 4250 + }, + { + "epoch": 68.71544715447155, + "grad_norm": 1.4483757019042969, + "learning_rate": 0.0001, + "loss": 63.5345, + "step": 4260 + }, + { + "epoch": 68.8780487804878, + "grad_norm": 1.0808624029159546, + "learning_rate": 0.0001, + "loss": 63.6885, + "step": 4270 + }, + { + "epoch": 69.03252032520325, + "grad_norm": 1.7775999307632446, + "learning_rate": 0.0001, + "loss": 60.7266, + "step": 4280 + }, + { + "epoch": 69.1951219512195, + "grad_norm": 1.3214768171310425, + "learning_rate": 0.0001, + "loss": 63.3496, + "step": 4290 + }, + { + "epoch": 69.35772357723577, + "grad_norm": 1.1698002815246582, + "learning_rate": 0.0001, + "loss": 63.5284, + "step": 4300 + }, + { + "epoch": 69.52032520325203, + "grad_norm": 1.2050056457519531, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 4310 + }, + { + "epoch": 69.6829268292683, + "grad_norm": 1.0873243808746338, + "learning_rate": 0.0001, + "loss": 65.1709, + "step": 4320 + }, + { + "epoch": 69.84552845528455, + "grad_norm": 1.14133620262146, + "learning_rate": 0.0001, + "loss": 63.1872, + "step": 4330 + }, + { + "epoch": 70.0, + "grad_norm": 0.9684088230133057, + "learning_rate": 0.0001, + "loss": 60.1486, + "step": 4340 + }, + { + "epoch": 70.16260162601625, + "grad_norm": 1.1206653118133545, + "learning_rate": 0.0001, + "loss": 61.117, + "step": 4350 + }, + { + "epoch": 70.32520325203252, + "grad_norm": 1.2754334211349487, + "learning_rate": 0.0001, + "loss": 62.5409, + "step": 4360 + }, + { + "epoch": 70.48780487804878, + "grad_norm": 1.0154718160629272, + "learning_rate": 0.0001, + "loss": 62.9428, + "step": 4370 + }, + { + "epoch": 70.65040650406505, + "grad_norm": 1.0597270727157593, + "learning_rate": 0.0001, + "loss": 63.079, + "step": 4380 + }, + { + "epoch": 70.8130081300813, + "grad_norm": 1.254758596420288, + "learning_rate": 0.0001, + "loss": 63.3035, + "step": 4390 + }, + { + "epoch": 70.97560975609755, + "grad_norm": 1.0185120105743408, + "learning_rate": 0.0001, + "loss": 63.8922, + "step": 4400 + }, + { + "epoch": 71.130081300813, + "grad_norm": 1.007300853729248, + "learning_rate": 0.0001, + "loss": 58.7735, + "step": 4410 + }, + { + "epoch": 71.29268292682927, + "grad_norm": 1.2012463808059692, + "learning_rate": 0.0001, + "loss": 63.1782, + "step": 4420 + }, + { + "epoch": 71.45528455284553, + "grad_norm": 1.0258845090866089, + "learning_rate": 0.0001, + "loss": 63.2483, + "step": 4430 + }, + { + "epoch": 71.6178861788618, + "grad_norm": 1.0611220598220825, + "learning_rate": 0.0001, + "loss": 63.2247, + "step": 4440 + }, + { + "epoch": 71.78048780487805, + "grad_norm": 1.060114860534668, + "learning_rate": 0.0001, + "loss": 61.8352, + "step": 4450 + }, + { + "epoch": 71.9430894308943, + "grad_norm": 1.0540589094161987, + "learning_rate": 0.0001, + "loss": 62.1156, + "step": 4460 + }, + { + "epoch": 72.09756097560975, + "grad_norm": 1.4923726320266724, + "learning_rate": 0.0001, + "loss": 59.3863, + "step": 4470 + }, + { + "epoch": 72.26016260162602, + "grad_norm": 1.13791024684906, + "learning_rate": 0.0001, + "loss": 62.203, + "step": 4480 + }, + { + "epoch": 72.42276422764228, + "grad_norm": 0.9737065434455872, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 4490 + }, + { + "epoch": 72.58536585365853, + "grad_norm": 1.3980190753936768, + "learning_rate": 0.0001, + "loss": 63.5627, + "step": 4500 + }, + { + "epoch": 72.7479674796748, + "grad_norm": 1.0274118185043335, + "learning_rate": 0.0001, + "loss": 63.6129, + "step": 4510 + }, + { + "epoch": 72.91056910569105, + "grad_norm": 1.3148925304412842, + "learning_rate": 0.0001, + "loss": 63.6506, + "step": 4520 + }, + { + "epoch": 73.0650406504065, + "grad_norm": 1.4182451963424683, + "learning_rate": 0.0001, + "loss": 60.3616, + "step": 4530 + }, + { + "epoch": 73.22764227642277, + "grad_norm": 0.9235166907310486, + "learning_rate": 0.0001, + "loss": 63.6465, + "step": 4540 + }, + { + "epoch": 73.39024390243902, + "grad_norm": 0.9293469786643982, + "learning_rate": 0.0001, + "loss": 61.0984, + "step": 4550 + }, + { + "epoch": 73.55284552845528, + "grad_norm": 1.1075892448425293, + "learning_rate": 0.0001, + "loss": 62.9356, + "step": 4560 + }, + { + "epoch": 73.71544715447155, + "grad_norm": 1.4955459833145142, + "learning_rate": 0.0001, + "loss": 61.3811, + "step": 4570 + }, + { + "epoch": 73.8780487804878, + "grad_norm": 1.220595359802246, + "learning_rate": 0.0001, + "loss": 62.4364, + "step": 4580 + }, + { + "epoch": 74.03252032520325, + "grad_norm": 0.9975704550743103, + "learning_rate": 0.0001, + "loss": 58.5194, + "step": 4590 + }, + { + "epoch": 74.1951219512195, + "grad_norm": 1.0160857439041138, + "learning_rate": 0.0001, + "loss": 64.6989, + "step": 4600 + }, + { + "epoch": 74.35772357723577, + "grad_norm": 1.0052287578582764, + "learning_rate": 0.0001, + "loss": 62.526, + "step": 4610 + }, + { + "epoch": 74.52032520325203, + "grad_norm": 1.015514612197876, + "learning_rate": 0.0001, + "loss": 63.8627, + "step": 4620 + }, + { + "epoch": 74.6829268292683, + "grad_norm": 1.139051914215088, + "learning_rate": 0.0001, + "loss": 62.4525, + "step": 4630 + }, + { + "epoch": 74.84552845528455, + "grad_norm": 1.1727714538574219, + "learning_rate": 0.0001, + "loss": 64.0046, + "step": 4640 + }, + { + "epoch": 75.0, + "grad_norm": 1.1611062288284302, + "learning_rate": 0.0001, + "loss": 57.6443, + "step": 4650 + }, + { + "epoch": 75.16260162601625, + "grad_norm": 0.9893108606338501, + "learning_rate": 0.0001, + "loss": 64.1109, + "step": 4660 + }, + { + "epoch": 75.32520325203252, + "grad_norm": 1.190146803855896, + "learning_rate": 0.0001, + "loss": 62.1988, + "step": 4670 + }, + { + "epoch": 75.48780487804878, + "grad_norm": 1.1648579835891724, + "learning_rate": 0.0001, + "loss": 63.1393, + "step": 4680 + }, + { + "epoch": 75.65040650406505, + "grad_norm": 1.3303166627883911, + "learning_rate": 0.0001, + "loss": 63.7, + "step": 4690 + }, + { + "epoch": 75.8130081300813, + "grad_norm": 1.407772421836853, + "learning_rate": 0.0001, + "loss": 60.6798, + "step": 4700 + }, + { + "epoch": 75.97560975609755, + "grad_norm": 1.039741039276123, + "learning_rate": 0.0001, + "loss": 61.6126, + "step": 4710 + }, + { + "epoch": 76.130081300813, + "grad_norm": 1.013515830039978, + "learning_rate": 0.0001, + "loss": 57.8452, + "step": 4720 + }, + { + "epoch": 76.29268292682927, + "grad_norm": 1.1279789209365845, + "learning_rate": 0.0001, + "loss": 63.0592, + "step": 4730 + }, + { + "epoch": 76.45528455284553, + "grad_norm": 1.158605694770813, + "learning_rate": 0.0001, + "loss": 62.0836, + "step": 4740 + }, + { + "epoch": 76.6178861788618, + "grad_norm": 1.0200021266937256, + "learning_rate": 0.0001, + "loss": 62.1775, + "step": 4750 + }, + { + "epoch": 76.78048780487805, + "grad_norm": 1.216412901878357, + "learning_rate": 0.0001, + "loss": 62.6316, + "step": 4760 + }, + { + "epoch": 76.9430894308943, + "grad_norm": 1.0513523817062378, + "learning_rate": 0.0001, + "loss": 60.1037, + "step": 4770 + }, + { + "epoch": 77.09756097560975, + "grad_norm": 1.10075843334198, + "learning_rate": 0.0001, + "loss": 58.7599, + "step": 4780 + }, + { + "epoch": 77.26016260162602, + "grad_norm": 1.1728724241256714, + "learning_rate": 0.0001, + "loss": 63.3476, + "step": 4790 + }, + { + "epoch": 77.42276422764228, + "grad_norm": 1.3511605262756348, + "learning_rate": 0.0001, + "loss": 60.9885, + "step": 4800 + }, + { + "epoch": 77.58536585365853, + "grad_norm": 0.9733042120933533, + "learning_rate": 0.0001, + "loss": 62.4245, + "step": 4810 + }, + { + "epoch": 77.7479674796748, + "grad_norm": 1.0380535125732422, + "learning_rate": 0.0001, + "loss": 64.0755, + "step": 4820 + }, + { + "epoch": 77.91056910569105, + "grad_norm": 1.1594985723495483, + "learning_rate": 0.0001, + "loss": 62.1099, + "step": 4830 + }, + { + "epoch": 78.0650406504065, + "grad_norm": 1.0392569303512573, + "learning_rate": 0.0001, + "loss": 58.3778, + "step": 4840 + }, + { + "epoch": 78.22764227642277, + "grad_norm": 1.0032405853271484, + "learning_rate": 0.0001, + "loss": 62.1384, + "step": 4850 + }, + { + "epoch": 78.39024390243902, + "grad_norm": 1.0576013326644897, + "learning_rate": 0.0001, + "loss": 63.3308, + "step": 4860 + }, + { + "epoch": 78.55284552845528, + "grad_norm": 1.34328031539917, + "learning_rate": 0.0001, + "loss": 62.5351, + "step": 4870 + }, + { + "epoch": 78.71544715447155, + "grad_norm": 1.1040822267532349, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 4880 + }, + { + "epoch": 78.8780487804878, + "grad_norm": 1.1554768085479736, + "learning_rate": 0.0001, + "loss": 62.7308, + "step": 4890 + }, + { + "epoch": 79.03252032520325, + "grad_norm": 1.1864408254623413, + "learning_rate": 0.0001, + "loss": 60.0741, + "step": 4900 + }, + { + "epoch": 79.1951219512195, + "grad_norm": 0.9181821346282959, + "learning_rate": 0.0001, + "loss": 63.1682, + "step": 4910 + }, + { + "epoch": 79.35772357723577, + "grad_norm": 1.1400946378707886, + "learning_rate": 0.0001, + "loss": 62.0736, + "step": 4920 + }, + { + "epoch": 79.52032520325203, + "grad_norm": 1.245202660560608, + "learning_rate": 0.0001, + "loss": 60.51, + "step": 4930 + }, + { + "epoch": 79.6829268292683, + "grad_norm": 1.033347725868225, + "learning_rate": 0.0001, + "loss": 61.7845, + "step": 4940 + }, + { + "epoch": 79.84552845528455, + "grad_norm": 1.6206163167953491, + "learning_rate": 0.0001, + "loss": 61.5232, + "step": 4950 + }, + { + "epoch": 80.0, + "grad_norm": 1.0924937725067139, + "learning_rate": 0.0001, + "loss": 57.24, + "step": 4960 + }, + { + "epoch": 80.16260162601625, + "grad_norm": 1.075985074043274, + "learning_rate": 0.0001, + "loss": 61.2615, + "step": 4970 + }, + { + "epoch": 80.32520325203252, + "grad_norm": 0.9964771866798401, + "learning_rate": 0.0001, + "loss": 62.287, + "step": 4980 + }, + { + "epoch": 80.48780487804878, + "grad_norm": 1.3707735538482666, + "learning_rate": 0.0001, + "loss": 60.9383, + "step": 4990 + }, + { + "epoch": 80.65040650406505, + "grad_norm": 0.9395170211791992, + "learning_rate": 0.0001, + "loss": 61.2697, + "step": 5000 + }, + { + "epoch": 80.8130081300813, + "grad_norm": 2.0208873748779297, + "learning_rate": 0.0001, + "loss": 61.7219, + "step": 5010 + }, + { + "epoch": 80.97560975609755, + "grad_norm": 1.468702793121338, + "learning_rate": 0.0001, + "loss": 62.6967, + "step": 5020 + }, + { + "epoch": 81.130081300813, + "grad_norm": 1.0525966882705688, + "learning_rate": 0.0001, + "loss": 59.9793, + "step": 5030 + }, + { + "epoch": 81.29268292682927, + "grad_norm": 1.0951132774353027, + "learning_rate": 0.0001, + "loss": 60.9807, + "step": 5040 + }, + { + "epoch": 81.45528455284553, + "grad_norm": 1.3457857370376587, + "learning_rate": 0.0001, + "loss": 61.4137, + "step": 5050 + }, + { + "epoch": 81.6178861788618, + "grad_norm": 1.0895649194717407, + "learning_rate": 0.0001, + "loss": 61.2747, + "step": 5060 + }, + { + "epoch": 81.78048780487805, + "grad_norm": 1.0552371740341187, + "learning_rate": 0.0001, + "loss": 62.0947, + "step": 5070 + }, + { + "epoch": 81.9430894308943, + "grad_norm": 1.039038896560669, + "learning_rate": 0.0001, + "loss": 61.2571, + "step": 5080 + }, + { + "epoch": 82.09756097560975, + "grad_norm": 0.970783531665802, + "learning_rate": 0.0001, + "loss": 57.4157, + "step": 5090 + }, + { + "epoch": 82.26016260162602, + "grad_norm": 1.0276020765304565, + "learning_rate": 0.0001, + "loss": 62.1324, + "step": 5100 + }, + { + "epoch": 82.42276422764228, + "grad_norm": 1.2092448472976685, + "learning_rate": 0.0001, + "loss": 60.6448, + "step": 5110 + }, + { + "epoch": 82.58536585365853, + "grad_norm": 0.9352007508277893, + "learning_rate": 0.0001, + "loss": 61.9299, + "step": 5120 + }, + { + "epoch": 82.7479674796748, + "grad_norm": 1.2678358554840088, + "learning_rate": 0.0001, + "loss": 61.6622, + "step": 5130 + }, + { + "epoch": 82.91056910569105, + "grad_norm": 1.200494647026062, + "learning_rate": 0.0001, + "loss": 62.3169, + "step": 5140 + }, + { + "epoch": 83.0650406504065, + "grad_norm": 1.1284401416778564, + "learning_rate": 0.0001, + "loss": 58.7768, + "step": 5150 + }, + { + "epoch": 83.22764227642277, + "grad_norm": 1.1578079462051392, + "learning_rate": 0.0001, + "loss": 62.8423, + "step": 5160 + }, + { + "epoch": 83.39024390243902, + "grad_norm": 1.3426504135131836, + "learning_rate": 0.0001, + "loss": 61.5332, + "step": 5170 + }, + { + "epoch": 83.55284552845528, + "grad_norm": 0.9397301077842712, + "learning_rate": 0.0001, + "loss": 60.1043, + "step": 5180 + }, + { + "epoch": 83.71544715447155, + "grad_norm": 1.0728143453598022, + "learning_rate": 0.0001, + "loss": 60.5927, + "step": 5190 + }, + { + "epoch": 83.8780487804878, + "grad_norm": 1.2620211839675903, + "learning_rate": 0.0001, + "loss": 60.9224, + "step": 5200 + }, + { + "epoch": 84.03252032520325, + "grad_norm": 0.8232231140136719, + "learning_rate": 0.0001, + "loss": 58.8128, + "step": 5210 + }, + { + "epoch": 84.1951219512195, + "grad_norm": 1.0222564935684204, + "learning_rate": 0.0001, + "loss": 59.6995, + "step": 5220 + }, + { + "epoch": 84.35772357723577, + "grad_norm": 0.9078415632247925, + "learning_rate": 0.0001, + "loss": 61.1521, + "step": 5230 + }, + { + "epoch": 84.52032520325203, + "grad_norm": 1.0269399881362915, + "learning_rate": 0.0001, + "loss": 61.0136, + "step": 5240 + }, + { + "epoch": 84.6829268292683, + "grad_norm": 1.2108154296875, + "learning_rate": 0.0001, + "loss": 61.3564, + "step": 5250 + }, + { + "epoch": 84.84552845528455, + "grad_norm": 1.1170622110366821, + "learning_rate": 0.0001, + "loss": 61.7047, + "step": 5260 + }, + { + "epoch": 85.0, + "grad_norm": 1.467298150062561, + "learning_rate": 0.0001, + "loss": 58.1667, + "step": 5270 + }, + { + "epoch": 85.16260162601625, + "grad_norm": 0.908743143081665, + "learning_rate": 0.0001, + "loss": 61.5694, + "step": 5280 + }, + { + "epoch": 85.32520325203252, + "grad_norm": 0.9148766398429871, + "learning_rate": 0.0001, + "loss": 61.5325, + "step": 5290 + }, + { + "epoch": 85.48780487804878, + "grad_norm": 0.940021276473999, + "learning_rate": 0.0001, + "loss": 62.8807, + "step": 5300 + }, + { + "epoch": 85.65040650406505, + "grad_norm": 1.1239454746246338, + "learning_rate": 0.0001, + "loss": 62.361, + "step": 5310 + }, + { + "epoch": 85.8130081300813, + "grad_norm": 0.840437114238739, + "learning_rate": 0.0001, + "loss": 61.9057, + "step": 5320 + }, + { + "epoch": 85.97560975609755, + "grad_norm": 0.9764263033866882, + "learning_rate": 0.0001, + "loss": 61.0075, + "step": 5330 + }, + { + "epoch": 86.130081300813, + "grad_norm": 0.998540997505188, + "learning_rate": 0.0001, + "loss": 57.7146, + "step": 5340 + }, + { + "epoch": 86.29268292682927, + "grad_norm": 1.121551513671875, + "learning_rate": 0.0001, + "loss": 62.2278, + "step": 5350 + }, + { + "epoch": 86.45528455284553, + "grad_norm": 1.101842999458313, + "learning_rate": 0.0001, + "loss": 60.8837, + "step": 5360 + }, + { + "epoch": 86.6178861788618, + "grad_norm": 1.1724908351898193, + "learning_rate": 0.0001, + "loss": 61.6788, + "step": 5370 + }, + { + "epoch": 86.78048780487805, + "grad_norm": 1.3108958005905151, + "learning_rate": 0.0001, + "loss": 60.1461, + "step": 5380 + }, + { + "epoch": 86.9430894308943, + "grad_norm": 1.482298731803894, + "learning_rate": 0.0001, + "loss": 61.0457, + "step": 5390 + }, + { + "epoch": 87.09756097560975, + "grad_norm": 1.1471034288406372, + "learning_rate": 0.0001, + "loss": 57.6835, + "step": 5400 + }, + { + "epoch": 87.26016260162602, + "grad_norm": 0.9349374175071716, + "learning_rate": 0.0001, + "loss": 62.7018, + "step": 5410 + }, + { + "epoch": 87.42276422764228, + "grad_norm": 0.9917856454849243, + "learning_rate": 0.0001, + "loss": 60.2201, + "step": 5420 + }, + { + "epoch": 87.58536585365853, + "grad_norm": 1.0112310647964478, + "learning_rate": 0.0001, + "loss": 62.1185, + "step": 5430 + }, + { + "epoch": 87.7479674796748, + "grad_norm": 1.288894534111023, + "learning_rate": 0.0001, + "loss": 61.4348, + "step": 5440 + }, + { + "epoch": 87.91056910569105, + "grad_norm": 0.8577322363853455, + "learning_rate": 0.0001, + "loss": 60.7867, + "step": 5450 + }, + { + "epoch": 88.0650406504065, + "grad_norm": 0.8162860870361328, + "learning_rate": 0.0001, + "loss": 59.5224, + "step": 5460 + }, + { + "epoch": 88.22764227642277, + "grad_norm": 0.9491421580314636, + "learning_rate": 0.0001, + "loss": 60.4936, + "step": 5470 + }, + { + "epoch": 88.39024390243902, + "grad_norm": 4.164788246154785, + "learning_rate": 0.0001, + "loss": 61.8397, + "step": 5480 + }, + { + "epoch": 88.55284552845528, + "grad_norm": 0.9954016804695129, + "learning_rate": 0.0001, + "loss": 61.286, + "step": 5490 + }, + { + "epoch": 88.71544715447155, + "grad_norm": 1.4991120100021362, + "learning_rate": 0.0001, + "loss": 60.6157, + "step": 5500 + }, + { + "epoch": 88.8780487804878, + "grad_norm": 1.3678480386734009, + "learning_rate": 0.0001, + "loss": 62.5187, + "step": 5510 + }, + { + "epoch": 89.03252032520325, + "grad_norm": 1.057661771774292, + "learning_rate": 0.0001, + "loss": 57.5134, + "step": 5520 + }, + { + "epoch": 89.1951219512195, + "grad_norm": 0.8331280946731567, + "learning_rate": 0.0001, + "loss": 62.007, + "step": 5530 + }, + { + "epoch": 89.35772357723577, + "grad_norm": 0.9842398762702942, + "learning_rate": 0.0001, + "loss": 60.6178, + "step": 5540 + }, + { + "epoch": 89.52032520325203, + "grad_norm": 1.0510483980178833, + "learning_rate": 0.0001, + "loss": 59.6382, + "step": 5550 + }, + { + "epoch": 89.6829268292683, + "grad_norm": 1.013115406036377, + "learning_rate": 0.0001, + "loss": 61.015, + "step": 5560 + }, + { + "epoch": 89.84552845528455, + "grad_norm": 1.1595944166183472, + "learning_rate": 0.0001, + "loss": 62.5374, + "step": 5570 + }, + { + "epoch": 90.0, + "grad_norm": 1.1506270170211792, + "learning_rate": 0.0001, + "loss": 61.0063, + "step": 5580 + }, + { + "epoch": 90.16260162601625, + "grad_norm": 0.9940153360366821, + "learning_rate": 0.0001, + "loss": 62.0944, + "step": 5590 + }, + { + "epoch": 90.32520325203252, + "grad_norm": 1.1302493810653687, + "learning_rate": 0.0001, + "loss": 61.616, + "step": 5600 + }, + { + "epoch": 90.48780487804878, + "grad_norm": 1.2370656728744507, + "learning_rate": 0.0001, + "loss": 61.291, + "step": 5610 + }, + { + "epoch": 90.65040650406505, + "grad_norm": 1.228763461112976, + "learning_rate": 0.0001, + "loss": 60.0629, + "step": 5620 + }, + { + "epoch": 90.8130081300813, + "grad_norm": 1.2230069637298584, + "learning_rate": 0.0001, + "loss": 61.5391, + "step": 5630 + }, + { + "epoch": 90.97560975609755, + "grad_norm": 0.9849008917808533, + "learning_rate": 0.0001, + "loss": 63.1633, + "step": 5640 + }, + { + "epoch": 91.130081300813, + "grad_norm": 1.408632755279541, + "learning_rate": 0.0001, + "loss": 58.4195, + "step": 5650 + }, + { + "epoch": 91.29268292682927, + "grad_norm": 0.929027795791626, + "learning_rate": 0.0001, + "loss": 60.9862, + "step": 5660 + }, + { + "epoch": 91.45528455284553, + "grad_norm": 0.9228377342224121, + "learning_rate": 0.0001, + "loss": 60.8337, + "step": 5670 + }, + { + "epoch": 91.6178861788618, + "grad_norm": 1.0354220867156982, + "learning_rate": 0.0001, + "loss": 59.4667, + "step": 5680 + }, + { + "epoch": 91.78048780487805, + "grad_norm": 1.0338927507400513, + "learning_rate": 0.0001, + "loss": 62.0064, + "step": 5690 + }, + { + "epoch": 91.9430894308943, + "grad_norm": 1.101967215538025, + "learning_rate": 0.0001, + "loss": 61.6317, + "step": 5700 + }, + { + "epoch": 92.09756097560975, + "grad_norm": 1.370689034461975, + "learning_rate": 0.0001, + "loss": 57.2157, + "step": 5710 + }, + { + "epoch": 92.26016260162602, + "grad_norm": 0.9904115796089172, + "learning_rate": 0.0001, + "loss": 61.2786, + "step": 5720 + }, + { + "epoch": 92.42276422764228, + "grad_norm": 1.0545448064804077, + "learning_rate": 0.0001, + "loss": 63.2384, + "step": 5730 + }, + { + "epoch": 92.58536585365853, + "grad_norm": 1.0657731294631958, + "learning_rate": 0.0001, + "loss": 60.582, + "step": 5740 + }, + { + "epoch": 92.7479674796748, + "grad_norm": 0.9333183765411377, + "learning_rate": 0.0001, + "loss": 62.4627, + "step": 5750 + }, + { + "epoch": 92.91056910569105, + "grad_norm": 0.9810492396354675, + "learning_rate": 0.0001, + "loss": 60.0055, + "step": 5760 + }, + { + "epoch": 93.0650406504065, + "grad_norm": 1.6973884105682373, + "learning_rate": 0.0001, + "loss": 57.5381, + "step": 5770 + }, + { + "epoch": 93.22764227642277, + "grad_norm": 1.0075498819351196, + "learning_rate": 0.0001, + "loss": 61.2272, + "step": 5780 + }, + { + "epoch": 93.39024390243902, + "grad_norm": 0.980633556842804, + "learning_rate": 0.0001, + "loss": 60.4794, + "step": 5790 + }, + { + "epoch": 93.55284552845528, + "grad_norm": 1.128214955329895, + "learning_rate": 0.0001, + "loss": 60.4825, + "step": 5800 + }, + { + "epoch": 93.71544715447155, + "grad_norm": 1.302827000617981, + "learning_rate": 0.0001, + "loss": 62.4397, + "step": 5810 + }, + { + "epoch": 93.8780487804878, + "grad_norm": 0.826600193977356, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 5820 + }, + { + "epoch": 94.03252032520325, + "grad_norm": 11.418835639953613, + "learning_rate": 0.0001, + "loss": 58.5383, + "step": 5830 + }, + { + "epoch": 94.1951219512195, + "grad_norm": 1.0686477422714233, + "learning_rate": 0.0001, + "loss": 60.5696, + "step": 5840 + }, + { + "epoch": 94.35772357723577, + "grad_norm": 1.1232597827911377, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 5850 + }, + { + "epoch": 94.52032520325203, + "grad_norm": 1.1051944494247437, + "learning_rate": 0.0001, + "loss": 60.5387, + "step": 5860 + }, + { + "epoch": 94.6829268292683, + "grad_norm": 0.9954739809036255, + "learning_rate": 0.0001, + "loss": 61.3899, + "step": 5870 + }, + { + "epoch": 94.84552845528455, + "grad_norm": 1.9687999486923218, + "learning_rate": 0.0001, + "loss": 60.7711, + "step": 5880 + }, + { + "epoch": 95.0, + "grad_norm": 1.3172541856765747, + "learning_rate": 0.0001, + "loss": 58.5394, + "step": 5890 + }, + { + "epoch": 95.16260162601625, + "grad_norm": 1.2251427173614502, + "learning_rate": 0.0001, + "loss": 61.8319, + "step": 5900 + }, + { + "epoch": 95.32520325203252, + "grad_norm": 1.1701544523239136, + "learning_rate": 0.0001, + "loss": 60.1928, + "step": 5910 + }, + { + "epoch": 95.48780487804878, + "grad_norm": 1.209375262260437, + "learning_rate": 0.0001, + "loss": 61.1347, + "step": 5920 + }, + { + "epoch": 95.65040650406505, + "grad_norm": 0.9307044148445129, + "learning_rate": 0.0001, + "loss": 62.6676, + "step": 5930 + }, + { + "epoch": 95.8130081300813, + "grad_norm": 0.9679144024848938, + "learning_rate": 0.0001, + "loss": 60.7623, + "step": 5940 + }, + { + "epoch": 95.97560975609755, + "grad_norm": 1.398711085319519, + "learning_rate": 0.0001, + "loss": 59.0407, + "step": 5950 + }, + { + "epoch": 96.130081300813, + "grad_norm": 1.9118247032165527, + "learning_rate": 0.0001, + "loss": 59.9417, + "step": 5960 + }, + { + "epoch": 96.29268292682927, + "grad_norm": 1.036737084388733, + "learning_rate": 0.0001, + "loss": 60.6891, + "step": 5970 + }, + { + "epoch": 96.45528455284553, + "grad_norm": 1.5916132926940918, + "learning_rate": 0.0001, + "loss": 59.44, + "step": 5980 + }, + { + "epoch": 96.6178861788618, + "grad_norm": 1.526591181755066, + "learning_rate": 0.0001, + "loss": 58.9081, + "step": 5990 + }, + { + "epoch": 96.78048780487805, + "grad_norm": 0.9556435942649841, + "learning_rate": 0.0001, + "loss": 61.5743, + "step": 6000 + }, + { + "epoch": 96.9430894308943, + "grad_norm": 1.0041887760162354, + "learning_rate": 0.0001, + "loss": 60.0724, + "step": 6010 + }, + { + "epoch": 97.09756097560975, + "grad_norm": 0.9792081117630005, + "learning_rate": 0.0001, + "loss": 60.0613, + "step": 6020 + }, + { + "epoch": 97.26016260162602, + "grad_norm": 1.029634714126587, + "learning_rate": 0.0001, + "loss": 60.3526, + "step": 6030 + }, + { + "epoch": 97.42276422764228, + "grad_norm": 0.9812865257263184, + "learning_rate": 0.0001, + "loss": 60.9818, + "step": 6040 + }, + { + "epoch": 97.58536585365853, + "grad_norm": 0.8826327323913574, + "learning_rate": 0.0001, + "loss": 61.6375, + "step": 6050 + }, + { + "epoch": 97.7479674796748, + "grad_norm": 1.2197599411010742, + "learning_rate": 0.0001, + "loss": 60.2698, + "step": 6060 + }, + { + "epoch": 97.91056910569105, + "grad_norm": 0.9068922400474548, + "learning_rate": 0.0001, + "loss": 58.8698, + "step": 6070 + }, + { + "epoch": 98.0650406504065, + "grad_norm": 0.9008221626281738, + "learning_rate": 0.0001, + "loss": 56.4687, + "step": 6080 + }, + { + "epoch": 98.22764227642277, + "grad_norm": 1.8165878057479858, + "learning_rate": 0.0001, + "loss": 61.4213, + "step": 6090 + }, + { + "epoch": 98.39024390243902, + "grad_norm": 0.9541304111480713, + "learning_rate": 0.0001, + "loss": 62.8381, + "step": 6100 + }, + { + "epoch": 98.55284552845528, + "grad_norm": 1.293872594833374, + "learning_rate": 0.0001, + "loss": 61.3292, + "step": 6110 + }, + { + "epoch": 98.71544715447155, + "grad_norm": 0.9569422006607056, + "learning_rate": 0.0001, + "loss": 61.436, + "step": 6120 + }, + { + "epoch": 98.8780487804878, + "grad_norm": 1.160468578338623, + "learning_rate": 0.0001, + "loss": 60.8419, + "step": 6130 + }, + { + "epoch": 99.03252032520325, + "grad_norm": 1.3102303743362427, + "learning_rate": 0.0001, + "loss": 59.0064, + "step": 6140 + }, + { + "epoch": 99.1951219512195, + "grad_norm": 0.9553463459014893, + "learning_rate": 0.0001, + "loss": 59.925, + "step": 6150 + }, + { + "epoch": 99.35772357723577, + "grad_norm": 1.1993688344955444, + "learning_rate": 0.0001, + "loss": 62.3683, + "step": 6160 + }, + { + "epoch": 99.52032520325203, + "grad_norm": 1.0867350101470947, + "learning_rate": 0.0001, + "loss": 61.8848, + "step": 6170 + }, + { + "epoch": 99.6829268292683, + "grad_norm": 0.9924171566963196, + "learning_rate": 0.0001, + "loss": 60.9243, + "step": 6180 + }, + { + "epoch": 99.84552845528455, + "grad_norm": 0.9548614025115967, + "learning_rate": 0.0001, + "loss": 61.0234, + "step": 6190 + }, + { + "epoch": 100.0, + "grad_norm": 1.0070925951004028, + "learning_rate": 0.0001, + "loss": 56.6554, + "step": 6200 + }, + { + "epoch": 100.16260162601625, + "grad_norm": 1.202168583869934, + "learning_rate": 0.0001, + "loss": 61.3235, + "step": 6210 + }, + { + "epoch": 100.32520325203252, + "grad_norm": 0.9931091070175171, + "learning_rate": 0.0001, + "loss": 59.651, + "step": 6220 + }, + { + "epoch": 100.48780487804878, + "grad_norm": 0.9679157137870789, + "learning_rate": 0.0001, + "loss": 61.5785, + "step": 6230 + }, + { + "epoch": 100.65040650406505, + "grad_norm": 1.049842119216919, + "learning_rate": 0.0001, + "loss": 58.9646, + "step": 6240 + }, + { + "epoch": 100.8130081300813, + "grad_norm": 1.0772044658660889, + "learning_rate": 0.0001, + "loss": 62.3309, + "step": 6250 + }, + { + "epoch": 100.97560975609755, + "grad_norm": 0.9494168162345886, + "learning_rate": 0.0001, + "loss": 61.8663, + "step": 6260 + }, + { + "epoch": 101.130081300813, + "grad_norm": 1.0491962432861328, + "learning_rate": 0.0001, + "loss": 56.5872, + "step": 6270 + }, + { + "epoch": 101.29268292682927, + "grad_norm": 1.118611216545105, + "learning_rate": 0.0001, + "loss": 58.1338, + "step": 6280 + }, + { + "epoch": 101.45528455284553, + "grad_norm": 1.0419825315475464, + "learning_rate": 0.0001, + "loss": 59.6661, + "step": 6290 + }, + { + "epoch": 101.6178861788618, + "grad_norm": 1.2351056337356567, + "learning_rate": 0.0001, + "loss": 61.1697, + "step": 6300 + }, + { + "epoch": 101.78048780487805, + "grad_norm": 1.0556049346923828, + "learning_rate": 0.0001, + "loss": 60.463, + "step": 6310 + }, + { + "epoch": 101.9430894308943, + "grad_norm": 1.090761661529541, + "learning_rate": 0.0001, + "loss": 61.983, + "step": 6320 + }, + { + "epoch": 102.09756097560975, + "grad_norm": 1.4305297136306763, + "learning_rate": 0.0001, + "loss": 55.559, + "step": 6330 + }, + { + "epoch": 102.26016260162602, + "grad_norm": 1.1773794889450073, + "learning_rate": 0.0001, + "loss": 59.5931, + "step": 6340 + }, + { + "epoch": 102.42276422764228, + "grad_norm": 1.0028730630874634, + "learning_rate": 0.0001, + "loss": 60.2714, + "step": 6350 + }, + { + "epoch": 102.58536585365853, + "grad_norm": 0.9494019150733948, + "learning_rate": 0.0001, + "loss": 59.7757, + "step": 6360 + }, + { + "epoch": 102.7479674796748, + "grad_norm": 1.0415751934051514, + "learning_rate": 0.0001, + "loss": 60.7123, + "step": 6370 + }, + { + "epoch": 102.91056910569105, + "grad_norm": 1.2036449909210205, + "learning_rate": 0.0001, + "loss": 60.1201, + "step": 6380 + }, + { + "epoch": 103.0650406504065, + "grad_norm": 0.8801053166389465, + "learning_rate": 0.0001, + "loss": 56.4943, + "step": 6390 + }, + { + "epoch": 103.22764227642277, + "grad_norm": 0.9018735289573669, + "learning_rate": 0.0001, + "loss": 62.4522, + "step": 6400 + }, + { + "epoch": 103.39024390243902, + "grad_norm": 0.971685528755188, + "learning_rate": 0.0001, + "loss": 62.2367, + "step": 6410 + }, + { + "epoch": 103.55284552845528, + "grad_norm": 0.8531002402305603, + "learning_rate": 0.0001, + "loss": 59.5436, + "step": 6420 + }, + { + "epoch": 103.71544715447155, + "grad_norm": 2.123033285140991, + "learning_rate": 0.0001, + "loss": 59.0421, + "step": 6430 + }, + { + "epoch": 103.8780487804878, + "grad_norm": 1.209804654121399, + "learning_rate": 0.0001, + "loss": 60.7791, + "step": 6440 + }, + { + "epoch": 104.03252032520325, + "grad_norm": 0.8219161033630371, + "learning_rate": 0.0001, + "loss": 55.7522, + "step": 6450 + }, + { + "epoch": 104.1951219512195, + "grad_norm": 1.8217495679855347, + "learning_rate": 0.0001, + "loss": 60.6916, + "step": 6460 + }, + { + "epoch": 104.35772357723577, + "grad_norm": 1.060073971748352, + "learning_rate": 0.0001, + "loss": 59.971, + "step": 6470 + }, + { + "epoch": 104.52032520325203, + "grad_norm": 0.9892823696136475, + "learning_rate": 0.0001, + "loss": 61.467, + "step": 6480 + }, + { + "epoch": 104.6829268292683, + "grad_norm": 1.128934621810913, + "learning_rate": 0.0001, + "loss": 62.0451, + "step": 6490 + }, + { + "epoch": 104.84552845528455, + "grad_norm": 0.9627387523651123, + "learning_rate": 0.0001, + "loss": 61.532, + "step": 6500 + }, + { + "epoch": 105.0, + "grad_norm": 1.5344375371932983, + "learning_rate": 0.0001, + "loss": 56.4323, + "step": 6510 + }, + { + "epoch": 105.16260162601625, + "grad_norm": 0.9524087309837341, + "learning_rate": 0.0001, + "loss": 59.6628, + "step": 6520 + }, + { + "epoch": 105.32520325203252, + "grad_norm": 1.0469943284988403, + "learning_rate": 0.0001, + "loss": 59.0605, + "step": 6530 + }, + { + "epoch": 105.48780487804878, + "grad_norm": 2.7462990283966064, + "learning_rate": 0.0001, + "loss": 61.1429, + "step": 6540 + }, + { + "epoch": 105.65040650406505, + "grad_norm": 1.3140382766723633, + "learning_rate": 0.0001, + "loss": 58.2711, + "step": 6550 + }, + { + "epoch": 105.8130081300813, + "grad_norm": 1.1528750658035278, + "learning_rate": 0.0001, + "loss": 59.9937, + "step": 6560 + }, + { + "epoch": 105.97560975609755, + "grad_norm": 1.4449951648712158, + "learning_rate": 0.0001, + "loss": 59.1526, + "step": 6570 + }, + { + "epoch": 106.130081300813, + "grad_norm": 1.1350126266479492, + "learning_rate": 0.0001, + "loss": 55.1142, + "step": 6580 + }, + { + "epoch": 106.29268292682927, + "grad_norm": 0.9332845211029053, + "learning_rate": 0.0001, + "loss": 59.8049, + "step": 6590 + }, + { + "epoch": 106.45528455284553, + "grad_norm": 0.9678266644477844, + "learning_rate": 0.0001, + "loss": 60.9627, + "step": 6600 + }, + { + "epoch": 106.6178861788618, + "grad_norm": 1.1794427633285522, + "learning_rate": 0.0001, + "loss": 59.5372, + "step": 6610 + }, + { + "epoch": 106.78048780487805, + "grad_norm": 1.0404181480407715, + "learning_rate": 0.0001, + "loss": 59.2715, + "step": 6620 + }, + { + "epoch": 106.9430894308943, + "grad_norm": 0.8667073845863342, + "learning_rate": 0.0001, + "loss": 60.2625, + "step": 6630 + }, + { + "epoch": 107.09756097560975, + "grad_norm": 0.9940314888954163, + "learning_rate": 0.0001, + "loss": 55.6906, + "step": 6640 + }, + { + "epoch": 107.26016260162602, + "grad_norm": 1.3880919218063354, + "learning_rate": 0.0001, + "loss": 59.8103, + "step": 6650 + }, + { + "epoch": 107.42276422764228, + "grad_norm": 0.900931179523468, + "learning_rate": 0.0001, + "loss": 59.1929, + "step": 6660 + }, + { + "epoch": 107.58536585365853, + "grad_norm": 0.9374089241027832, + "learning_rate": 0.0001, + "loss": 60.1047, + "step": 6670 + }, + { + "epoch": 107.7479674796748, + "grad_norm": 1.0847140550613403, + "learning_rate": 0.0001, + "loss": 58.6943, + "step": 6680 + }, + { + "epoch": 107.91056910569105, + "grad_norm": 1.086506962776184, + "learning_rate": 0.0001, + "loss": 60.0341, + "step": 6690 + }, + { + "epoch": 108.0650406504065, + "grad_norm": 0.8984777331352234, + "learning_rate": 0.0001, + "loss": 57.0422, + "step": 6700 + }, + { + "epoch": 108.22764227642277, + "grad_norm": 1.187198519706726, + "learning_rate": 0.0001, + "loss": 60.7935, + "step": 6710 + }, + { + "epoch": 108.39024390243902, + "grad_norm": 0.8393696546554565, + "learning_rate": 0.0001, + "loss": 59.7896, + "step": 6720 + }, + { + "epoch": 108.55284552845528, + "grad_norm": 1.0898207426071167, + "learning_rate": 0.0001, + "loss": 60.2579, + "step": 6730 + }, + { + "epoch": 108.71544715447155, + "grad_norm": 1.016926646232605, + "learning_rate": 0.0001, + "loss": 60.9018, + "step": 6740 + }, + { + "epoch": 108.8780487804878, + "grad_norm": 1.3991293907165527, + "learning_rate": 0.0001, + "loss": 60.889, + "step": 6750 + }, + { + "epoch": 109.03252032520325, + "grad_norm": 0.9950843453407288, + "learning_rate": 0.0001, + "loss": 55.1098, + "step": 6760 + }, + { + "epoch": 109.1951219512195, + "grad_norm": 1.0843164920806885, + "learning_rate": 0.0001, + "loss": 60.3208, + "step": 6770 + }, + { + "epoch": 109.35772357723577, + "grad_norm": 0.9067014455795288, + "learning_rate": 0.0001, + "loss": 59.1254, + "step": 6780 + }, + { + "epoch": 109.52032520325203, + "grad_norm": 1.2166807651519775, + "learning_rate": 0.0001, + "loss": 59.9978, + "step": 6790 + }, + { + "epoch": 109.6829268292683, + "grad_norm": 1.256101131439209, + "learning_rate": 0.0001, + "loss": 60.2544, + "step": 6800 + }, + { + "epoch": 109.84552845528455, + "grad_norm": 1.312761664390564, + "learning_rate": 0.0001, + "loss": 59.1564, + "step": 6810 + }, + { + "epoch": 110.0, + "grad_norm": 0.9668724536895752, + "learning_rate": 0.0001, + "loss": 58.067, + "step": 6820 + }, + { + "epoch": 110.16260162601625, + "grad_norm": 1.1048885583877563, + "learning_rate": 0.0001, + "loss": 59.1506, + "step": 6830 + }, + { + "epoch": 110.32520325203252, + "grad_norm": 1.498469352722168, + "learning_rate": 0.0001, + "loss": 59.1366, + "step": 6840 + }, + { + "epoch": 110.48780487804878, + "grad_norm": 1.1747376918792725, + "learning_rate": 0.0001, + "loss": 59.8449, + "step": 6850 + }, + { + "epoch": 110.65040650406505, + "grad_norm": 1.1041772365570068, + "learning_rate": 0.0001, + "loss": 61.0879, + "step": 6860 + }, + { + "epoch": 110.8130081300813, + "grad_norm": 0.9452431797981262, + "learning_rate": 0.0001, + "loss": 60.8911, + "step": 6870 + }, + { + "epoch": 110.97560975609755, + "grad_norm": 1.0108182430267334, + "learning_rate": 0.0001, + "loss": 58.7924, + "step": 6880 + }, + { + "epoch": 111.130081300813, + "grad_norm": 0.9914487600326538, + "learning_rate": 0.0001, + "loss": 57.4749, + "step": 6890 + }, + { + "epoch": 111.29268292682927, + "grad_norm": 1.0404850244522095, + "learning_rate": 0.0001, + "loss": 59.5329, + "step": 6900 + }, + { + "epoch": 111.45528455284553, + "grad_norm": 1.0363966226577759, + "learning_rate": 0.0001, + "loss": 58.8972, + "step": 6910 + }, + { + "epoch": 111.6178861788618, + "grad_norm": 1.0189651250839233, + "learning_rate": 0.0001, + "loss": 59.3613, + "step": 6920 + }, + { + "epoch": 111.78048780487805, + "grad_norm": 0.9696914553642273, + "learning_rate": 0.0001, + "loss": 59.8001, + "step": 6930 + }, + { + "epoch": 111.9430894308943, + "grad_norm": 1.406821370124817, + "learning_rate": 0.0001, + "loss": 58.6405, + "step": 6940 + }, + { + "epoch": 112.09756097560975, + "grad_norm": 1.0969383716583252, + "learning_rate": 0.0001, + "loss": 57.4458, + "step": 6950 + }, + { + "epoch": 112.26016260162602, + "grad_norm": 1.117593765258789, + "learning_rate": 0.0001, + "loss": 62.599, + "step": 6960 + }, + { + "epoch": 112.42276422764228, + "grad_norm": 0.9327741861343384, + "learning_rate": 0.0001, + "loss": 60.632, + "step": 6970 + }, + { + "epoch": 112.58536585365853, + "grad_norm": 1.1198201179504395, + "learning_rate": 0.0001, + "loss": 59.6119, + "step": 6980 + }, + { + "epoch": 112.7479674796748, + "grad_norm": 0.8709354400634766, + "learning_rate": 0.0001, + "loss": 60.831, + "step": 6990 + }, + { + "epoch": 112.91056910569105, + "grad_norm": 0.9589444994926453, + "learning_rate": 0.0001, + "loss": 58.0002, + "step": 7000 + }, + { + "epoch": 113.0650406504065, + "grad_norm": 1.0971899032592773, + "learning_rate": 0.0001, + "loss": 56.8572, + "step": 7010 + }, + { + "epoch": 113.22764227642277, + "grad_norm": 0.9764513969421387, + "learning_rate": 0.0001, + "loss": 59.3038, + "step": 7020 + }, + { + "epoch": 113.39024390243902, + "grad_norm": 0.9050093293190002, + "learning_rate": 0.0001, + "loss": 59.9146, + "step": 7030 + }, + { + "epoch": 113.55284552845528, + "grad_norm": 0.9506774544715881, + "learning_rate": 0.0001, + "loss": 59.7523, + "step": 7040 + }, + { + "epoch": 113.71544715447155, + "grad_norm": 0.9678346514701843, + "learning_rate": 0.0001, + "loss": 61.3103, + "step": 7050 + }, + { + "epoch": 113.8780487804878, + "grad_norm": 0.9859515428543091, + "learning_rate": 0.0001, + "loss": 56.3866, + "step": 7060 + }, + { + "epoch": 114.03252032520325, + "grad_norm": 1.2696164846420288, + "learning_rate": 0.0001, + "loss": 58.16, + "step": 7070 + }, + { + "epoch": 114.1951219512195, + "grad_norm": 0.9323938488960266, + "learning_rate": 0.0001, + "loss": 58.9483, + "step": 7080 + }, + { + "epoch": 114.35772357723577, + "grad_norm": 1.4123785495758057, + "learning_rate": 0.0001, + "loss": 60.4666, + "step": 7090 + }, + { + "epoch": 114.52032520325203, + "grad_norm": 1.009020447731018, + "learning_rate": 0.0001, + "loss": 58.7152, + "step": 7100 + }, + { + "epoch": 114.6829268292683, + "grad_norm": 1.219383955001831, + "learning_rate": 0.0001, + "loss": 59.1494, + "step": 7110 + }, + { + "epoch": 114.84552845528455, + "grad_norm": 0.9579033255577087, + "learning_rate": 0.0001, + "loss": 58.9879, + "step": 7120 + }, + { + "epoch": 115.0, + "grad_norm": 0.899074912071228, + "learning_rate": 0.0001, + "loss": 55.2031, + "step": 7130 + }, + { + "epoch": 115.16260162601625, + "grad_norm": 0.9924830794334412, + "learning_rate": 0.0001, + "loss": 58.583, + "step": 7140 + }, + { + "epoch": 115.32520325203252, + "grad_norm": 1.0280295610427856, + "learning_rate": 0.0001, + "loss": 60.0948, + "step": 7150 + }, + { + "epoch": 115.48780487804878, + "grad_norm": 0.8108797669410706, + "learning_rate": 0.0001, + "loss": 60.9433, + "step": 7160 + }, + { + "epoch": 115.65040650406505, + "grad_norm": 1.2597944736480713, + "learning_rate": 0.0001, + "loss": 59.1905, + "step": 7170 + }, + { + "epoch": 115.8130081300813, + "grad_norm": 1.2565767765045166, + "learning_rate": 0.0001, + "loss": 59.3545, + "step": 7180 + }, + { + "epoch": 115.97560975609755, + "grad_norm": 1.2012240886688232, + "learning_rate": 0.0001, + "loss": 58.348, + "step": 7190 + }, + { + "epoch": 116.130081300813, + "grad_norm": 1.2883665561676025, + "learning_rate": 0.0001, + "loss": 57.462, + "step": 7200 + }, + { + "epoch": 116.29268292682927, + "grad_norm": 0.904152512550354, + "learning_rate": 0.0001, + "loss": 58.958, + "step": 7210 + }, + { + "epoch": 116.45528455284553, + "grad_norm": 1.1928468942642212, + "learning_rate": 0.0001, + "loss": 58.6106, + "step": 7220 + }, + { + "epoch": 116.6178861788618, + "grad_norm": 1.0655683279037476, + "learning_rate": 0.0001, + "loss": 58.4948, + "step": 7230 + }, + { + "epoch": 116.78048780487805, + "grad_norm": 1.0527434349060059, + "learning_rate": 0.0001, + "loss": 60.1771, + "step": 7240 + }, + { + "epoch": 116.9430894308943, + "grad_norm": 1.1439871788024902, + "learning_rate": 0.0001, + "loss": 58.6737, + "step": 7250 + }, + { + "epoch": 117.09756097560975, + "grad_norm": 1.506571650505066, + "learning_rate": 0.0001, + "loss": 58.0901, + "step": 7260 + }, + { + "epoch": 117.26016260162602, + "grad_norm": 1.1419955492019653, + "learning_rate": 0.0001, + "loss": 59.0705, + "step": 7270 + }, + { + "epoch": 117.42276422764228, + "grad_norm": 1.1211196184158325, + "learning_rate": 0.0001, + "loss": 57.801, + "step": 7280 + }, + { + "epoch": 117.58536585365853, + "grad_norm": 1.1424155235290527, + "learning_rate": 0.0001, + "loss": 59.7449, + "step": 7290 + }, + { + "epoch": 117.7479674796748, + "grad_norm": 1.0593281984329224, + "learning_rate": 0.0001, + "loss": 60.1789, + "step": 7300 + }, + { + "epoch": 117.91056910569105, + "grad_norm": 1.3680177927017212, + "learning_rate": 0.0001, + "loss": 58.4225, + "step": 7310 + }, + { + "epoch": 118.0650406504065, + "grad_norm": 0.8391700983047485, + "learning_rate": 0.0001, + "loss": 56.2102, + "step": 7320 + }, + { + "epoch": 118.22764227642277, + "grad_norm": 1.3166592121124268, + "learning_rate": 0.0001, + "loss": 61.0399, + "step": 7330 + }, + { + "epoch": 118.39024390243902, + "grad_norm": 1.929949402809143, + "learning_rate": 0.0001, + "loss": 59.6514, + "step": 7340 + }, + { + "epoch": 118.55284552845528, + "grad_norm": 0.9464254975318909, + "learning_rate": 0.0001, + "loss": 58.6488, + "step": 7350 + }, + { + "epoch": 118.71544715447155, + "grad_norm": 1.1066193580627441, + "learning_rate": 0.0001, + "loss": 61.5959, + "step": 7360 + }, + { + "epoch": 118.8780487804878, + "grad_norm": 1.216125726699829, + "learning_rate": 0.0001, + "loss": 58.8231, + "step": 7370 + }, + { + "epoch": 119.03252032520325, + "grad_norm": 1.1182583570480347, + "learning_rate": 0.0001, + "loss": 56.9496, + "step": 7380 + }, + { + "epoch": 119.1951219512195, + "grad_norm": 1.105729341506958, + "learning_rate": 0.0001, + "loss": 58.4291, + "step": 7390 + }, + { + "epoch": 119.35772357723577, + "grad_norm": 1.017879843711853, + "learning_rate": 0.0001, + "loss": 59.4963, + "step": 7400 + }, + { + "epoch": 119.52032520325203, + "grad_norm": 0.7882587909698486, + "learning_rate": 0.0001, + "loss": 57.7243, + "step": 7410 + }, + { + "epoch": 119.6829268292683, + "grad_norm": 0.8356695175170898, + "learning_rate": 0.0001, + "loss": 60.4962, + "step": 7420 + }, + { + "epoch": 119.84552845528455, + "grad_norm": 1.1358702182769775, + "learning_rate": 0.0001, + "loss": 60.7173, + "step": 7430 + }, + { + "epoch": 120.0, + "grad_norm": 1.116322636604309, + "learning_rate": 0.0001, + "loss": 53.8387, + "step": 7440 + }, + { + "epoch": 120.16260162601625, + "grad_norm": 0.8522642254829407, + "learning_rate": 0.0001, + "loss": 57.9095, + "step": 7450 + }, + { + "epoch": 120.32520325203252, + "grad_norm": 0.890731692314148, + "learning_rate": 0.0001, + "loss": 57.2296, + "step": 7460 + }, + { + "epoch": 120.48780487804878, + "grad_norm": 1.6366336345672607, + "learning_rate": 0.0001, + "loss": 58.9417, + "step": 7470 + }, + { + "epoch": 120.65040650406505, + "grad_norm": 0.9774733781814575, + "learning_rate": 0.0001, + "loss": 58.8446, + "step": 7480 + }, + { + "epoch": 120.8130081300813, + "grad_norm": 1.6188803911209106, + "learning_rate": 0.0001, + "loss": 59.9794, + "step": 7490 + }, + { + "epoch": 120.97560975609755, + "grad_norm": 1.146599292755127, + "learning_rate": 0.0001, + "loss": 59.8794, + "step": 7500 + }, + { + "epoch": 121.130081300813, + "grad_norm": 0.9472863078117371, + "learning_rate": 0.0001, + "loss": 55.2454, + "step": 7510 + }, + { + "epoch": 121.29268292682927, + "grad_norm": 0.8604205250740051, + "learning_rate": 0.0001, + "loss": 57.6314, + "step": 7520 + }, + { + "epoch": 121.45528455284553, + "grad_norm": 1.0755794048309326, + "learning_rate": 0.0001, + "loss": 60.909, + "step": 7530 + }, + { + "epoch": 121.6178861788618, + "grad_norm": 0.7967071533203125, + "learning_rate": 0.0001, + "loss": 60.1755, + "step": 7540 + }, + { + "epoch": 121.78048780487805, + "grad_norm": 1.037655234336853, + "learning_rate": 0.0001, + "loss": 59.283, + "step": 7550 + }, + { + "epoch": 121.9430894308943, + "grad_norm": 1.070142388343811, + "learning_rate": 0.0001, + "loss": 57.0941, + "step": 7560 + }, + { + "epoch": 122.09756097560975, + "grad_norm": 1.4201022386550903, + "learning_rate": 0.0001, + "loss": 56.2891, + "step": 7570 + }, + { + "epoch": 122.26016260162602, + "grad_norm": 1.0909277200698853, + "learning_rate": 0.0001, + "loss": 57.5213, + "step": 7580 + }, + { + "epoch": 122.42276422764228, + "grad_norm": 0.7573383450508118, + "learning_rate": 0.0001, + "loss": 59.0847, + "step": 7590 + }, + { + "epoch": 122.58536585365853, + "grad_norm": 0.8931725025177002, + "learning_rate": 0.0001, + "loss": 58.9706, + "step": 7600 + }, + { + "epoch": 122.7479674796748, + "grad_norm": 0.9459935426712036, + "learning_rate": 0.0001, + "loss": 58.6267, + "step": 7610 + }, + { + "epoch": 122.91056910569105, + "grad_norm": 0.9463319778442383, + "learning_rate": 0.0001, + "loss": 60.8536, + "step": 7620 + }, + { + "epoch": 123.0650406504065, + "grad_norm": 0.9286382794380188, + "learning_rate": 0.0001, + "loss": 55.2711, + "step": 7630 + }, + { + "epoch": 123.22764227642277, + "grad_norm": 0.9317494630813599, + "learning_rate": 0.0001, + "loss": 60.1578, + "step": 7640 + }, + { + "epoch": 123.39024390243902, + "grad_norm": 0.9834237098693848, + "learning_rate": 0.0001, + "loss": 58.8121, + "step": 7650 + }, + { + "epoch": 123.55284552845528, + "grad_norm": 0.9992889761924744, + "learning_rate": 0.0001, + "loss": 59.8061, + "step": 7660 + }, + { + "epoch": 123.71544715447155, + "grad_norm": 1.0354453325271606, + "learning_rate": 0.0001, + "loss": 59.5853, + "step": 7670 + }, + { + "epoch": 123.8780487804878, + "grad_norm": 1.0609121322631836, + "learning_rate": 0.0001, + "loss": 59.6556, + "step": 7680 + }, + { + "epoch": 124.03252032520325, + "grad_norm": 0.9742912650108337, + "learning_rate": 0.0001, + "loss": 57.2294, + "step": 7690 + }, + { + "epoch": 124.1951219512195, + "grad_norm": 0.9758710265159607, + "learning_rate": 0.0001, + "loss": 59.8581, + "step": 7700 + }, + { + "epoch": 124.35772357723577, + "grad_norm": 0.9492778182029724, + "learning_rate": 0.0001, + "loss": 60.0624, + "step": 7710 + }, + { + "epoch": 124.52032520325203, + "grad_norm": 0.8311835527420044, + "learning_rate": 0.0001, + "loss": 60.0849, + "step": 7720 + }, + { + "epoch": 124.6829268292683, + "grad_norm": 0.8652944564819336, + "learning_rate": 0.0001, + "loss": 58.2321, + "step": 7730 + }, + { + "epoch": 124.84552845528455, + "grad_norm": 1.2980835437774658, + "learning_rate": 0.0001, + "loss": 57.5315, + "step": 7740 + }, + { + "epoch": 125.0, + "grad_norm": 1.0550237894058228, + "learning_rate": 0.0001, + "loss": 56.9615, + "step": 7750 + }, + { + "epoch": 125.16260162601625, + "grad_norm": 1.1248353719711304, + "learning_rate": 0.0001, + "loss": 58.4853, + "step": 7760 + }, + { + "epoch": 125.32520325203252, + "grad_norm": 1.338152289390564, + "learning_rate": 0.0001, + "loss": 57.8604, + "step": 7770 + }, + { + "epoch": 125.48780487804878, + "grad_norm": 0.8162221908569336, + "learning_rate": 0.0001, + "loss": 56.6396, + "step": 7780 + }, + { + "epoch": 125.65040650406505, + "grad_norm": 1.0431848764419556, + "learning_rate": 0.0001, + "loss": 58.9896, + "step": 7790 + }, + { + "epoch": 125.8130081300813, + "grad_norm": 0.848052978515625, + "learning_rate": 0.0001, + "loss": 57.7459, + "step": 7800 + }, + { + "epoch": 125.97560975609755, + "grad_norm": 0.8441061973571777, + "learning_rate": 0.0001, + "loss": 57.9488, + "step": 7810 + }, + { + "epoch": 126.130081300813, + "grad_norm": 0.774566650390625, + "learning_rate": 0.0001, + "loss": 55.9949, + "step": 7820 + }, + { + "epoch": 126.29268292682927, + "grad_norm": 1.178572416305542, + "learning_rate": 0.0001, + "loss": 58.3554, + "step": 7830 + }, + { + "epoch": 126.45528455284553, + "grad_norm": 0.9363889098167419, + "learning_rate": 0.0001, + "loss": 60.6901, + "step": 7840 + }, + { + "epoch": 126.6178861788618, + "grad_norm": 1.0010961294174194, + "learning_rate": 0.0001, + "loss": 57.0078, + "step": 7850 + }, + { + "epoch": 126.78048780487805, + "grad_norm": 0.9573602080345154, + "learning_rate": 0.0001, + "loss": 58.0567, + "step": 7860 + }, + { + "epoch": 126.9430894308943, + "grad_norm": 1.1903672218322754, + "learning_rate": 0.0001, + "loss": 58.3699, + "step": 7870 + }, + { + "epoch": 127.09756097560975, + "grad_norm": 1.2158106565475464, + "learning_rate": 0.0001, + "loss": 57.1154, + "step": 7880 + }, + { + "epoch": 127.26016260162602, + "grad_norm": 1.198789119720459, + "learning_rate": 0.0001, + "loss": 59.1004, + "step": 7890 + }, + { + "epoch": 127.42276422764228, + "grad_norm": 0.8893532752990723, + "learning_rate": 0.0001, + "loss": 59.6481, + "step": 7900 + }, + { + "epoch": 127.58536585365853, + "grad_norm": 0.9834772348403931, + "learning_rate": 0.0001, + "loss": 57.4394, + "step": 7910 + }, + { + "epoch": 127.7479674796748, + "grad_norm": 1.1957768201828003, + "learning_rate": 0.0001, + "loss": 59.1622, + "step": 7920 + }, + { + "epoch": 127.91056910569105, + "grad_norm": 0.822515070438385, + "learning_rate": 0.0001, + "loss": 58.4967, + "step": 7930 + }, + { + "epoch": 128.0650406504065, + "grad_norm": 1.0374864339828491, + "learning_rate": 0.0001, + "loss": 53.9391, + "step": 7940 + }, + { + "epoch": 128.22764227642276, + "grad_norm": 1.1767654418945312, + "learning_rate": 0.0001, + "loss": 59.4305, + "step": 7950 + }, + { + "epoch": 128.390243902439, + "grad_norm": 1.0048503875732422, + "learning_rate": 0.0001, + "loss": 59.7954, + "step": 7960 + }, + { + "epoch": 128.5528455284553, + "grad_norm": 0.9039503931999207, + "learning_rate": 0.0001, + "loss": 58.6945, + "step": 7970 + }, + { + "epoch": 128.71544715447155, + "grad_norm": 1.047418236732483, + "learning_rate": 0.0001, + "loss": 58.6324, + "step": 7980 + }, + { + "epoch": 128.8780487804878, + "grad_norm": 1.7829573154449463, + "learning_rate": 0.0001, + "loss": 57.3699, + "step": 7990 + }, + { + "epoch": 129.03252032520325, + "grad_norm": 0.8874698281288147, + "learning_rate": 0.0001, + "loss": 57.8226, + "step": 8000 + }, + { + "epoch": 129.1951219512195, + "grad_norm": 0.8585357069969177, + "learning_rate": 0.0001, + "loss": 59.1828, + "step": 8010 + }, + { + "epoch": 129.35772357723576, + "grad_norm": 0.7905164957046509, + "learning_rate": 0.0001, + "loss": 58.4257, + "step": 8020 + }, + { + "epoch": 129.52032520325204, + "grad_norm": 1.15513014793396, + "learning_rate": 0.0001, + "loss": 57.712, + "step": 8030 + }, + { + "epoch": 129.6829268292683, + "grad_norm": 1.4509782791137695, + "learning_rate": 0.0001, + "loss": 59.965, + "step": 8040 + }, + { + "epoch": 129.84552845528455, + "grad_norm": 0.8998057246208191, + "learning_rate": 0.0001, + "loss": 58.6614, + "step": 8050 + }, + { + "epoch": 130.0, + "grad_norm": 1.3381586074829102, + "learning_rate": 0.0001, + "loss": 55.2756, + "step": 8060 + }, + { + "epoch": 130.16260162601625, + "grad_norm": 0.9927449226379395, + "learning_rate": 0.0001, + "loss": 59.6809, + "step": 8070 + }, + { + "epoch": 130.3252032520325, + "grad_norm": 0.9900999665260315, + "learning_rate": 0.0001, + "loss": 60.0017, + "step": 8080 + }, + { + "epoch": 130.4878048780488, + "grad_norm": 0.8317362070083618, + "learning_rate": 0.0001, + "loss": 58.2334, + "step": 8090 + }, + { + "epoch": 130.65040650406505, + "grad_norm": 1.0413036346435547, + "learning_rate": 0.0001, + "loss": 58.987, + "step": 8100 + }, + { + "epoch": 130.8130081300813, + "grad_norm": 1.0915170907974243, + "learning_rate": 0.0001, + "loss": 58.9502, + "step": 8110 + }, + { + "epoch": 130.97560975609755, + "grad_norm": 0.9717564582824707, + "learning_rate": 0.0001, + "loss": 58.6578, + "step": 8120 + }, + { + "epoch": 131.130081300813, + "grad_norm": 0.9847933650016785, + "learning_rate": 0.0001, + "loss": 55.0067, + "step": 8130 + }, + { + "epoch": 131.29268292682926, + "grad_norm": 1.2512580156326294, + "learning_rate": 0.0001, + "loss": 56.0223, + "step": 8140 + }, + { + "epoch": 131.45528455284554, + "grad_norm": 0.8859831094741821, + "learning_rate": 0.0001, + "loss": 57.8123, + "step": 8150 + }, + { + "epoch": 131.6178861788618, + "grad_norm": 1.1545639038085938, + "learning_rate": 0.0001, + "loss": 60.2219, + "step": 8160 + }, + { + "epoch": 131.78048780487805, + "grad_norm": 1.3861064910888672, + "learning_rate": 0.0001, + "loss": 60.2156, + "step": 8170 + }, + { + "epoch": 131.9430894308943, + "grad_norm": 1.2621673345565796, + "learning_rate": 0.0001, + "loss": 58.6422, + "step": 8180 + }, + { + "epoch": 132.09756097560975, + "grad_norm": 1.2075496912002563, + "learning_rate": 0.0001, + "loss": 55.4202, + "step": 8190 + }, + { + "epoch": 132.260162601626, + "grad_norm": 1.1960432529449463, + "learning_rate": 0.0001, + "loss": 59.6405, + "step": 8200 + }, + { + "epoch": 132.4227642276423, + "grad_norm": 1.683369755744934, + "learning_rate": 0.0001, + "loss": 60.1361, + "step": 8210 + }, + { + "epoch": 132.58536585365854, + "grad_norm": 0.9033818244934082, + "learning_rate": 0.0001, + "loss": 58.2383, + "step": 8220 + }, + { + "epoch": 132.7479674796748, + "grad_norm": 1.0529452562332153, + "learning_rate": 0.0001, + "loss": 58.9644, + "step": 8230 + }, + { + "epoch": 132.91056910569105, + "grad_norm": 1.0898828506469727, + "learning_rate": 0.0001, + "loss": 57.0898, + "step": 8240 + }, + { + "epoch": 133.0650406504065, + "grad_norm": 1.0203288793563843, + "learning_rate": 0.0001, + "loss": 56.8206, + "step": 8250 + }, + { + "epoch": 133.22764227642276, + "grad_norm": 0.9973819255828857, + "learning_rate": 0.0001, + "loss": 59.2097, + "step": 8260 + }, + { + "epoch": 133.390243902439, + "grad_norm": 0.9728502035140991, + "learning_rate": 0.0001, + "loss": 57.9959, + "step": 8270 + }, + { + "epoch": 133.5528455284553, + "grad_norm": 0.8645833134651184, + "learning_rate": 0.0001, + "loss": 58.9452, + "step": 8280 + }, + { + "epoch": 133.71544715447155, + "grad_norm": 1.0179742574691772, + "learning_rate": 0.0001, + "loss": 57.8577, + "step": 8290 + }, + { + "epoch": 133.8780487804878, + "grad_norm": 0.8542449474334717, + "learning_rate": 0.0001, + "loss": 57.5798, + "step": 8300 + }, + { + "epoch": 134.03252032520325, + "grad_norm": 0.9958596229553223, + "learning_rate": 0.0001, + "loss": 56.6277, + "step": 8310 + }, + { + "epoch": 134.1951219512195, + "grad_norm": 1.1476788520812988, + "learning_rate": 0.0001, + "loss": 60.8397, + "step": 8320 + }, + { + "epoch": 134.35772357723576, + "grad_norm": 0.9098069071769714, + "learning_rate": 0.0001, + "loss": 59.6647, + "step": 8330 + }, + { + "epoch": 134.52032520325204, + "grad_norm": 0.8126556277275085, + "learning_rate": 0.0001, + "loss": 57.147, + "step": 8340 + }, + { + "epoch": 134.6829268292683, + "grad_norm": 1.4846469163894653, + "learning_rate": 0.0001, + "loss": 58.1245, + "step": 8350 + }, + { + "epoch": 134.84552845528455, + "grad_norm": 1.0117210149765015, + "learning_rate": 0.0001, + "loss": 58.7183, + "step": 8360 + }, + { + "epoch": 135.0, + "grad_norm": 0.9517931938171387, + "learning_rate": 0.0001, + "loss": 57.0423, + "step": 8370 + }, + { + "epoch": 135.16260162601625, + "grad_norm": 1.0269144773483276, + "learning_rate": 0.0001, + "loss": 59.6931, + "step": 8380 + }, + { + "epoch": 135.3252032520325, + "grad_norm": 1.1544915437698364, + "learning_rate": 0.0001, + "loss": 57.4079, + "step": 8390 + }, + { + "epoch": 135.4878048780488, + "grad_norm": 1.1180702447891235, + "learning_rate": 0.0001, + "loss": 57.855, + "step": 8400 + }, + { + "epoch": 135.65040650406505, + "grad_norm": 1.2864829301834106, + "learning_rate": 0.0001, + "loss": 57.677, + "step": 8410 + }, + { + "epoch": 135.8130081300813, + "grad_norm": 1.1786772012710571, + "learning_rate": 0.0001, + "loss": 59.5238, + "step": 8420 + }, + { + "epoch": 135.97560975609755, + "grad_norm": 1.8596229553222656, + "learning_rate": 0.0001, + "loss": 57.1596, + "step": 8430 + }, + { + "epoch": 136.130081300813, + "grad_norm": 0.8087555170059204, + "learning_rate": 0.0001, + "loss": 55.8342, + "step": 8440 + }, + { + "epoch": 136.29268292682926, + "grad_norm": 0.8107356429100037, + "learning_rate": 0.0001, + "loss": 58.5965, + "step": 8450 + }, + { + "epoch": 136.45528455284554, + "grad_norm": 1.2416011095046997, + "learning_rate": 0.0001, + "loss": 57.5909, + "step": 8460 + }, + { + "epoch": 136.6178861788618, + "grad_norm": 0.9763500690460205, + "learning_rate": 0.0001, + "loss": 58.1822, + "step": 8470 + }, + { + "epoch": 136.78048780487805, + "grad_norm": 1.1418349742889404, + "learning_rate": 0.0001, + "loss": 59.3942, + "step": 8480 + }, + { + "epoch": 136.9430894308943, + "grad_norm": 1.206878900527954, + "learning_rate": 0.0001, + "loss": 58.4128, + "step": 8490 + }, + { + "epoch": 137.09756097560975, + "grad_norm": 0.861142098903656, + "learning_rate": 0.0001, + "loss": 55.8831, + "step": 8500 + }, + { + "epoch": 137.260162601626, + "grad_norm": 1.0507313013076782, + "learning_rate": 0.0001, + "loss": 59.755, + "step": 8510 + }, + { + "epoch": 137.4227642276423, + "grad_norm": 0.7070153951644897, + "learning_rate": 0.0001, + "loss": 55.6448, + "step": 8520 + }, + { + "epoch": 137.58536585365854, + "grad_norm": 0.9338926672935486, + "learning_rate": 0.0001, + "loss": 58.4242, + "step": 8530 + }, + { + "epoch": 137.7479674796748, + "grad_norm": 1.0600488185882568, + "learning_rate": 0.0001, + "loss": 59.4947, + "step": 8540 + }, + { + "epoch": 137.91056910569105, + "grad_norm": 1.069604754447937, + "learning_rate": 0.0001, + "loss": 57.5164, + "step": 8550 + }, + { + "epoch": 138.0650406504065, + "grad_norm": 0.872332751750946, + "learning_rate": 0.0001, + "loss": 56.453, + "step": 8560 + }, + { + "epoch": 138.22764227642276, + "grad_norm": 0.9670246243476868, + "learning_rate": 0.0001, + "loss": 59.6467, + "step": 8570 + }, + { + "epoch": 138.390243902439, + "grad_norm": 1.1045281887054443, + "learning_rate": 0.0001, + "loss": 59.3334, + "step": 8580 + }, + { + "epoch": 138.5528455284553, + "grad_norm": 1.0553908348083496, + "learning_rate": 0.0001, + "loss": 59.9081, + "step": 8590 + }, + { + "epoch": 138.71544715447155, + "grad_norm": 0.7342197895050049, + "learning_rate": 0.0001, + "loss": 60.2319, + "step": 8600 + }, + { + "epoch": 138.8780487804878, + "grad_norm": 0.9574354290962219, + "learning_rate": 0.0001, + "loss": 56.7984, + "step": 8610 + }, + { + "epoch": 139.03252032520325, + "grad_norm": 0.9238156676292419, + "learning_rate": 0.0001, + "loss": 56.0007, + "step": 8620 + }, + { + "epoch": 139.1951219512195, + "grad_norm": 1.0704458951950073, + "learning_rate": 0.0001, + "loss": 59.4573, + "step": 8630 + }, + { + "epoch": 139.35772357723576, + "grad_norm": 0.8392401337623596, + "learning_rate": 0.0001, + "loss": 57.7589, + "step": 8640 + }, + { + "epoch": 139.52032520325204, + "grad_norm": 0.9042978286743164, + "learning_rate": 0.0001, + "loss": 59.0548, + "step": 8650 + }, + { + "epoch": 139.6829268292683, + "grad_norm": 2.024853467941284, + "learning_rate": 0.0001, + "loss": 58.0019, + "step": 8660 + }, + { + "epoch": 139.84552845528455, + "grad_norm": 1.9823323488235474, + "learning_rate": 0.0001, + "loss": 57.4398, + "step": 8670 + }, + { + "epoch": 140.0, + "grad_norm": 1.626451849937439, + "learning_rate": 0.0001, + "loss": 54.969, + "step": 8680 + }, + { + "epoch": 140.16260162601625, + "grad_norm": 2.435973644256592, + "learning_rate": 0.0001, + "loss": 57.8945, + "step": 8690 + }, + { + "epoch": 140.3252032520325, + "grad_norm": 1.0417332649230957, + "learning_rate": 0.0001, + "loss": 59.9172, + "step": 8700 + }, + { + "epoch": 140.4878048780488, + "grad_norm": 1.0246574878692627, + "learning_rate": 0.0001, + "loss": 57.7337, + "step": 8710 + }, + { + "epoch": 140.65040650406505, + "grad_norm": 0.9492414593696594, + "learning_rate": 0.0001, + "loss": 57.6049, + "step": 8720 + }, + { + "epoch": 140.8130081300813, + "grad_norm": 1.054689645767212, + "learning_rate": 0.0001, + "loss": 56.4394, + "step": 8730 + }, + { + "epoch": 140.97560975609755, + "grad_norm": 0.7142108082771301, + "learning_rate": 0.0001, + "loss": 58.0006, + "step": 8740 + }, + { + "epoch": 141.130081300813, + "grad_norm": 1.1387385129928589, + "learning_rate": 0.0001, + "loss": 55.2639, + "step": 8750 + }, + { + "epoch": 141.29268292682926, + "grad_norm": 2.105823040008545, + "learning_rate": 0.0001, + "loss": 59.3938, + "step": 8760 + }, + { + "epoch": 141.45528455284554, + "grad_norm": 0.9188179969787598, + "learning_rate": 0.0001, + "loss": 58.731, + "step": 8770 + }, + { + "epoch": 141.6178861788618, + "grad_norm": 1.043778896331787, + "learning_rate": 0.0001, + "loss": 56.2486, + "step": 8780 + }, + { + "epoch": 141.78048780487805, + "grad_norm": 1.438591718673706, + "learning_rate": 0.0001, + "loss": 58.4228, + "step": 8790 + }, + { + "epoch": 141.9430894308943, + "grad_norm": 1.116957187652588, + "learning_rate": 0.0001, + "loss": 59.2317, + "step": 8800 + }, + { + "epoch": 142.09756097560975, + "grad_norm": 0.8644933104515076, + "learning_rate": 0.0001, + "loss": 55.3905, + "step": 8810 + }, + { + "epoch": 142.260162601626, + "grad_norm": 1.0249485969543457, + "learning_rate": 0.0001, + "loss": 59.0372, + "step": 8820 + }, + { + "epoch": 142.4227642276423, + "grad_norm": 0.9939082264900208, + "learning_rate": 0.0001, + "loss": 57.3947, + "step": 8830 + }, + { + "epoch": 142.58536585365854, + "grad_norm": 0.9836716055870056, + "learning_rate": 0.0001, + "loss": 57.9229, + "step": 8840 + }, + { + "epoch": 142.7479674796748, + "grad_norm": 0.9596876502037048, + "learning_rate": 0.0001, + "loss": 57.0285, + "step": 8850 + }, + { + "epoch": 142.91056910569105, + "grad_norm": 1.052899956703186, + "learning_rate": 0.0001, + "loss": 60.1384, + "step": 8860 + }, + { + "epoch": 143.0650406504065, + "grad_norm": 1.1220297813415527, + "learning_rate": 0.0001, + "loss": 55.3928, + "step": 8870 + }, + { + "epoch": 143.22764227642276, + "grad_norm": 0.8395596742630005, + "learning_rate": 0.0001, + "loss": 57.8185, + "step": 8880 + }, + { + "epoch": 143.390243902439, + "grad_norm": 1.1940062046051025, + "learning_rate": 0.0001, + "loss": 58.0322, + "step": 8890 + }, + { + "epoch": 143.5528455284553, + "grad_norm": 1.1662484407424927, + "learning_rate": 0.0001, + "loss": 58.2746, + "step": 8900 + }, + { + "epoch": 143.71544715447155, + "grad_norm": 0.9028235077857971, + "learning_rate": 0.0001, + "loss": 56.5817, + "step": 8910 + }, + { + "epoch": 143.8780487804878, + "grad_norm": 1.0597296953201294, + "learning_rate": 0.0001, + "loss": 56.8426, + "step": 8920 + }, + { + "epoch": 144.03252032520325, + "grad_norm": 0.9968701004981995, + "learning_rate": 0.0001, + "loss": 54.9727, + "step": 8930 + }, + { + "epoch": 144.1951219512195, + "grad_norm": 0.9306491613388062, + "learning_rate": 0.0001, + "loss": 56.5321, + "step": 8940 + }, + { + "epoch": 144.35772357723576, + "grad_norm": 1.1079609394073486, + "learning_rate": 0.0001, + "loss": 56.9714, + "step": 8950 + }, + { + "epoch": 144.52032520325204, + "grad_norm": 0.9644935131072998, + "learning_rate": 0.0001, + "loss": 58.613, + "step": 8960 + }, + { + "epoch": 144.6829268292683, + "grad_norm": 1.2037341594696045, + "learning_rate": 0.0001, + "loss": 56.7768, + "step": 8970 + }, + { + "epoch": 144.84552845528455, + "grad_norm": 0.7347294092178345, + "learning_rate": 0.0001, + "loss": 58.5653, + "step": 8980 + }, + { + "epoch": 145.0, + "grad_norm": 0.8877835273742676, + "learning_rate": 0.0001, + "loss": 53.7768, + "step": 8990 + }, + { + "epoch": 145.16260162601625, + "grad_norm": 0.9649038910865784, + "learning_rate": 0.0001, + "loss": 57.7391, + "step": 9000 + }, + { + "epoch": 145.3252032520325, + "grad_norm": 1.1804879903793335, + "learning_rate": 0.0001, + "loss": 60.3727, + "step": 9010 + }, + { + "epoch": 145.4878048780488, + "grad_norm": 0.8406301140785217, + "learning_rate": 0.0001, + "loss": 57.9417, + "step": 9020 + }, + { + "epoch": 145.65040650406505, + "grad_norm": 0.9555912613868713, + "learning_rate": 0.0001, + "loss": 58.0936, + "step": 9030 + }, + { + "epoch": 145.8130081300813, + "grad_norm": 1.20796537399292, + "learning_rate": 0.0001, + "loss": 57.0784, + "step": 9040 + }, + { + "epoch": 145.97560975609755, + "grad_norm": 1.0527915954589844, + "learning_rate": 0.0001, + "loss": 56.752, + "step": 9050 + }, + { + "epoch": 146.130081300813, + "grad_norm": 1.0104299783706665, + "learning_rate": 0.0001, + "loss": 56.2158, + "step": 9060 + }, + { + "epoch": 146.29268292682926, + "grad_norm": 1.4884357452392578, + "learning_rate": 0.0001, + "loss": 57.8631, + "step": 9070 + }, + { + "epoch": 146.45528455284554, + "grad_norm": 0.9095257520675659, + "learning_rate": 0.0001, + "loss": 57.6492, + "step": 9080 + }, + { + "epoch": 146.6178861788618, + "grad_norm": 0.8465835452079773, + "learning_rate": 0.0001, + "loss": 57.78, + "step": 9090 + }, + { + "epoch": 146.78048780487805, + "grad_norm": 1.0254603624343872, + "learning_rate": 0.0001, + "loss": 57.7678, + "step": 9100 + }, + { + "epoch": 146.9430894308943, + "grad_norm": 1.0137311220169067, + "learning_rate": 0.0001, + "loss": 58.2813, + "step": 9110 + }, + { + "epoch": 147.09756097560975, + "grad_norm": 0.9816886186599731, + "learning_rate": 0.0001, + "loss": 54.4789, + "step": 9120 + }, + { + "epoch": 147.260162601626, + "grad_norm": 0.9038514494895935, + "learning_rate": 0.0001, + "loss": 58.5366, + "step": 9130 + }, + { + "epoch": 147.4227642276423, + "grad_norm": 0.7649261355400085, + "learning_rate": 0.0001, + "loss": 57.2898, + "step": 9140 + }, + { + "epoch": 147.58536585365854, + "grad_norm": 0.8841894865036011, + "learning_rate": 0.0001, + "loss": 57.9109, + "step": 9150 + }, + { + "epoch": 147.7479674796748, + "grad_norm": 1.0480542182922363, + "learning_rate": 0.0001, + "loss": 56.9625, + "step": 9160 + }, + { + "epoch": 147.91056910569105, + "grad_norm": 0.8710455894470215, + "learning_rate": 0.0001, + "loss": 57.1721, + "step": 9170 + }, + { + "epoch": 148.0650406504065, + "grad_norm": 1.374118447303772, + "learning_rate": 0.0001, + "loss": 55.3243, + "step": 9180 + }, + { + "epoch": 148.22764227642276, + "grad_norm": 1.0879178047180176, + "learning_rate": 0.0001, + "loss": 57.1631, + "step": 9190 + }, + { + "epoch": 148.390243902439, + "grad_norm": 0.8242596387863159, + "learning_rate": 0.0001, + "loss": 56.9993, + "step": 9200 + }, + { + "epoch": 148.5528455284553, + "grad_norm": 1.02505362033844, + "learning_rate": 0.0001, + "loss": 57.9201, + "step": 9210 + }, + { + "epoch": 148.71544715447155, + "grad_norm": 0.7904821038246155, + "learning_rate": 0.0001, + "loss": 59.2553, + "step": 9220 + }, + { + "epoch": 148.8780487804878, + "grad_norm": 1.4904934167861938, + "learning_rate": 0.0001, + "loss": 59.498, + "step": 9230 + }, + { + "epoch": 149.03252032520325, + "grad_norm": 1.4203144311904907, + "learning_rate": 0.0001, + "loss": 54.6357, + "step": 9240 + }, + { + "epoch": 149.1951219512195, + "grad_norm": 0.8256629705429077, + "learning_rate": 0.0001, + "loss": 57.5996, + "step": 9250 + }, + { + "epoch": 149.35772357723576, + "grad_norm": 0.8736788630485535, + "learning_rate": 0.0001, + "loss": 56.8112, + "step": 9260 + }, + { + "epoch": 149.52032520325204, + "grad_norm": 3.20312762260437, + "learning_rate": 0.0001, + "loss": 55.8763, + "step": 9270 + }, + { + "epoch": 149.6829268292683, + "grad_norm": 0.7431173920631409, + "learning_rate": 0.0001, + "loss": 56.7385, + "step": 9280 + }, + { + "epoch": 149.84552845528455, + "grad_norm": 0.8165793418884277, + "learning_rate": 0.0001, + "loss": 57.8441, + "step": 9290 + }, + { + "epoch": 150.0, + "grad_norm": 1.2387259006500244, + "learning_rate": 0.0001, + "loss": 53.47, + "step": 9300 + }, + { + "epoch": 150.16260162601625, + "grad_norm": 0.8952508568763733, + "learning_rate": 0.0001, + "loss": 57.7642, + "step": 9310 + }, + { + "epoch": 150.3252032520325, + "grad_norm": 0.9837163686752319, + "learning_rate": 0.0001, + "loss": 57.2288, + "step": 9320 + }, + { + "epoch": 150.4878048780488, + "grad_norm": 0.9019481539726257, + "learning_rate": 0.0001, + "loss": 57.4239, + "step": 9330 + }, + { + "epoch": 150.65040650406505, + "grad_norm": 1.0404818058013916, + "learning_rate": 0.0001, + "loss": 57.7678, + "step": 9340 + }, + { + "epoch": 150.8130081300813, + "grad_norm": 1.0074576139450073, + "learning_rate": 0.0001, + "loss": 58.1508, + "step": 9350 + }, + { + "epoch": 150.97560975609755, + "grad_norm": 0.9951609969139099, + "learning_rate": 0.0001, + "loss": 56.1586, + "step": 9360 + }, + { + "epoch": 151.130081300813, + "grad_norm": 1.0799294710159302, + "learning_rate": 0.0001, + "loss": 52.2597, + "step": 9370 + }, + { + "epoch": 151.29268292682926, + "grad_norm": 0.9001291394233704, + "learning_rate": 0.0001, + "loss": 58.1814, + "step": 9380 + }, + { + "epoch": 151.45528455284554, + "grad_norm": 1.0388239622116089, + "learning_rate": 0.0001, + "loss": 57.1812, + "step": 9390 + }, + { + "epoch": 151.6178861788618, + "grad_norm": 0.7662557363510132, + "learning_rate": 0.0001, + "loss": 56.087, + "step": 9400 + }, + { + "epoch": 151.78048780487805, + "grad_norm": 1.0790860652923584, + "learning_rate": 0.0001, + "loss": 55.9247, + "step": 9410 + }, + { + "epoch": 151.9430894308943, + "grad_norm": 0.8173624277114868, + "learning_rate": 0.0001, + "loss": 57.5506, + "step": 9420 + }, + { + "epoch": 152.09756097560975, + "grad_norm": 1.0954891443252563, + "learning_rate": 0.0001, + "loss": 54.6799, + "step": 9430 + }, + { + "epoch": 152.260162601626, + "grad_norm": 1.0103733539581299, + "learning_rate": 0.0001, + "loss": 58.5454, + "step": 9440 + }, + { + "epoch": 152.4227642276423, + "grad_norm": 0.6966420412063599, + "learning_rate": 0.0001, + "loss": 57.561, + "step": 9450 + }, + { + "epoch": 152.58536585365854, + "grad_norm": 0.9085037708282471, + "learning_rate": 0.0001, + "loss": 57.1746, + "step": 9460 + }, + { + "epoch": 152.7479674796748, + "grad_norm": 1.0009453296661377, + "learning_rate": 0.0001, + "loss": 56.8, + "step": 9470 + }, + { + "epoch": 152.91056910569105, + "grad_norm": 0.9139633178710938, + "learning_rate": 0.0001, + "loss": 56.8229, + "step": 9480 + }, + { + "epoch": 153.0650406504065, + "grad_norm": 1.4209275245666504, + "learning_rate": 0.0001, + "loss": 55.3074, + "step": 9490 + }, + { + "epoch": 153.22764227642276, + "grad_norm": 1.3912067413330078, + "learning_rate": 0.0001, + "loss": 58.1401, + "step": 9500 + }, + { + "epoch": 153.390243902439, + "grad_norm": 0.8208648562431335, + "learning_rate": 0.0001, + "loss": 55.2981, + "step": 9510 + }, + { + "epoch": 153.5528455284553, + "grad_norm": 1.1563907861709595, + "learning_rate": 0.0001, + "loss": 59.0262, + "step": 9520 + }, + { + "epoch": 153.71544715447155, + "grad_norm": 1.1142042875289917, + "learning_rate": 0.0001, + "loss": 55.5979, + "step": 9530 + }, + { + "epoch": 153.8780487804878, + "grad_norm": 0.8886147141456604, + "learning_rate": 0.0001, + "loss": 58.5592, + "step": 9540 + }, + { + "epoch": 154.03252032520325, + "grad_norm": 0.9479674696922302, + "learning_rate": 0.0001, + "loss": 53.956, + "step": 9550 + }, + { + "epoch": 154.1951219512195, + "grad_norm": 1.0572675466537476, + "learning_rate": 0.0001, + "loss": 59.5914, + "step": 9560 + }, + { + "epoch": 154.35772357723576, + "grad_norm": 0.8284531831741333, + "learning_rate": 0.0001, + "loss": 56.4313, + "step": 9570 + }, + { + "epoch": 154.52032520325204, + "grad_norm": 0.8882488012313843, + "learning_rate": 0.0001, + "loss": 57.707, + "step": 9580 + }, + { + "epoch": 154.6829268292683, + "grad_norm": 0.9115352630615234, + "learning_rate": 0.0001, + "loss": 58.0544, + "step": 9590 + }, + { + "epoch": 154.84552845528455, + "grad_norm": 1.0006799697875977, + "learning_rate": 0.0001, + "loss": 59.4528, + "step": 9600 + }, + { + "epoch": 155.0, + "grad_norm": 0.9925333261489868, + "learning_rate": 0.0001, + "loss": 52.977, + "step": 9610 + }, + { + "epoch": 155.16260162601625, + "grad_norm": 1.423391580581665, + "learning_rate": 0.0001, + "loss": 57.0083, + "step": 9620 + }, + { + "epoch": 155.3252032520325, + "grad_norm": 1.0822738409042358, + "learning_rate": 0.0001, + "loss": 58.187, + "step": 9630 + }, + { + "epoch": 155.4878048780488, + "grad_norm": 0.932689905166626, + "learning_rate": 0.0001, + "loss": 55.6032, + "step": 9640 + }, + { + "epoch": 155.65040650406505, + "grad_norm": 0.895504355430603, + "learning_rate": 0.0001, + "loss": 57.1324, + "step": 9650 + }, + { + "epoch": 155.8130081300813, + "grad_norm": 1.0883748531341553, + "learning_rate": 0.0001, + "loss": 58.8422, + "step": 9660 + }, + { + "epoch": 155.97560975609755, + "grad_norm": 1.0161819458007812, + "learning_rate": 0.0001, + "loss": 56.8333, + "step": 9670 + }, + { + "epoch": 156.130081300813, + "grad_norm": 0.9483168721199036, + "learning_rate": 0.0001, + "loss": 52.8434, + "step": 9680 + }, + { + "epoch": 156.29268292682926, + "grad_norm": 1.0677295923233032, + "learning_rate": 0.0001, + "loss": 57.749, + "step": 9690 + }, + { + "epoch": 156.45528455284554, + "grad_norm": 0.8123661875724792, + "learning_rate": 0.0001, + "loss": 56.0896, + "step": 9700 + }, + { + "epoch": 156.6178861788618, + "grad_norm": 0.7926080822944641, + "learning_rate": 0.0001, + "loss": 57.8899, + "step": 9710 + }, + { + "epoch": 156.78048780487805, + "grad_norm": 1.0433725118637085, + "learning_rate": 0.0001, + "loss": 59.6728, + "step": 9720 + }, + { + "epoch": 156.9430894308943, + "grad_norm": 0.8284909129142761, + "learning_rate": 0.0001, + "loss": 55.6093, + "step": 9730 + }, + { + "epoch": 157.09756097560975, + "grad_norm": 0.9702232480049133, + "learning_rate": 0.0001, + "loss": 55.3595, + "step": 9740 + }, + { + "epoch": 157.260162601626, + "grad_norm": 0.8909348249435425, + "learning_rate": 0.0001, + "loss": 57.6344, + "step": 9750 + }, + { + "epoch": 157.4227642276423, + "grad_norm": 0.9539676308631897, + "learning_rate": 0.0001, + "loss": 55.8826, + "step": 9760 + }, + { + "epoch": 157.58536585365854, + "grad_norm": 1.1801178455352783, + "learning_rate": 0.0001, + "loss": 55.7141, + "step": 9770 + }, + { + "epoch": 157.7479674796748, + "grad_norm": 0.8674126863479614, + "learning_rate": 0.0001, + "loss": 56.0571, + "step": 9780 + }, + { + "epoch": 157.91056910569105, + "grad_norm": 1.0690264701843262, + "learning_rate": 0.0001, + "loss": 56.1485, + "step": 9790 + }, + { + "epoch": 158.0650406504065, + "grad_norm": 0.8807869553565979, + "learning_rate": 0.0001, + "loss": 56.9874, + "step": 9800 + }, + { + "epoch": 158.22764227642276, + "grad_norm": 0.9477714896202087, + "learning_rate": 0.0001, + "loss": 58.2035, + "step": 9810 + }, + { + "epoch": 158.390243902439, + "grad_norm": 1.4875513315200806, + "learning_rate": 0.0001, + "loss": 59.0302, + "step": 9820 + }, + { + "epoch": 158.5528455284553, + "grad_norm": 0.7979354858398438, + "learning_rate": 0.0001, + "loss": 57.2801, + "step": 9830 + }, + { + "epoch": 158.71544715447155, + "grad_norm": 0.8941402435302734, + "learning_rate": 0.0001, + "loss": 56.3615, + "step": 9840 + }, + { + "epoch": 158.8780487804878, + "grad_norm": 0.8475911021232605, + "learning_rate": 0.0001, + "loss": 57.266, + "step": 9850 + }, + { + "epoch": 159.03252032520325, + "grad_norm": 0.8971277475357056, + "learning_rate": 0.0001, + "loss": 52.792, + "step": 9860 + }, + { + "epoch": 159.1951219512195, + "grad_norm": 0.930777907371521, + "learning_rate": 0.0001, + "loss": 57.4098, + "step": 9870 + }, + { + "epoch": 159.35772357723576, + "grad_norm": 0.9232887625694275, + "learning_rate": 0.0001, + "loss": 56.0267, + "step": 9880 + }, + { + "epoch": 159.52032520325204, + "grad_norm": 0.8904425501823425, + "learning_rate": 0.0001, + "loss": 57.6255, + "step": 9890 + }, + { + "epoch": 159.6829268292683, + "grad_norm": 0.9250755906105042, + "learning_rate": 0.0001, + "loss": 56.5248, + "step": 9900 + }, + { + "epoch": 159.84552845528455, + "grad_norm": 1.3746178150177002, + "learning_rate": 0.0001, + "loss": 56.7843, + "step": 9910 + }, + { + "epoch": 160.0, + "grad_norm": 0.9464873671531677, + "learning_rate": 0.0001, + "loss": 54.1325, + "step": 9920 + }, + { + "epoch": 160.16260162601625, + "grad_norm": 0.9019768238067627, + "learning_rate": 0.0001, + "loss": 58.3641, + "step": 9930 + }, + { + "epoch": 160.3252032520325, + "grad_norm": 0.9282474517822266, + "learning_rate": 0.0001, + "loss": 57.9474, + "step": 9940 + }, + { + "epoch": 160.4878048780488, + "grad_norm": 1.1150450706481934, + "learning_rate": 0.0001, + "loss": 58.2999, + "step": 9950 + }, + { + "epoch": 160.65040650406505, + "grad_norm": 0.9780807495117188, + "learning_rate": 0.0001, + "loss": 55.6409, + "step": 9960 + }, + { + "epoch": 160.8130081300813, + "grad_norm": 0.9708426594734192, + "learning_rate": 0.0001, + "loss": 58.651, + "step": 9970 + }, + { + "epoch": 160.97560975609755, + "grad_norm": 0.9891226887702942, + "learning_rate": 0.0001, + "loss": 57.3798, + "step": 9980 + }, + { + "epoch": 161.130081300813, + "grad_norm": 0.8949639797210693, + "learning_rate": 0.0001, + "loss": 53.4708, + "step": 9990 + }, + { + "epoch": 161.29268292682926, + "grad_norm": 0.831688404083252, + "learning_rate": 0.0001, + "loss": 57.1608, + "step": 10000 + }, + { + "epoch": 161.45528455284554, + "grad_norm": 1.1002815961837769, + "learning_rate": 0.0001, + "loss": 57.5943, + "step": 10010 + }, + { + "epoch": 161.6178861788618, + "grad_norm": 0.823810338973999, + "learning_rate": 0.0001, + "loss": 56.3465, + "step": 10020 + }, + { + "epoch": 161.78048780487805, + "grad_norm": 0.8624166250228882, + "learning_rate": 0.0001, + "loss": 58.2043, + "step": 10030 + }, + { + "epoch": 161.9430894308943, + "grad_norm": 0.9643419981002808, + "learning_rate": 0.0001, + "loss": 58.6286, + "step": 10040 + }, + { + "epoch": 162.09756097560975, + "grad_norm": 0.8477599024772644, + "learning_rate": 0.0001, + "loss": 54.0391, + "step": 10050 + }, + { + "epoch": 162.260162601626, + "grad_norm": 0.932232677936554, + "learning_rate": 0.0001, + "loss": 57.3955, + "step": 10060 + }, + { + "epoch": 162.4227642276423, + "grad_norm": 0.9273334741592407, + "learning_rate": 0.0001, + "loss": 57.3421, + "step": 10070 + }, + { + "epoch": 162.58536585365854, + "grad_norm": 0.746417224407196, + "learning_rate": 0.0001, + "loss": 56.1001, + "step": 10080 + }, + { + "epoch": 162.7479674796748, + "grad_norm": 0.9870802760124207, + "learning_rate": 0.0001, + "loss": 57.1234, + "step": 10090 + }, + { + "epoch": 162.91056910569105, + "grad_norm": 1.2378058433532715, + "learning_rate": 0.0001, + "loss": 58.035, + "step": 10100 + }, + { + "epoch": 163.0650406504065, + "grad_norm": 0.8810584545135498, + "learning_rate": 0.0001, + "loss": 52.1406, + "step": 10110 + }, + { + "epoch": 163.22764227642276, + "grad_norm": 1.1010875701904297, + "learning_rate": 0.0001, + "loss": 57.2534, + "step": 10120 + }, + { + "epoch": 163.390243902439, + "grad_norm": 1.1182267665863037, + "learning_rate": 0.0001, + "loss": 58.0887, + "step": 10130 + }, + { + "epoch": 163.5528455284553, + "grad_norm": 0.8677687048912048, + "learning_rate": 0.0001, + "loss": 56.3502, + "step": 10140 + }, + { + "epoch": 163.71544715447155, + "grad_norm": 1.3890044689178467, + "learning_rate": 0.0001, + "loss": 57.1057, + "step": 10150 + }, + { + "epoch": 163.8780487804878, + "grad_norm": 1.340419054031372, + "learning_rate": 0.0001, + "loss": 56.9196, + "step": 10160 + }, + { + "epoch": 164.03252032520325, + "grad_norm": 0.8187946081161499, + "learning_rate": 0.0001, + "loss": 53.388, + "step": 10170 + }, + { + "epoch": 164.1951219512195, + "grad_norm": 0.7856727838516235, + "learning_rate": 0.0001, + "loss": 57.0134, + "step": 10180 + }, + { + "epoch": 164.35772357723576, + "grad_norm": 0.8677143454551697, + "learning_rate": 0.0001, + "loss": 56.1385, + "step": 10190 + }, + { + "epoch": 164.52032520325204, + "grad_norm": 0.8334061503410339, + "learning_rate": 0.0001, + "loss": 58.4353, + "step": 10200 + }, + { + "epoch": 164.6829268292683, + "grad_norm": 0.9447886943817139, + "learning_rate": 0.0001, + "loss": 56.2007, + "step": 10210 + }, + { + "epoch": 164.84552845528455, + "grad_norm": 0.7787997722625732, + "learning_rate": 0.0001, + "loss": 56.7128, + "step": 10220 + }, + { + "epoch": 165.0, + "grad_norm": 0.864169716835022, + "learning_rate": 0.0001, + "loss": 53.6559, + "step": 10230 + }, + { + "epoch": 165.16260162601625, + "grad_norm": 0.9252000451087952, + "learning_rate": 0.0001, + "loss": 57.1013, + "step": 10240 + }, + { + "epoch": 165.3252032520325, + "grad_norm": 0.9552235007286072, + "learning_rate": 0.0001, + "loss": 58.1834, + "step": 10250 + }, + { + "epoch": 165.4878048780488, + "grad_norm": 1.0003581047058105, + "learning_rate": 0.0001, + "loss": 56.3756, + "step": 10260 + }, + { + "epoch": 165.65040650406505, + "grad_norm": 1.1489440202713013, + "learning_rate": 0.0001, + "loss": 54.8963, + "step": 10270 + }, + { + "epoch": 165.8130081300813, + "grad_norm": 0.9719933867454529, + "learning_rate": 0.0001, + "loss": 59.2853, + "step": 10280 + }, + { + "epoch": 165.97560975609755, + "grad_norm": 0.888495147228241, + "learning_rate": 0.0001, + "loss": 56.7793, + "step": 10290 + }, + { + "epoch": 166.130081300813, + "grad_norm": 0.793580174446106, + "learning_rate": 0.0001, + "loss": 53.6792, + "step": 10300 + }, + { + "epoch": 166.29268292682926, + "grad_norm": 0.928779125213623, + "learning_rate": 0.0001, + "loss": 57.5329, + "step": 10310 + }, + { + "epoch": 166.45528455284554, + "grad_norm": 1.2024686336517334, + "learning_rate": 0.0001, + "loss": 58.08, + "step": 10320 + }, + { + "epoch": 166.6178861788618, + "grad_norm": 1.421026349067688, + "learning_rate": 0.0001, + "loss": 57.5318, + "step": 10330 + }, + { + "epoch": 166.78048780487805, + "grad_norm": 0.9189603328704834, + "learning_rate": 0.0001, + "loss": 57.1355, + "step": 10340 + }, + { + "epoch": 166.9430894308943, + "grad_norm": 1.0614054203033447, + "learning_rate": 0.0001, + "loss": 56.0604, + "step": 10350 + }, + { + "epoch": 167.09756097560975, + "grad_norm": 1.0266008377075195, + "learning_rate": 0.0001, + "loss": 55.0443, + "step": 10360 + }, + { + "epoch": 167.260162601626, + "grad_norm": 0.8291250467300415, + "learning_rate": 0.0001, + "loss": 57.6836, + "step": 10370 + }, + { + "epoch": 167.4227642276423, + "grad_norm": 1.0415455102920532, + "learning_rate": 0.0001, + "loss": 56.1081, + "step": 10380 + }, + { + "epoch": 167.58536585365854, + "grad_norm": 0.8045644164085388, + "learning_rate": 0.0001, + "loss": 54.8943, + "step": 10390 + }, + { + "epoch": 167.7479674796748, + "grad_norm": 0.9025611281394958, + "learning_rate": 0.0001, + "loss": 56.5143, + "step": 10400 + }, + { + "epoch": 167.91056910569105, + "grad_norm": 0.93849116563797, + "learning_rate": 0.0001, + "loss": 57.4787, + "step": 10410 + }, + { + "epoch": 168.0650406504065, + "grad_norm": 0.8948699235916138, + "learning_rate": 0.0001, + "loss": 54.9249, + "step": 10420 + }, + { + "epoch": 168.22764227642276, + "grad_norm": 0.7900148630142212, + "learning_rate": 0.0001, + "loss": 55.9813, + "step": 10430 + }, + { + "epoch": 168.390243902439, + "grad_norm": 1.1853744983673096, + "learning_rate": 0.0001, + "loss": 54.9089, + "step": 10440 + }, + { + "epoch": 168.5528455284553, + "grad_norm": 1.225123643875122, + "learning_rate": 0.0001, + "loss": 58.238, + "step": 10450 + }, + { + "epoch": 168.71544715447155, + "grad_norm": 0.8648725748062134, + "learning_rate": 0.0001, + "loss": 55.4905, + "step": 10460 + }, + { + "epoch": 168.8780487804878, + "grad_norm": 0.9054272770881653, + "learning_rate": 0.0001, + "loss": 58.2697, + "step": 10470 + }, + { + "epoch": 169.03252032520325, + "grad_norm": 0.946158230304718, + "learning_rate": 0.0001, + "loss": 53.3644, + "step": 10480 + }, + { + "epoch": 169.1951219512195, + "grad_norm": 0.8751044273376465, + "learning_rate": 0.0001, + "loss": 54.3469, + "step": 10490 + }, + { + "epoch": 169.35772357723576, + "grad_norm": 0.9814269542694092, + "learning_rate": 0.0001, + "loss": 56.5358, + "step": 10500 + }, + { + "epoch": 169.52032520325204, + "grad_norm": 0.8131198883056641, + "learning_rate": 0.0001, + "loss": 56.2025, + "step": 10510 + }, + { + "epoch": 169.6829268292683, + "grad_norm": 0.7442125678062439, + "learning_rate": 0.0001, + "loss": 54.7679, + "step": 10520 + }, + { + "epoch": 169.84552845528455, + "grad_norm": 0.8504229187965393, + "learning_rate": 0.0001, + "loss": 59.024, + "step": 10530 + }, + { + "epoch": 170.0, + "grad_norm": 0.8483893871307373, + "learning_rate": 0.0001, + "loss": 53.8021, + "step": 10540 + }, + { + "epoch": 170.16260162601625, + "grad_norm": 0.8889706134796143, + "learning_rate": 0.0001, + "loss": 55.1988, + "step": 10550 + }, + { + "epoch": 170.3252032520325, + "grad_norm": 0.9083985686302185, + "learning_rate": 0.0001, + "loss": 57.092, + "step": 10560 + }, + { + "epoch": 170.4878048780488, + "grad_norm": 27.54094123840332, + "learning_rate": 0.0001, + "loss": 56.945, + "step": 10570 + }, + { + "epoch": 170.65040650406505, + "grad_norm": 0.6707919836044312, + "learning_rate": 0.0001, + "loss": 55.8705, + "step": 10580 + }, + { + "epoch": 170.8130081300813, + "grad_norm": 1.0104289054870605, + "learning_rate": 0.0001, + "loss": 56.6519, + "step": 10590 + }, + { + "epoch": 170.97560975609755, + "grad_norm": 1.221200704574585, + "learning_rate": 0.0001, + "loss": 57.0875, + "step": 10600 + }, + { + "epoch": 171.130081300813, + "grad_norm": 1.3986961841583252, + "learning_rate": 0.0001, + "loss": 53.9815, + "step": 10610 + }, + { + "epoch": 171.29268292682926, + "grad_norm": 1.0881519317626953, + "learning_rate": 0.0001, + "loss": 55.8683, + "step": 10620 + }, + { + "epoch": 171.45528455284554, + "grad_norm": 1.172620415687561, + "learning_rate": 0.0001, + "loss": 56.7678, + "step": 10630 + }, + { + "epoch": 171.6178861788618, + "grad_norm": 0.867314875125885, + "learning_rate": 0.0001, + "loss": 55.0647, + "step": 10640 + }, + { + "epoch": 171.78048780487805, + "grad_norm": 1.415421962738037, + "learning_rate": 0.0001, + "loss": 57.5599, + "step": 10650 + }, + { + "epoch": 171.9430894308943, + "grad_norm": 1.0275741815567017, + "learning_rate": 0.0001, + "loss": 55.4741, + "step": 10660 + }, + { + "epoch": 172.09756097560975, + "grad_norm": 0.9037189483642578, + "learning_rate": 0.0001, + "loss": 51.9268, + "step": 10670 + }, + { + "epoch": 172.260162601626, + "grad_norm": 0.9657711386680603, + "learning_rate": 0.0001, + "loss": 57.3183, + "step": 10680 + }, + { + "epoch": 172.4227642276423, + "grad_norm": 0.9031921029090881, + "learning_rate": 0.0001, + "loss": 58.6986, + "step": 10690 + }, + { + "epoch": 172.58536585365854, + "grad_norm": 0.9555232524871826, + "learning_rate": 0.0001, + "loss": 55.7602, + "step": 10700 + }, + { + "epoch": 172.7479674796748, + "grad_norm": 0.985213041305542, + "learning_rate": 0.0001, + "loss": 55.3734, + "step": 10710 + }, + { + "epoch": 172.91056910569105, + "grad_norm": 1.1574981212615967, + "learning_rate": 0.0001, + "loss": 57.8304, + "step": 10720 + }, + { + "epoch": 173.0650406504065, + "grad_norm": 1.0298277139663696, + "learning_rate": 0.0001, + "loss": 54.2185, + "step": 10730 + }, + { + "epoch": 173.22764227642276, + "grad_norm": 1.0708621740341187, + "learning_rate": 0.0001, + "loss": 57.1618, + "step": 10740 + }, + { + "epoch": 173.390243902439, + "grad_norm": 1.0337039232254028, + "learning_rate": 0.0001, + "loss": 56.8093, + "step": 10750 + }, + { + "epoch": 173.5528455284553, + "grad_norm": 1.1115336418151855, + "learning_rate": 0.0001, + "loss": 56.1233, + "step": 10760 + }, + { + "epoch": 173.71544715447155, + "grad_norm": 0.9465245008468628, + "learning_rate": 0.0001, + "loss": 57.79, + "step": 10770 + }, + { + "epoch": 173.8780487804878, + "grad_norm": 1.0282483100891113, + "learning_rate": 0.0001, + "loss": 56.7261, + "step": 10780 + }, + { + "epoch": 174.03252032520325, + "grad_norm": 0.8371811509132385, + "learning_rate": 0.0001, + "loss": 54.3208, + "step": 10790 + }, + { + "epoch": 174.1951219512195, + "grad_norm": 0.75701504945755, + "learning_rate": 0.0001, + "loss": 58.4083, + "step": 10800 + }, + { + "epoch": 174.35772357723576, + "grad_norm": 0.8716368675231934, + "learning_rate": 0.0001, + "loss": 56.3668, + "step": 10810 + }, + { + "epoch": 174.52032520325204, + "grad_norm": 0.8204325437545776, + "learning_rate": 0.0001, + "loss": 57.5088, + "step": 10820 + }, + { + "epoch": 174.6829268292683, + "grad_norm": 0.912240743637085, + "learning_rate": 0.0001, + "loss": 54.392, + "step": 10830 + }, + { + "epoch": 174.84552845528455, + "grad_norm": 1.0001140832901, + "learning_rate": 0.0001, + "loss": 56.2384, + "step": 10840 + }, + { + "epoch": 175.0, + "grad_norm": 1.1657922267913818, + "learning_rate": 0.0001, + "loss": 53.9591, + "step": 10850 + }, + { + "epoch": 175.16260162601625, + "grad_norm": 0.9120311141014099, + "learning_rate": 0.0001, + "loss": 56.7532, + "step": 10860 + }, + { + "epoch": 175.3252032520325, + "grad_norm": 0.821774423122406, + "learning_rate": 0.0001, + "loss": 56.1124, + "step": 10870 + }, + { + "epoch": 175.4878048780488, + "grad_norm": 0.8408815264701843, + "learning_rate": 0.0001, + "loss": 56.8947, + "step": 10880 + }, + { + "epoch": 175.65040650406505, + "grad_norm": 1.2977283000946045, + "learning_rate": 0.0001, + "loss": 55.476, + "step": 10890 + }, + { + "epoch": 175.8130081300813, + "grad_norm": 0.8232496380805969, + "learning_rate": 0.0001, + "loss": 56.0607, + "step": 10900 + }, + { + "epoch": 175.97560975609755, + "grad_norm": 0.8235589265823364, + "learning_rate": 0.0001, + "loss": 56.3365, + "step": 10910 + }, + { + "epoch": 176.130081300813, + "grad_norm": 1.0942591428756714, + "learning_rate": 0.0001, + "loss": 53.065, + "step": 10920 + }, + { + "epoch": 176.29268292682926, + "grad_norm": 0.9026378989219666, + "learning_rate": 0.0001, + "loss": 55.6453, + "step": 10930 + }, + { + "epoch": 176.45528455284554, + "grad_norm": 0.9491869211196899, + "learning_rate": 0.0001, + "loss": 56.4853, + "step": 10940 + }, + { + "epoch": 176.6178861788618, + "grad_norm": 1.072492241859436, + "learning_rate": 0.0001, + "loss": 55.7488, + "step": 10950 + }, + { + "epoch": 176.78048780487805, + "grad_norm": 0.9384275674819946, + "learning_rate": 0.0001, + "loss": 56.8056, + "step": 10960 + }, + { + "epoch": 176.9430894308943, + "grad_norm": 0.9458582997322083, + "learning_rate": 0.0001, + "loss": 57.187, + "step": 10970 + }, + { + "epoch": 177.09756097560975, + "grad_norm": 0.9112198352813721, + "learning_rate": 0.0001, + "loss": 52.9576, + "step": 10980 + }, + { + "epoch": 177.260162601626, + "grad_norm": 1.0807278156280518, + "learning_rate": 0.0001, + "loss": 56.0773, + "step": 10990 + }, + { + "epoch": 177.4227642276423, + "grad_norm": 0.8122154474258423, + "learning_rate": 0.0001, + "loss": 56.0297, + "step": 11000 + }, + { + "epoch": 177.58536585365854, + "grad_norm": 0.7836926579475403, + "learning_rate": 0.0001, + "loss": 56.74, + "step": 11010 + }, + { + "epoch": 177.7479674796748, + "grad_norm": 1.0483779907226562, + "learning_rate": 0.0001, + "loss": 55.1174, + "step": 11020 + }, + { + "epoch": 177.91056910569105, + "grad_norm": 0.8454721570014954, + "learning_rate": 0.0001, + "loss": 55.3519, + "step": 11030 + }, + { + "epoch": 178.0650406504065, + "grad_norm": 1.2100988626480103, + "learning_rate": 0.0001, + "loss": 54.5955, + "step": 11040 + }, + { + "epoch": 178.22764227642276, + "grad_norm": 1.0322265625, + "learning_rate": 0.0001, + "loss": 57.2623, + "step": 11050 + }, + { + "epoch": 178.390243902439, + "grad_norm": 0.8880140781402588, + "learning_rate": 0.0001, + "loss": 56.5935, + "step": 11060 + }, + { + "epoch": 178.5528455284553, + "grad_norm": 0.8864359259605408, + "learning_rate": 0.0001, + "loss": 56.802, + "step": 11070 + }, + { + "epoch": 178.71544715447155, + "grad_norm": 1.1736643314361572, + "learning_rate": 0.0001, + "loss": 56.8208, + "step": 11080 + }, + { + "epoch": 178.8780487804878, + "grad_norm": 0.975029706954956, + "learning_rate": 0.0001, + "loss": 57.5741, + "step": 11090 + }, + { + "epoch": 179.03252032520325, + "grad_norm": 0.852942168712616, + "learning_rate": 0.0001, + "loss": 54.327, + "step": 11100 + }, + { + "epoch": 179.1951219512195, + "grad_norm": 1.1381421089172363, + "learning_rate": 0.0001, + "loss": 56.0875, + "step": 11110 + }, + { + "epoch": 179.35772357723576, + "grad_norm": 1.0923271179199219, + "learning_rate": 0.0001, + "loss": 58.1302, + "step": 11120 + }, + { + "epoch": 179.52032520325204, + "grad_norm": 0.9422796964645386, + "learning_rate": 0.0001, + "loss": 56.2919, + "step": 11130 + }, + { + "epoch": 179.6829268292683, + "grad_norm": 0.9511571526527405, + "learning_rate": 0.0001, + "loss": 55.1329, + "step": 11140 + }, + { + "epoch": 179.84552845528455, + "grad_norm": 0.9221625328063965, + "learning_rate": 0.0001, + "loss": 56.6727, + "step": 11150 + }, + { + "epoch": 180.0, + "grad_norm": 0.8903642296791077, + "learning_rate": 0.0001, + "loss": 53.7472, + "step": 11160 + }, + { + "epoch": 180.16260162601625, + "grad_norm": 0.9752726554870605, + "learning_rate": 0.0001, + "loss": 56.2322, + "step": 11170 + }, + { + "epoch": 180.3252032520325, + "grad_norm": 0.8058391809463501, + "learning_rate": 0.0001, + "loss": 57.7904, + "step": 11180 + }, + { + "epoch": 180.4878048780488, + "grad_norm": 0.9203750491142273, + "learning_rate": 0.0001, + "loss": 55.3608, + "step": 11190 + }, + { + "epoch": 180.65040650406505, + "grad_norm": 1.4366613626480103, + "learning_rate": 0.0001, + "loss": 54.5204, + "step": 11200 + }, + { + "epoch": 180.8130081300813, + "grad_norm": 1.040142297744751, + "learning_rate": 0.0001, + "loss": 56.3904, + "step": 11210 + }, + { + "epoch": 180.97560975609755, + "grad_norm": 1.1629812717437744, + "learning_rate": 0.0001, + "loss": 55.4976, + "step": 11220 + }, + { + "epoch": 181.130081300813, + "grad_norm": 0.847724437713623, + "learning_rate": 0.0001, + "loss": 54.4277, + "step": 11230 + }, + { + "epoch": 181.29268292682926, + "grad_norm": 1.0196763277053833, + "learning_rate": 0.0001, + "loss": 58.1868, + "step": 11240 + }, + { + "epoch": 181.45528455284554, + "grad_norm": 1.2344937324523926, + "learning_rate": 0.0001, + "loss": 56.1104, + "step": 11250 + }, + { + "epoch": 181.6178861788618, + "grad_norm": 0.8804627060890198, + "learning_rate": 0.0001, + "loss": 55.728, + "step": 11260 + }, + { + "epoch": 181.78048780487805, + "grad_norm": 0.8948118686676025, + "learning_rate": 0.0001, + "loss": 56.9878, + "step": 11270 + }, + { + "epoch": 181.9430894308943, + "grad_norm": 0.6648201942443848, + "learning_rate": 0.0001, + "loss": 56.5425, + "step": 11280 + }, + { + "epoch": 182.09756097560975, + "grad_norm": 0.8076626658439636, + "learning_rate": 0.0001, + "loss": 53.4703, + "step": 11290 + }, + { + "epoch": 182.260162601626, + "grad_norm": 0.9063478708267212, + "learning_rate": 0.0001, + "loss": 57.6056, + "step": 11300 + }, + { + "epoch": 182.4227642276423, + "grad_norm": 0.8387530446052551, + "learning_rate": 0.0001, + "loss": 56.0165, + "step": 11310 + }, + { + "epoch": 182.58536585365854, + "grad_norm": 0.8866046667098999, + "learning_rate": 0.0001, + "loss": 57.774, + "step": 11320 + }, + { + "epoch": 182.7479674796748, + "grad_norm": 0.9022682905197144, + "learning_rate": 0.0001, + "loss": 57.5811, + "step": 11330 + }, + { + "epoch": 182.91056910569105, + "grad_norm": 0.8135976195335388, + "learning_rate": 0.0001, + "loss": 56.048, + "step": 11340 + }, + { + "epoch": 183.0650406504065, + "grad_norm": 1.0014373064041138, + "learning_rate": 0.0001, + "loss": 52.5195, + "step": 11350 + }, + { + "epoch": 183.22764227642276, + "grad_norm": 1.0275895595550537, + "learning_rate": 0.0001, + "loss": 57.1421, + "step": 11360 + }, + { + "epoch": 183.390243902439, + "grad_norm": 0.8826107382774353, + "learning_rate": 0.0001, + "loss": 57.5582, + "step": 11370 + }, + { + "epoch": 183.5528455284553, + "grad_norm": 0.9023060202598572, + "learning_rate": 0.0001, + "loss": 56.6695, + "step": 11380 + }, + { + "epoch": 183.71544715447155, + "grad_norm": 0.7782049179077148, + "learning_rate": 0.0001, + "loss": 55.4362, + "step": 11390 + }, + { + "epoch": 183.8780487804878, + "grad_norm": 0.9904818534851074, + "learning_rate": 0.0001, + "loss": 56.7164, + "step": 11400 + }, + { + "epoch": 184.03252032520325, + "grad_norm": 0.9218631982803345, + "learning_rate": 0.0001, + "loss": 53.8536, + "step": 11410 + }, + { + "epoch": 184.1951219512195, + "grad_norm": 0.7807294130325317, + "learning_rate": 0.0001, + "loss": 57.1502, + "step": 11420 + }, + { + "epoch": 184.35772357723576, + "grad_norm": 0.8353742957115173, + "learning_rate": 0.0001, + "loss": 56.8276, + "step": 11430 + }, + { + "epoch": 184.52032520325204, + "grad_norm": 0.7909665107727051, + "learning_rate": 0.0001, + "loss": 56.4164, + "step": 11440 + }, + { + "epoch": 184.6829268292683, + "grad_norm": 1.0313756465911865, + "learning_rate": 0.0001, + "loss": 56.1275, + "step": 11450 + }, + { + "epoch": 184.84552845528455, + "grad_norm": 0.9377022385597229, + "learning_rate": 0.0001, + "loss": 55.9829, + "step": 11460 + }, + { + "epoch": 185.0, + "grad_norm": 1.2855385541915894, + "learning_rate": 0.0001, + "loss": 53.6264, + "step": 11470 + }, + { + "epoch": 185.16260162601625, + "grad_norm": 1.7889217138290405, + "learning_rate": 0.0001, + "loss": 55.1279, + "step": 11480 + }, + { + "epoch": 185.3252032520325, + "grad_norm": 0.8415203094482422, + "learning_rate": 0.0001, + "loss": 57.1529, + "step": 11490 + }, + { + "epoch": 185.4878048780488, + "grad_norm": 1.1245211362838745, + "learning_rate": 0.0001, + "loss": 56.2543, + "step": 11500 + }, + { + "epoch": 185.65040650406505, + "grad_norm": 1.607924222946167, + "learning_rate": 0.0001, + "loss": 55.9007, + "step": 11510 + }, + { + "epoch": 185.8130081300813, + "grad_norm": 0.8068832755088806, + "learning_rate": 0.0001, + "loss": 56.5567, + "step": 11520 + }, + { + "epoch": 185.97560975609755, + "grad_norm": 1.109636902809143, + "learning_rate": 0.0001, + "loss": 55.8676, + "step": 11530 + }, + { + "epoch": 186.130081300813, + "grad_norm": 0.9668437242507935, + "learning_rate": 0.0001, + "loss": 52.9045, + "step": 11540 + }, + { + "epoch": 186.29268292682926, + "grad_norm": 1.1277968883514404, + "learning_rate": 0.0001, + "loss": 55.0794, + "step": 11550 + }, + { + "epoch": 186.45528455284554, + "grad_norm": 0.6413527727127075, + "learning_rate": 0.0001, + "loss": 55.4634, + "step": 11560 + }, + { + "epoch": 186.6178861788618, + "grad_norm": 1.039292812347412, + "learning_rate": 0.0001, + "loss": 56.2108, + "step": 11570 + }, + { + "epoch": 186.78048780487805, + "grad_norm": 0.9112428426742554, + "learning_rate": 0.0001, + "loss": 56.4036, + "step": 11580 + }, + { + "epoch": 186.9430894308943, + "grad_norm": 0.8918914794921875, + "learning_rate": 0.0001, + "loss": 54.9626, + "step": 11590 + }, + { + "epoch": 187.09756097560975, + "grad_norm": 0.9715940952301025, + "learning_rate": 0.0001, + "loss": 54.0208, + "step": 11600 + }, + { + "epoch": 187.260162601626, + "grad_norm": 0.8578653931617737, + "learning_rate": 0.0001, + "loss": 56.4479, + "step": 11610 + }, + { + "epoch": 187.4227642276423, + "grad_norm": 1.7716244459152222, + "learning_rate": 0.0001, + "loss": 55.3543, + "step": 11620 + }, + { + "epoch": 187.58536585365854, + "grad_norm": 0.9152880311012268, + "learning_rate": 0.0001, + "loss": 56.8464, + "step": 11630 + }, + { + "epoch": 187.7479674796748, + "grad_norm": 0.8824559450149536, + "learning_rate": 0.0001, + "loss": 55.9501, + "step": 11640 + }, + { + "epoch": 187.91056910569105, + "grad_norm": 0.780864953994751, + "learning_rate": 0.0001, + "loss": 57.3594, + "step": 11650 + }, + { + "epoch": 188.0650406504065, + "grad_norm": 0.7242524027824402, + "learning_rate": 0.0001, + "loss": 53.9958, + "step": 11660 + }, + { + "epoch": 188.22764227642276, + "grad_norm": 0.9025135636329651, + "learning_rate": 0.0001, + "loss": 54.6309, + "step": 11670 + }, + { + "epoch": 188.390243902439, + "grad_norm": 0.7995520830154419, + "learning_rate": 0.0001, + "loss": 56.7882, + "step": 11680 + }, + { + "epoch": 188.5528455284553, + "grad_norm": 0.8525217175483704, + "learning_rate": 0.0001, + "loss": 54.7559, + "step": 11690 + }, + { + "epoch": 188.71544715447155, + "grad_norm": 0.9954654574394226, + "learning_rate": 0.0001, + "loss": 56.4574, + "step": 11700 + }, + { + "epoch": 188.8780487804878, + "grad_norm": 0.851827085018158, + "learning_rate": 0.0001, + "loss": 55.996, + "step": 11710 + }, + { + "epoch": 189.03252032520325, + "grad_norm": 0.9404662847518921, + "learning_rate": 0.0001, + "loss": 53.8035, + "step": 11720 + }, + { + "epoch": 189.1951219512195, + "grad_norm": 0.8355186581611633, + "learning_rate": 0.0001, + "loss": 54.9002, + "step": 11730 + }, + { + "epoch": 189.35772357723576, + "grad_norm": 1.0156077146530151, + "learning_rate": 0.0001, + "loss": 55.8483, + "step": 11740 + }, + { + "epoch": 189.52032520325204, + "grad_norm": 0.7925308346748352, + "learning_rate": 0.0001, + "loss": 54.1772, + "step": 11750 + }, + { + "epoch": 189.6829268292683, + "grad_norm": 0.8137376308441162, + "learning_rate": 0.0001, + "loss": 53.6278, + "step": 11760 + }, + { + "epoch": 189.84552845528455, + "grad_norm": 0.7871745824813843, + "learning_rate": 0.0001, + "loss": 57.7457, + "step": 11770 + }, + { + "epoch": 190.0, + "grad_norm": 0.7344806790351868, + "learning_rate": 0.0001, + "loss": 55.6082, + "step": 11780 + }, + { + "epoch": 190.16260162601625, + "grad_norm": 0.9094588756561279, + "learning_rate": 0.0001, + "loss": 57.2329, + "step": 11790 + }, + { + "epoch": 190.3252032520325, + "grad_norm": 1.2261676788330078, + "learning_rate": 0.0001, + "loss": 56.2071, + "step": 11800 + }, + { + "epoch": 190.4878048780488, + "grad_norm": 0.9519370794296265, + "learning_rate": 0.0001, + "loss": 56.2488, + "step": 11810 + }, + { + "epoch": 190.65040650406505, + "grad_norm": 0.8949872851371765, + "learning_rate": 0.0001, + "loss": 57.9283, + "step": 11820 + }, + { + "epoch": 190.8130081300813, + "grad_norm": 0.8764474391937256, + "learning_rate": 0.0001, + "loss": 55.2962, + "step": 11830 + }, + { + "epoch": 190.97560975609755, + "grad_norm": 0.9472342729568481, + "learning_rate": 0.0001, + "loss": 56.2262, + "step": 11840 + }, + { + "epoch": 191.130081300813, + "grad_norm": 0.7477512359619141, + "learning_rate": 0.0001, + "loss": 51.2613, + "step": 11850 + }, + { + "epoch": 191.29268292682926, + "grad_norm": 1.0346648693084717, + "learning_rate": 0.0001, + "loss": 56.0406, + "step": 11860 + }, + { + "epoch": 191.45528455284554, + "grad_norm": 1.0473452806472778, + "learning_rate": 0.0001, + "loss": 56.3583, + "step": 11870 + }, + { + "epoch": 191.6178861788618, + "grad_norm": 1.0167906284332275, + "learning_rate": 0.0001, + "loss": 56.3803, + "step": 11880 + }, + { + "epoch": 191.78048780487805, + "grad_norm": 0.8768972754478455, + "learning_rate": 0.0001, + "loss": 57.2783, + "step": 11890 + }, + { + "epoch": 191.9430894308943, + "grad_norm": 0.9370661973953247, + "learning_rate": 0.0001, + "loss": 55.9976, + "step": 11900 + }, + { + "epoch": 192.09756097560975, + "grad_norm": 0.7853850722312927, + "learning_rate": 0.0001, + "loss": 51.9341, + "step": 11910 + }, + { + "epoch": 192.260162601626, + "grad_norm": 1.122916579246521, + "learning_rate": 0.0001, + "loss": 55.832, + "step": 11920 + }, + { + "epoch": 192.4227642276423, + "grad_norm": 0.982596755027771, + "learning_rate": 0.0001, + "loss": 54.2484, + "step": 11930 + }, + { + "epoch": 192.58536585365854, + "grad_norm": 1.0893633365631104, + "learning_rate": 0.0001, + "loss": 54.4806, + "step": 11940 + }, + { + "epoch": 192.7479674796748, + "grad_norm": 0.8936525583267212, + "learning_rate": 0.0001, + "loss": 55.1085, + "step": 11950 + }, + { + "epoch": 192.91056910569105, + "grad_norm": 0.9169464111328125, + "learning_rate": 0.0001, + "loss": 56.8134, + "step": 11960 + }, + { + "epoch": 193.0650406504065, + "grad_norm": 1.2202202081680298, + "learning_rate": 0.0001, + "loss": 53.3595, + "step": 11970 + }, + { + "epoch": 193.22764227642276, + "grad_norm": 0.7786313891410828, + "learning_rate": 0.0001, + "loss": 57.6186, + "step": 11980 + }, + { + "epoch": 193.390243902439, + "grad_norm": 0.9757220149040222, + "learning_rate": 0.0001, + "loss": 54.5559, + "step": 11990 + }, + { + "epoch": 193.5528455284553, + "grad_norm": 0.8599177002906799, + "learning_rate": 0.0001, + "loss": 54.9894, + "step": 12000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 323, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4575849820890176e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c7f229ebe1034caaf3e92006c8c7528c21af93d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517586849d044483051c7a59cec7c6ce389951878a7357ef837edb1cbc9983c8 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-12000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/latest new file mode 100644 index 0000000000000000000000000000000000000000..6c2e99ab39c9aa1d68dfdcb53dc9234015ced784 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/latest @@ -0,0 +1 @@ +global_step14000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9517be8c7b6851218885fd7bba8999ba6237bddc --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c091d5d88cba07993eec9666f34ef451ebdcca89d64edefe7cffacf44776714 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..6d65b0bad7a96eace8ab105d828aa0c8dcda7194 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a28290fb1e92ae0853dd9c0e6866054725d4b2f13d350ba31e6054a711c456a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..761338ddfeba94af79c826c93d6d6a0eeca94fe5 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23f173e44a4f2484051812acf1f8267e9ff3d45df800fea5c5700e4bfb52c5c5 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..3a0b1741564aa77f604caefe0dc4a48afb1f3812 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fb5ea44555457b236d70b75631518911a18c92d4a5a1f65d4123532dc578e6d +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..eeecd5affef3c52f23bb9e7804610f9cdcc4c3f7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69562ed935e5595e38289fda751028bbeafdaf35535b856a55b282b703b5dc65 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..1a07055c8ddfa26bb99e015133dfe08915332e33 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57caffe1c31ce713b7c2715f367221ff0fda96a8c6f7777e2470f8cc50ee4486 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..ffbcd2835d2537e5da31221b8077e06749acf49f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d08c2c4231079077e8f932b00de9349cc999cf96452bb39db418d30d9876cfe +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..2ed930a04015b061de531f0924c5ca11c339570a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1d277563b2c2dc9306c53ae6175ccb04db9c64dd20b81e01174e394008f2094 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..d6e942f831102d905e69bd9c587ea3aece845684 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de64123b7ab06d1933a5fe6b37c14111853b41e819e809db19d17725e8c28e50 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..f41b87afdce1fcc0b7b9eff4e6b9589700d352b3 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77cff55491dcd98e210e6c07623399a07e8a123042ff2c25bda661be0e7132f3 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b0a6134c27387515a98cecda965af20a00634675 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/trainer_state.json @@ -0,0 +1,9834 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 225.8130081300813, + "eval_steps": 2000, + "global_step": 14000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.16260162601626016, + "grad_norm": 23.72986602783203, + "learning_rate": 0.0001, + "loss": 271.6598, + "step": 10 + }, + { + "epoch": 0.3252032520325203, + "grad_norm": 18.99613380432129, + "learning_rate": 0.0001, + "loss": 233.029, + "step": 20 + }, + { + "epoch": 0.4878048780487805, + "grad_norm": 3.9815304279327393, + "learning_rate": 0.0001, + "loss": 231.0776, + "step": 30 + }, + { + "epoch": 0.6504065040650406, + "grad_norm": 4.641753196716309, + "learning_rate": 0.0001, + "loss": 228.6894, + "step": 40 + }, + { + "epoch": 0.8130081300813008, + "grad_norm": 3.000852108001709, + "learning_rate": 0.0001, + "loss": 228.4694, + "step": 50 + }, + { + "epoch": 0.975609756097561, + "grad_norm": 12.213018417358398, + "learning_rate": 0.0001, + "loss": 226.4258, + "step": 60 + }, + { + "epoch": 1.1300813008130082, + "grad_norm": 2.535187005996704, + "learning_rate": 0.0001, + "loss": 209.9772, + "step": 70 + }, + { + "epoch": 1.2926829268292683, + "grad_norm": 2.0984582901000977, + "learning_rate": 0.0001, + "loss": 217.2421, + "step": 80 + }, + { + "epoch": 1.4552845528455285, + "grad_norm": 3.4802305698394775, + "learning_rate": 0.0001, + "loss": 210.9241, + "step": 90 + }, + { + "epoch": 1.6178861788617886, + "grad_norm": 3.813626527786255, + "learning_rate": 0.0001, + "loss": 205.7566, + "step": 100 + }, + { + "epoch": 1.7804878048780488, + "grad_norm": 2.6378512382507324, + "learning_rate": 0.0001, + "loss": 201.8789, + "step": 110 + }, + { + "epoch": 1.943089430894309, + "grad_norm": 2.7502875328063965, + "learning_rate": 0.0001, + "loss": 196.373, + "step": 120 + }, + { + "epoch": 2.097560975609756, + "grad_norm": 2.659212112426758, + "learning_rate": 0.0001, + "loss": 183.1456, + "step": 130 + }, + { + "epoch": 2.2601626016260163, + "grad_norm": 2.513044595718384, + "learning_rate": 0.0001, + "loss": 190.1661, + "step": 140 + }, + { + "epoch": 2.4227642276422765, + "grad_norm": 2.3192145824432373, + "learning_rate": 0.0001, + "loss": 184.9997, + "step": 150 + }, + { + "epoch": 2.5853658536585367, + "grad_norm": 3.056708574295044, + "learning_rate": 0.0001, + "loss": 182.0164, + "step": 160 + }, + { + "epoch": 2.747967479674797, + "grad_norm": 2.9687108993530273, + "learning_rate": 0.0001, + "loss": 179.6539, + "step": 170 + }, + { + "epoch": 2.910569105691057, + "grad_norm": 2.1498093605041504, + "learning_rate": 0.0001, + "loss": 176.7159, + "step": 180 + }, + { + "epoch": 3.065040650406504, + "grad_norm": 3.0103771686553955, + "learning_rate": 0.0001, + "loss": 166.1703, + "step": 190 + }, + { + "epoch": 3.227642276422764, + "grad_norm": 2.992462635040283, + "learning_rate": 0.0001, + "loss": 171.124, + "step": 200 + }, + { + "epoch": 3.3902439024390243, + "grad_norm": 4.8617119789123535, + "learning_rate": 0.0001, + "loss": 168.2646, + "step": 210 + }, + { + "epoch": 3.5528455284552845, + "grad_norm": 3.1047444343566895, + "learning_rate": 0.0001, + "loss": 163.0889, + "step": 220 + }, + { + "epoch": 3.7154471544715446, + "grad_norm": 6.473227500915527, + "learning_rate": 0.0001, + "loss": 151.3667, + "step": 230 + }, + { + "epoch": 3.8780487804878048, + "grad_norm": 6.16747522354126, + "learning_rate": 0.0001, + "loss": 137.5915, + "step": 240 + }, + { + "epoch": 4.032520325203252, + "grad_norm": 7.994515419006348, + "learning_rate": 0.0001, + "loss": 118.8875, + "step": 250 + }, + { + "epoch": 4.195121951219512, + "grad_norm": 5.882101058959961, + "learning_rate": 0.0001, + "loss": 119.4793, + "step": 260 + }, + { + "epoch": 4.357723577235772, + "grad_norm": 12.32968521118164, + "learning_rate": 0.0001, + "loss": 113.1209, + "step": 270 + }, + { + "epoch": 4.520325203252033, + "grad_norm": 5.697225570678711, + "learning_rate": 0.0001, + "loss": 107.51, + "step": 280 + }, + { + "epoch": 4.682926829268292, + "grad_norm": 4.654167175292969, + "learning_rate": 0.0001, + "loss": 107.0172, + "step": 290 + }, + { + "epoch": 4.845528455284553, + "grad_norm": 2.9595484733581543, + "learning_rate": 0.0001, + "loss": 100.9164, + "step": 300 + }, + { + "epoch": 5.0, + "grad_norm": 21.242591857910156, + "learning_rate": 0.0001, + "loss": 94.9884, + "step": 310 + }, + { + "epoch": 5.16260162601626, + "grad_norm": 5.760314464569092, + "learning_rate": 0.0001, + "loss": 97.5901, + "step": 320 + }, + { + "epoch": 5.32520325203252, + "grad_norm": 3.1373329162597656, + "learning_rate": 0.0001, + "loss": 94.8485, + "step": 330 + }, + { + "epoch": 5.487804878048781, + "grad_norm": 2.3777568340301514, + "learning_rate": 0.0001, + "loss": 96.9982, + "step": 340 + }, + { + "epoch": 5.650406504065041, + "grad_norm": 3.0607948303222656, + "learning_rate": 0.0001, + "loss": 96.1586, + "step": 350 + }, + { + "epoch": 5.8130081300813, + "grad_norm": 4.168269634246826, + "learning_rate": 0.0001, + "loss": 94.5743, + "step": 360 + }, + { + "epoch": 5.975609756097561, + "grad_norm": 2.4383699893951416, + "learning_rate": 0.0001, + "loss": 95.5197, + "step": 370 + }, + { + "epoch": 6.130081300813008, + "grad_norm": 2.302259683609009, + "learning_rate": 0.0001, + "loss": 85.6599, + "step": 380 + }, + { + "epoch": 6.2926829268292686, + "grad_norm": 2.552753210067749, + "learning_rate": 0.0001, + "loss": 91.2999, + "step": 390 + }, + { + "epoch": 6.455284552845528, + "grad_norm": 2.2092080116271973, + "learning_rate": 0.0001, + "loss": 90.7041, + "step": 400 + }, + { + "epoch": 6.617886178861789, + "grad_norm": 2.2935690879821777, + "learning_rate": 0.0001, + "loss": 89.9949, + "step": 410 + }, + { + "epoch": 6.780487804878049, + "grad_norm": 2.487510919570923, + "learning_rate": 0.0001, + "loss": 86.9056, + "step": 420 + }, + { + "epoch": 6.943089430894309, + "grad_norm": 2.8630545139312744, + "learning_rate": 0.0001, + "loss": 88.3347, + "step": 430 + }, + { + "epoch": 7.097560975609756, + "grad_norm": 5.87056303024292, + "learning_rate": 0.0001, + "loss": 84.9632, + "step": 440 + }, + { + "epoch": 7.260162601626016, + "grad_norm": 2.587661027908325, + "learning_rate": 0.0001, + "loss": 86.2024, + "step": 450 + }, + { + "epoch": 7.4227642276422765, + "grad_norm": 3.5671255588531494, + "learning_rate": 0.0001, + "loss": 86.218, + "step": 460 + }, + { + "epoch": 7.585365853658536, + "grad_norm": 2.5036680698394775, + "learning_rate": 0.0001, + "loss": 86.5914, + "step": 470 + }, + { + "epoch": 7.747967479674797, + "grad_norm": 3.2656471729278564, + "learning_rate": 0.0001, + "loss": 82.6081, + "step": 480 + }, + { + "epoch": 7.9105691056910565, + "grad_norm": 2.5053915977478027, + "learning_rate": 0.0001, + "loss": 84.6116, + "step": 490 + }, + { + "epoch": 8.065040650406504, + "grad_norm": 2.1669957637786865, + "learning_rate": 0.0001, + "loss": 80.4354, + "step": 500 + }, + { + "epoch": 8.227642276422765, + "grad_norm": 4.2003703117370605, + "learning_rate": 0.0001, + "loss": 85.2286, + "step": 510 + }, + { + "epoch": 8.390243902439025, + "grad_norm": 3.1909608840942383, + "learning_rate": 0.0001, + "loss": 85.0358, + "step": 520 + }, + { + "epoch": 8.552845528455284, + "grad_norm": 1.8420121669769287, + "learning_rate": 0.0001, + "loss": 85.4647, + "step": 530 + }, + { + "epoch": 8.715447154471544, + "grad_norm": 1.9822801351547241, + "learning_rate": 0.0001, + "loss": 84.7134, + "step": 540 + }, + { + "epoch": 8.878048780487806, + "grad_norm": 2.936758518218994, + "learning_rate": 0.0001, + "loss": 83.8068, + "step": 550 + }, + { + "epoch": 9.032520325203253, + "grad_norm": 1.5713881254196167, + "learning_rate": 0.0001, + "loss": 80.0695, + "step": 560 + }, + { + "epoch": 9.195121951219512, + "grad_norm": 2.0584616661071777, + "learning_rate": 0.0001, + "loss": 84.5158, + "step": 570 + }, + { + "epoch": 9.357723577235772, + "grad_norm": 2.101391553878784, + "learning_rate": 0.0001, + "loss": 82.4719, + "step": 580 + }, + { + "epoch": 9.520325203252032, + "grad_norm": 1.9956543445587158, + "learning_rate": 0.0001, + "loss": 81.6715, + "step": 590 + }, + { + "epoch": 9.682926829268293, + "grad_norm": 2.7622063159942627, + "learning_rate": 0.0001, + "loss": 85.4178, + "step": 600 + }, + { + "epoch": 9.845528455284553, + "grad_norm": 2.2084789276123047, + "learning_rate": 0.0001, + "loss": 84.5702, + "step": 610 + }, + { + "epoch": 10.0, + "grad_norm": 1.9635663032531738, + "learning_rate": 0.0001, + "loss": 77.3838, + "step": 620 + }, + { + "epoch": 10.16260162601626, + "grad_norm": 2.5895581245422363, + "learning_rate": 0.0001, + "loss": 81.723, + "step": 630 + }, + { + "epoch": 10.32520325203252, + "grad_norm": 2.743727445602417, + "learning_rate": 0.0001, + "loss": 83.1332, + "step": 640 + }, + { + "epoch": 10.487804878048781, + "grad_norm": 2.024585008621216, + "learning_rate": 0.0001, + "loss": 81.5968, + "step": 650 + }, + { + "epoch": 10.65040650406504, + "grad_norm": 1.8254153728485107, + "learning_rate": 0.0001, + "loss": 78.9589, + "step": 660 + }, + { + "epoch": 10.8130081300813, + "grad_norm": 3.2015740871429443, + "learning_rate": 0.0001, + "loss": 79.1829, + "step": 670 + }, + { + "epoch": 10.975609756097562, + "grad_norm": 1.788453459739685, + "learning_rate": 0.0001, + "loss": 81.527, + "step": 680 + }, + { + "epoch": 11.130081300813009, + "grad_norm": 1.5744423866271973, + "learning_rate": 0.0001, + "loss": 74.9869, + "step": 690 + }, + { + "epoch": 11.292682926829269, + "grad_norm": 2.006002187728882, + "learning_rate": 0.0001, + "loss": 80.2677, + "step": 700 + }, + { + "epoch": 11.455284552845528, + "grad_norm": 1.8847148418426514, + "learning_rate": 0.0001, + "loss": 78.8709, + "step": 710 + }, + { + "epoch": 11.617886178861788, + "grad_norm": 1.8384478092193604, + "learning_rate": 0.0001, + "loss": 79.008, + "step": 720 + }, + { + "epoch": 11.78048780487805, + "grad_norm": 1.9486687183380127, + "learning_rate": 0.0001, + "loss": 76.1525, + "step": 730 + }, + { + "epoch": 11.94308943089431, + "grad_norm": 1.8644076585769653, + "learning_rate": 0.0001, + "loss": 77.5802, + "step": 740 + }, + { + "epoch": 12.097560975609756, + "grad_norm": 1.6035499572753906, + "learning_rate": 0.0001, + "loss": 75.2422, + "step": 750 + }, + { + "epoch": 12.260162601626016, + "grad_norm": 1.6935622692108154, + "learning_rate": 0.0001, + "loss": 77.5838, + "step": 760 + }, + { + "epoch": 12.422764227642276, + "grad_norm": 1.9327987432479858, + "learning_rate": 0.0001, + "loss": 78.3237, + "step": 770 + }, + { + "epoch": 12.585365853658537, + "grad_norm": 1.7903852462768555, + "learning_rate": 0.0001, + "loss": 78.7459, + "step": 780 + }, + { + "epoch": 12.747967479674797, + "grad_norm": 1.4962575435638428, + "learning_rate": 0.0001, + "loss": 78.9193, + "step": 790 + }, + { + "epoch": 12.910569105691057, + "grad_norm": 42.10435104370117, + "learning_rate": 0.0001, + "loss": 80.1429, + "step": 800 + }, + { + "epoch": 13.065040650406504, + "grad_norm": 1.904065728187561, + "learning_rate": 0.0001, + "loss": 73.7798, + "step": 810 + }, + { + "epoch": 13.227642276422765, + "grad_norm": 1.8768481016159058, + "learning_rate": 0.0001, + "loss": 76.8692, + "step": 820 + }, + { + "epoch": 13.390243902439025, + "grad_norm": 16.140565872192383, + "learning_rate": 0.0001, + "loss": 78.122, + "step": 830 + }, + { + "epoch": 13.552845528455284, + "grad_norm": 2.160288095474243, + "learning_rate": 0.0001, + "loss": 76.2169, + "step": 840 + }, + { + "epoch": 13.715447154471544, + "grad_norm": 27.682231903076172, + "learning_rate": 0.0001, + "loss": 76.6266, + "step": 850 + }, + { + "epoch": 13.878048780487806, + "grad_norm": 1.779772162437439, + "learning_rate": 0.0001, + "loss": 80.0434, + "step": 860 + }, + { + "epoch": 14.032520325203253, + "grad_norm": 1.6516485214233398, + "learning_rate": 0.0001, + "loss": 73.3943, + "step": 870 + }, + { + "epoch": 14.195121951219512, + "grad_norm": 1.846498966217041, + "learning_rate": 0.0001, + "loss": 77.5157, + "step": 880 + }, + { + "epoch": 14.357723577235772, + "grad_norm": 1.942946434020996, + "learning_rate": 0.0001, + "loss": 77.6944, + "step": 890 + }, + { + "epoch": 14.520325203252032, + "grad_norm": 2.0929224491119385, + "learning_rate": 0.0001, + "loss": 76.9411, + "step": 900 + }, + { + "epoch": 14.682926829268293, + "grad_norm": 3.4498767852783203, + "learning_rate": 0.0001, + "loss": 74.9234, + "step": 910 + }, + { + "epoch": 14.845528455284553, + "grad_norm": 1.6880728006362915, + "learning_rate": 0.0001, + "loss": 75.3617, + "step": 920 + }, + { + "epoch": 15.0, + "grad_norm": 1.8403468132019043, + "learning_rate": 0.0001, + "loss": 73.2543, + "step": 930 + }, + { + "epoch": 15.16260162601626, + "grad_norm": 1.4368852376937866, + "learning_rate": 0.0001, + "loss": 76.6271, + "step": 940 + }, + { + "epoch": 15.32520325203252, + "grad_norm": 2.9467780590057373, + "learning_rate": 0.0001, + "loss": 75.8744, + "step": 950 + }, + { + "epoch": 15.487804878048781, + "grad_norm": 1.567134141921997, + "learning_rate": 0.0001, + "loss": 76.6629, + "step": 960 + }, + { + "epoch": 15.65040650406504, + "grad_norm": 1.4503871202468872, + "learning_rate": 0.0001, + "loss": 76.3525, + "step": 970 + }, + { + "epoch": 15.8130081300813, + "grad_norm": 1.8871147632598877, + "learning_rate": 0.0001, + "loss": 77.4684, + "step": 980 + }, + { + "epoch": 15.975609756097562, + "grad_norm": 1.6707026958465576, + "learning_rate": 0.0001, + "loss": 76.3063, + "step": 990 + }, + { + "epoch": 16.130081300813007, + "grad_norm": 3.6300716400146484, + "learning_rate": 0.0001, + "loss": 73.0586, + "step": 1000 + }, + { + "epoch": 16.29268292682927, + "grad_norm": 1.6669234037399292, + "learning_rate": 0.0001, + "loss": 75.8027, + "step": 1010 + }, + { + "epoch": 16.45528455284553, + "grad_norm": 1.6920552253723145, + "learning_rate": 0.0001, + "loss": 74.7252, + "step": 1020 + }, + { + "epoch": 16.617886178861788, + "grad_norm": 1.8184431791305542, + "learning_rate": 0.0001, + "loss": 75.3625, + "step": 1030 + }, + { + "epoch": 16.78048780487805, + "grad_norm": 1.5828496217727661, + "learning_rate": 0.0001, + "loss": 78.0302, + "step": 1040 + }, + { + "epoch": 16.943089430894307, + "grad_norm": 1.7372915744781494, + "learning_rate": 0.0001, + "loss": 78.6949, + "step": 1050 + }, + { + "epoch": 17.097560975609756, + "grad_norm": 1.837161898612976, + "learning_rate": 0.0001, + "loss": 74.4726, + "step": 1060 + }, + { + "epoch": 17.260162601626018, + "grad_norm": 3.8733808994293213, + "learning_rate": 0.0001, + "loss": 76.1711, + "step": 1070 + }, + { + "epoch": 17.422764227642276, + "grad_norm": 1.5907851457595825, + "learning_rate": 0.0001, + "loss": 75.8292, + "step": 1080 + }, + { + "epoch": 17.585365853658537, + "grad_norm": 1.8271747827529907, + "learning_rate": 0.0001, + "loss": 74.5316, + "step": 1090 + }, + { + "epoch": 17.747967479674795, + "grad_norm": 1.827980875968933, + "learning_rate": 0.0001, + "loss": 74.7281, + "step": 1100 + }, + { + "epoch": 17.910569105691057, + "grad_norm": 1.4184074401855469, + "learning_rate": 0.0001, + "loss": 75.736, + "step": 1110 + }, + { + "epoch": 18.065040650406505, + "grad_norm": 2.3459699153900146, + "learning_rate": 0.0001, + "loss": 71.9586, + "step": 1120 + }, + { + "epoch": 18.227642276422763, + "grad_norm": 47.673641204833984, + "learning_rate": 0.0001, + "loss": 75.6649, + "step": 1130 + }, + { + "epoch": 18.390243902439025, + "grad_norm": 1.677398443222046, + "learning_rate": 0.0001, + "loss": 76.0574, + "step": 1140 + }, + { + "epoch": 18.552845528455286, + "grad_norm": 1.4180850982666016, + "learning_rate": 0.0001, + "loss": 74.3077, + "step": 1150 + }, + { + "epoch": 18.715447154471544, + "grad_norm": 2.0041465759277344, + "learning_rate": 0.0001, + "loss": 73.5916, + "step": 1160 + }, + { + "epoch": 18.878048780487806, + "grad_norm": 1.9098238945007324, + "learning_rate": 0.0001, + "loss": 74.5002, + "step": 1170 + }, + { + "epoch": 19.03252032520325, + "grad_norm": 1.405555248260498, + "learning_rate": 0.0001, + "loss": 69.8115, + "step": 1180 + }, + { + "epoch": 19.195121951219512, + "grad_norm": 1.424198865890503, + "learning_rate": 0.0001, + "loss": 73.7289, + "step": 1190 + }, + { + "epoch": 19.357723577235774, + "grad_norm": 7.542550563812256, + "learning_rate": 0.0001, + "loss": 73.7696, + "step": 1200 + }, + { + "epoch": 19.520325203252032, + "grad_norm": 1.425329566001892, + "learning_rate": 0.0001, + "loss": 72.1261, + "step": 1210 + }, + { + "epoch": 19.682926829268293, + "grad_norm": 1.525724172592163, + "learning_rate": 0.0001, + "loss": 73.6578, + "step": 1220 + }, + { + "epoch": 19.84552845528455, + "grad_norm": 1.6387193202972412, + "learning_rate": 0.0001, + "loss": 74.3027, + "step": 1230 + }, + { + "epoch": 20.0, + "grad_norm": 1.8429840803146362, + "learning_rate": 0.0001, + "loss": 69.5056, + "step": 1240 + }, + { + "epoch": 20.16260162601626, + "grad_norm": 2.5166006088256836, + "learning_rate": 0.0001, + "loss": 73.5965, + "step": 1250 + }, + { + "epoch": 20.32520325203252, + "grad_norm": 1.6006672382354736, + "learning_rate": 0.0001, + "loss": 72.4581, + "step": 1260 + }, + { + "epoch": 20.48780487804878, + "grad_norm": 1.6722878217697144, + "learning_rate": 0.0001, + "loss": 72.3097, + "step": 1270 + }, + { + "epoch": 20.65040650406504, + "grad_norm": 1.8715728521347046, + "learning_rate": 0.0001, + "loss": 73.4752, + "step": 1280 + }, + { + "epoch": 20.8130081300813, + "grad_norm": 1.3280900716781616, + "learning_rate": 0.0001, + "loss": 73.2145, + "step": 1290 + }, + { + "epoch": 20.975609756097562, + "grad_norm": 1.941451907157898, + "learning_rate": 0.0001, + "loss": 73.2085, + "step": 1300 + }, + { + "epoch": 21.130081300813007, + "grad_norm": 4.277497291564941, + "learning_rate": 0.0001, + "loss": 68.8305, + "step": 1310 + }, + { + "epoch": 21.29268292682927, + "grad_norm": 1.9949846267700195, + "learning_rate": 0.0001, + "loss": 72.6689, + "step": 1320 + }, + { + "epoch": 21.45528455284553, + "grad_norm": 1.524044394493103, + "learning_rate": 0.0001, + "loss": 72.9705, + "step": 1330 + }, + { + "epoch": 21.617886178861788, + "grad_norm": 1.384194016456604, + "learning_rate": 0.0001, + "loss": 72.5465, + "step": 1340 + }, + { + "epoch": 21.78048780487805, + "grad_norm": 1.724225640296936, + "learning_rate": 0.0001, + "loss": 72.9203, + "step": 1350 + }, + { + "epoch": 21.943089430894307, + "grad_norm": 2.0856704711914062, + "learning_rate": 0.0001, + "loss": 72.4034, + "step": 1360 + }, + { + "epoch": 22.097560975609756, + "grad_norm": 5.128044128417969, + "learning_rate": 0.0001, + "loss": 68.7754, + "step": 1370 + }, + { + "epoch": 22.260162601626018, + "grad_norm": 1.7385634183883667, + "learning_rate": 0.0001, + "loss": 73.9349, + "step": 1380 + }, + { + "epoch": 22.422764227642276, + "grad_norm": 1.3946012258529663, + "learning_rate": 0.0001, + "loss": 70.9627, + "step": 1390 + }, + { + "epoch": 22.585365853658537, + "grad_norm": 1.285414457321167, + "learning_rate": 0.0001, + "loss": 72.4578, + "step": 1400 + }, + { + "epoch": 22.747967479674795, + "grad_norm": 1.4197710752487183, + "learning_rate": 0.0001, + "loss": 72.5252, + "step": 1410 + }, + { + "epoch": 22.910569105691057, + "grad_norm": 3.048942804336548, + "learning_rate": 0.0001, + "loss": 71.1107, + "step": 1420 + }, + { + "epoch": 23.065040650406505, + "grad_norm": 1.3114765882492065, + "learning_rate": 0.0001, + "loss": 68.2725, + "step": 1430 + }, + { + "epoch": 23.227642276422763, + "grad_norm": 1.2763880491256714, + "learning_rate": 0.0001, + "loss": 71.5783, + "step": 1440 + }, + { + "epoch": 23.390243902439025, + "grad_norm": 1.3913089036941528, + "learning_rate": 0.0001, + "loss": 72.5986, + "step": 1450 + }, + { + "epoch": 23.552845528455286, + "grad_norm": 1.628751516342163, + "learning_rate": 0.0001, + "loss": 71.8383, + "step": 1460 + }, + { + "epoch": 23.715447154471544, + "grad_norm": 1.4331274032592773, + "learning_rate": 0.0001, + "loss": 70.8051, + "step": 1470 + }, + { + "epoch": 23.878048780487806, + "grad_norm": 1.4154727458953857, + "learning_rate": 0.0001, + "loss": 71.3283, + "step": 1480 + }, + { + "epoch": 24.03252032520325, + "grad_norm": 1.7208784818649292, + "learning_rate": 0.0001, + "loss": 68.4719, + "step": 1490 + }, + { + "epoch": 24.195121951219512, + "grad_norm": 1.466123342514038, + "learning_rate": 0.0001, + "loss": 71.9597, + "step": 1500 + }, + { + "epoch": 24.357723577235774, + "grad_norm": 1.4823886156082153, + "learning_rate": 0.0001, + "loss": 72.4533, + "step": 1510 + }, + { + "epoch": 24.520325203252032, + "grad_norm": 1.8407355546951294, + "learning_rate": 0.0001, + "loss": 71.1468, + "step": 1520 + }, + { + "epoch": 24.682926829268293, + "grad_norm": 1.630565881729126, + "learning_rate": 0.0001, + "loss": 71.5192, + "step": 1530 + }, + { + "epoch": 24.84552845528455, + "grad_norm": 1.5248785018920898, + "learning_rate": 0.0001, + "loss": 68.4804, + "step": 1540 + }, + { + "epoch": 25.0, + "grad_norm": 1.4690219163894653, + "learning_rate": 0.0001, + "loss": 67.5716, + "step": 1550 + }, + { + "epoch": 25.16260162601626, + "grad_norm": 2.0246646404266357, + "learning_rate": 0.0001, + "loss": 71.2554, + "step": 1560 + }, + { + "epoch": 25.32520325203252, + "grad_norm": 1.3611458539962769, + "learning_rate": 0.0001, + "loss": 69.8558, + "step": 1570 + }, + { + "epoch": 25.48780487804878, + "grad_norm": 2.0006039142608643, + "learning_rate": 0.0001, + "loss": 72.5344, + "step": 1580 + }, + { + "epoch": 25.65040650406504, + "grad_norm": 1.7373167276382446, + "learning_rate": 0.0001, + "loss": 70.6253, + "step": 1590 + }, + { + "epoch": 25.8130081300813, + "grad_norm": 1.2878378629684448, + "learning_rate": 0.0001, + "loss": 72.165, + "step": 1600 + }, + { + "epoch": 25.975609756097562, + "grad_norm": 1.5337013006210327, + "learning_rate": 0.0001, + "loss": 70.6989, + "step": 1610 + }, + { + "epoch": 26.130081300813007, + "grad_norm": 1.364615559577942, + "learning_rate": 0.0001, + "loss": 67.3898, + "step": 1620 + }, + { + "epoch": 26.29268292682927, + "grad_norm": 1.3249316215515137, + "learning_rate": 0.0001, + "loss": 72.7102, + "step": 1630 + }, + { + "epoch": 26.45528455284553, + "grad_norm": 4.209898471832275, + "learning_rate": 0.0001, + "loss": 70.3185, + "step": 1640 + }, + { + "epoch": 26.617886178861788, + "grad_norm": 1.5488810539245605, + "learning_rate": 0.0001, + "loss": 72.3678, + "step": 1650 + }, + { + "epoch": 26.78048780487805, + "grad_norm": 1.7743597030639648, + "learning_rate": 0.0001, + "loss": 71.0917, + "step": 1660 + }, + { + "epoch": 26.943089430894307, + "grad_norm": 1.5433533191680908, + "learning_rate": 0.0001, + "loss": 71.4001, + "step": 1670 + }, + { + "epoch": 27.097560975609756, + "grad_norm": 1.6320385932922363, + "learning_rate": 0.0001, + "loss": 66.7441, + "step": 1680 + }, + { + "epoch": 27.260162601626018, + "grad_norm": 1.3137969970703125, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 1690 + }, + { + "epoch": 27.422764227642276, + "grad_norm": 1.9150484800338745, + "learning_rate": 0.0001, + "loss": 69.8956, + "step": 1700 + }, + { + "epoch": 27.585365853658537, + "grad_norm": 1.4442802667617798, + "learning_rate": 0.0001, + "loss": 70.7185, + "step": 1710 + }, + { + "epoch": 27.747967479674795, + "grad_norm": 2.572497606277466, + "learning_rate": 0.0001, + "loss": 70.8934, + "step": 1720 + }, + { + "epoch": 27.910569105691057, + "grad_norm": 1.6669617891311646, + "learning_rate": 0.0001, + "loss": 71.8221, + "step": 1730 + }, + { + "epoch": 28.065040650406505, + "grad_norm": 1.8449876308441162, + "learning_rate": 0.0001, + "loss": 66.3923, + "step": 1740 + }, + { + "epoch": 28.227642276422763, + "grad_norm": 1.6600004434585571, + "learning_rate": 0.0001, + "loss": 69.5301, + "step": 1750 + }, + { + "epoch": 28.390243902439025, + "grad_norm": 1.4655731916427612, + "learning_rate": 0.0001, + "loss": 72.1924, + "step": 1760 + }, + { + "epoch": 28.552845528455286, + "grad_norm": 1.134505033493042, + "learning_rate": 0.0001, + "loss": 69.5855, + "step": 1770 + }, + { + "epoch": 28.715447154471544, + "grad_norm": 1.301984190940857, + "learning_rate": 0.0001, + "loss": 70.7999, + "step": 1780 + }, + { + "epoch": 28.878048780487806, + "grad_norm": 1.5641013383865356, + "learning_rate": 0.0001, + "loss": 69.9091, + "step": 1790 + }, + { + "epoch": 29.03252032520325, + "grad_norm": 1.4850797653198242, + "learning_rate": 0.0001, + "loss": 66.6266, + "step": 1800 + }, + { + "epoch": 29.195121951219512, + "grad_norm": 1.2069118022918701, + "learning_rate": 0.0001, + "loss": 69.3802, + "step": 1810 + }, + { + "epoch": 29.357723577235774, + "grad_norm": 1.2890352010726929, + "learning_rate": 0.0001, + "loss": 68.5044, + "step": 1820 + }, + { + "epoch": 29.520325203252032, + "grad_norm": 1.4150917530059814, + "learning_rate": 0.0001, + "loss": 69.807, + "step": 1830 + }, + { + "epoch": 29.682926829268293, + "grad_norm": 1.514807939529419, + "learning_rate": 0.0001, + "loss": 71.3858, + "step": 1840 + }, + { + "epoch": 29.84552845528455, + "grad_norm": 1.4246330261230469, + "learning_rate": 0.0001, + "loss": 70.494, + "step": 1850 + }, + { + "epoch": 30.0, + "grad_norm": 1.1789597272872925, + "learning_rate": 0.0001, + "loss": 67.1056, + "step": 1860 + }, + { + "epoch": 30.16260162601626, + "grad_norm": 1.23678719997406, + "learning_rate": 0.0001, + "loss": 70.336, + "step": 1870 + }, + { + "epoch": 30.32520325203252, + "grad_norm": 1.1756274700164795, + "learning_rate": 0.0001, + "loss": 67.9909, + "step": 1880 + }, + { + "epoch": 30.48780487804878, + "grad_norm": 1.4175690412521362, + "learning_rate": 0.0001, + "loss": 69.2617, + "step": 1890 + }, + { + "epoch": 30.65040650406504, + "grad_norm": 1.3084465265274048, + "learning_rate": 0.0001, + "loss": 70.2963, + "step": 1900 + }, + { + "epoch": 30.8130081300813, + "grad_norm": 1.486464500427246, + "learning_rate": 0.0001, + "loss": 68.6317, + "step": 1910 + }, + { + "epoch": 30.975609756097562, + "grad_norm": 10.415762901306152, + "learning_rate": 0.0001, + "loss": 69.7767, + "step": 1920 + }, + { + "epoch": 31.130081300813007, + "grad_norm": 1.4484038352966309, + "learning_rate": 0.0001, + "loss": 65.6345, + "step": 1930 + }, + { + "epoch": 31.29268292682927, + "grad_norm": 2.297306537628174, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 1940 + }, + { + "epoch": 31.45528455284553, + "grad_norm": 1.223490595817566, + "learning_rate": 0.0001, + "loss": 69.2139, + "step": 1950 + }, + { + "epoch": 31.617886178861788, + "grad_norm": 1.2144125699996948, + "learning_rate": 0.0001, + "loss": 70.3061, + "step": 1960 + }, + { + "epoch": 31.78048780487805, + "grad_norm": 2.633528470993042, + "learning_rate": 0.0001, + "loss": 68.7096, + "step": 1970 + }, + { + "epoch": 31.943089430894307, + "grad_norm": 1.5048350095748901, + "learning_rate": 0.0001, + "loss": 68.6711, + "step": 1980 + }, + { + "epoch": 32.09756097560975, + "grad_norm": 1.5154467821121216, + "learning_rate": 0.0001, + "loss": 66.7989, + "step": 1990 + }, + { + "epoch": 32.260162601626014, + "grad_norm": 2.721277952194214, + "learning_rate": 0.0001, + "loss": 70.4987, + "step": 2000 + }, + { + "epoch": 32.422764227642276, + "grad_norm": 1.44464910030365, + "learning_rate": 0.0001, + "loss": 69.9712, + "step": 2010 + }, + { + "epoch": 32.58536585365854, + "grad_norm": 1.3462835550308228, + "learning_rate": 0.0001, + "loss": 68.9381, + "step": 2020 + }, + { + "epoch": 32.7479674796748, + "grad_norm": 1.3283029794692993, + "learning_rate": 0.0001, + "loss": 68.5301, + "step": 2030 + }, + { + "epoch": 32.91056910569106, + "grad_norm": 1.5172960758209229, + "learning_rate": 0.0001, + "loss": 70.1894, + "step": 2040 + }, + { + "epoch": 33.0650406504065, + "grad_norm": 1.1872576475143433, + "learning_rate": 0.0001, + "loss": 65.0158, + "step": 2050 + }, + { + "epoch": 33.22764227642276, + "grad_norm": 1.5414419174194336, + "learning_rate": 0.0001, + "loss": 69.1537, + "step": 2060 + }, + { + "epoch": 33.390243902439025, + "grad_norm": 1.3143430948257446, + "learning_rate": 0.0001, + "loss": 68.6792, + "step": 2070 + }, + { + "epoch": 33.552845528455286, + "grad_norm": 1.3863723278045654, + "learning_rate": 0.0001, + "loss": 70.3867, + "step": 2080 + }, + { + "epoch": 33.71544715447155, + "grad_norm": 1.4060479402542114, + "learning_rate": 0.0001, + "loss": 68.4981, + "step": 2090 + }, + { + "epoch": 33.8780487804878, + "grad_norm": 1.34132981300354, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 2100 + }, + { + "epoch": 34.03252032520325, + "grad_norm": 1.8303673267364502, + "learning_rate": 0.0001, + "loss": 65.819, + "step": 2110 + }, + { + "epoch": 34.19512195121951, + "grad_norm": 1.387339472770691, + "learning_rate": 0.0001, + "loss": 69.9825, + "step": 2120 + }, + { + "epoch": 34.357723577235774, + "grad_norm": 1.4125195741653442, + "learning_rate": 0.0001, + "loss": 67.5149, + "step": 2130 + }, + { + "epoch": 34.520325203252035, + "grad_norm": 1.2909412384033203, + "learning_rate": 0.0001, + "loss": 69.7848, + "step": 2140 + }, + { + "epoch": 34.68292682926829, + "grad_norm": 1.8860795497894287, + "learning_rate": 0.0001, + "loss": 68.6145, + "step": 2150 + }, + { + "epoch": 34.84552845528455, + "grad_norm": 1.5111079216003418, + "learning_rate": 0.0001, + "loss": 68.2612, + "step": 2160 + }, + { + "epoch": 35.0, + "grad_norm": 1.3275264501571655, + "learning_rate": 0.0001, + "loss": 64.0298, + "step": 2170 + }, + { + "epoch": 35.16260162601626, + "grad_norm": 1.174864411354065, + "learning_rate": 0.0001, + "loss": 66.6153, + "step": 2180 + }, + { + "epoch": 35.32520325203252, + "grad_norm": 1.6873915195465088, + "learning_rate": 0.0001, + "loss": 69.6278, + "step": 2190 + }, + { + "epoch": 35.48780487804878, + "grad_norm": 2.4457507133483887, + "learning_rate": 0.0001, + "loss": 68.3611, + "step": 2200 + }, + { + "epoch": 35.65040650406504, + "grad_norm": 1.4779958724975586, + "learning_rate": 0.0001, + "loss": 69.2609, + "step": 2210 + }, + { + "epoch": 35.8130081300813, + "grad_norm": 2.273709774017334, + "learning_rate": 0.0001, + "loss": 69.1312, + "step": 2220 + }, + { + "epoch": 35.97560975609756, + "grad_norm": 1.5741862058639526, + "learning_rate": 0.0001, + "loss": 69.1191, + "step": 2230 + }, + { + "epoch": 36.13008130081301, + "grad_norm": 1.2730036973953247, + "learning_rate": 0.0001, + "loss": 65.692, + "step": 2240 + }, + { + "epoch": 36.292682926829265, + "grad_norm": 1.2936429977416992, + "learning_rate": 0.0001, + "loss": 68.2962, + "step": 2250 + }, + { + "epoch": 36.45528455284553, + "grad_norm": 1.354279637336731, + "learning_rate": 0.0001, + "loss": 70.5507, + "step": 2260 + }, + { + "epoch": 36.61788617886179, + "grad_norm": 1.4165853261947632, + "learning_rate": 0.0001, + "loss": 68.0048, + "step": 2270 + }, + { + "epoch": 36.78048780487805, + "grad_norm": 1.2992445230484009, + "learning_rate": 0.0001, + "loss": 67.7072, + "step": 2280 + }, + { + "epoch": 36.94308943089431, + "grad_norm": 1.5053461790084839, + "learning_rate": 0.0001, + "loss": 66.7587, + "step": 2290 + }, + { + "epoch": 37.09756097560975, + "grad_norm": 1.337390661239624, + "learning_rate": 0.0001, + "loss": 63.786, + "step": 2300 + }, + { + "epoch": 37.260162601626014, + "grad_norm": 1.0407774448394775, + "learning_rate": 0.0001, + "loss": 66.6253, + "step": 2310 + }, + { + "epoch": 37.422764227642276, + "grad_norm": 1.564152479171753, + "learning_rate": 0.0001, + "loss": 68.3606, + "step": 2320 + }, + { + "epoch": 37.58536585365854, + "grad_norm": 1.2028454542160034, + "learning_rate": 0.0001, + "loss": 66.2841, + "step": 2330 + }, + { + "epoch": 37.7479674796748, + "grad_norm": 1.4528603553771973, + "learning_rate": 0.0001, + "loss": 67.4277, + "step": 2340 + }, + { + "epoch": 37.91056910569106, + "grad_norm": 1.4954335689544678, + "learning_rate": 0.0001, + "loss": 68.9954, + "step": 2350 + }, + { + "epoch": 38.0650406504065, + "grad_norm": 1.2820416688919067, + "learning_rate": 0.0001, + "loss": 65.2449, + "step": 2360 + }, + { + "epoch": 38.22764227642276, + "grad_norm": 1.2392579317092896, + "learning_rate": 0.0001, + "loss": 67.5006, + "step": 2370 + }, + { + "epoch": 38.390243902439025, + "grad_norm": 1.2135238647460938, + "learning_rate": 0.0001, + "loss": 69.2214, + "step": 2380 + }, + { + "epoch": 38.552845528455286, + "grad_norm": 1.5964744091033936, + "learning_rate": 0.0001, + "loss": 67.934, + "step": 2390 + }, + { + "epoch": 38.71544715447155, + "grad_norm": 1.2841702699661255, + "learning_rate": 0.0001, + "loss": 67.0848, + "step": 2400 + }, + { + "epoch": 38.8780487804878, + "grad_norm": 1.8026081323623657, + "learning_rate": 0.0001, + "loss": 67.5358, + "step": 2410 + }, + { + "epoch": 39.03252032520325, + "grad_norm": 1.4178807735443115, + "learning_rate": 0.0001, + "loss": 64.771, + "step": 2420 + }, + { + "epoch": 39.19512195121951, + "grad_norm": 1.3671854734420776, + "learning_rate": 0.0001, + "loss": 68.0531, + "step": 2430 + }, + { + "epoch": 39.357723577235774, + "grad_norm": 1.43596613407135, + "learning_rate": 0.0001, + "loss": 67.7735, + "step": 2440 + }, + { + "epoch": 39.520325203252035, + "grad_norm": 1.464250922203064, + "learning_rate": 0.0001, + "loss": 67.8331, + "step": 2450 + }, + { + "epoch": 39.68292682926829, + "grad_norm": 3.101461410522461, + "learning_rate": 0.0001, + "loss": 65.8956, + "step": 2460 + }, + { + "epoch": 39.84552845528455, + "grad_norm": 1.8332289457321167, + "learning_rate": 0.0001, + "loss": 66.8218, + "step": 2470 + }, + { + "epoch": 40.0, + "grad_norm": 1.2660984992980957, + "learning_rate": 0.0001, + "loss": 63.9969, + "step": 2480 + }, + { + "epoch": 40.16260162601626, + "grad_norm": 2.0309898853302, + "learning_rate": 0.0001, + "loss": 67.3538, + "step": 2490 + }, + { + "epoch": 40.32520325203252, + "grad_norm": 1.207000494003296, + "learning_rate": 0.0001, + "loss": 68.0898, + "step": 2500 + }, + { + "epoch": 40.48780487804878, + "grad_norm": 1.3609018325805664, + "learning_rate": 0.0001, + "loss": 67.0381, + "step": 2510 + }, + { + "epoch": 40.65040650406504, + "grad_norm": 1.310826301574707, + "learning_rate": 0.0001, + "loss": 69.0267, + "step": 2520 + }, + { + "epoch": 40.8130081300813, + "grad_norm": 1.4150960445404053, + "learning_rate": 0.0001, + "loss": 69.1934, + "step": 2530 + }, + { + "epoch": 40.97560975609756, + "grad_norm": 1.5414319038391113, + "learning_rate": 0.0001, + "loss": 68.0604, + "step": 2540 + }, + { + "epoch": 41.13008130081301, + "grad_norm": 2.3246140480041504, + "learning_rate": 0.0001, + "loss": 61.5698, + "step": 2550 + }, + { + "epoch": 41.292682926829265, + "grad_norm": 1.206296443939209, + "learning_rate": 0.0001, + "loss": 64.9898, + "step": 2560 + }, + { + "epoch": 41.45528455284553, + "grad_norm": 1.6080626249313354, + "learning_rate": 0.0001, + "loss": 68.2685, + "step": 2570 + }, + { + "epoch": 41.61788617886179, + "grad_norm": 1.2404112815856934, + "learning_rate": 0.0001, + "loss": 67.8457, + "step": 2580 + }, + { + "epoch": 41.78048780487805, + "grad_norm": 1.2304514646530151, + "learning_rate": 0.0001, + "loss": 67.9052, + "step": 2590 + }, + { + "epoch": 41.94308943089431, + "grad_norm": 1.5806187391281128, + "learning_rate": 0.0001, + "loss": 66.0663, + "step": 2600 + }, + { + "epoch": 42.09756097560975, + "grad_norm": 1.2564151287078857, + "learning_rate": 0.0001, + "loss": 65.7763, + "step": 2610 + }, + { + "epoch": 42.260162601626014, + "grad_norm": 1.4972114562988281, + "learning_rate": 0.0001, + "loss": 66.4269, + "step": 2620 + }, + { + "epoch": 42.422764227642276, + "grad_norm": 1.2913843393325806, + "learning_rate": 0.0001, + "loss": 67.7554, + "step": 2630 + }, + { + "epoch": 42.58536585365854, + "grad_norm": 1.3743863105773926, + "learning_rate": 0.0001, + "loss": 67.2627, + "step": 2640 + }, + { + "epoch": 42.7479674796748, + "grad_norm": 1.1267883777618408, + "learning_rate": 0.0001, + "loss": 67.4597, + "step": 2650 + }, + { + "epoch": 42.91056910569106, + "grad_norm": 2.0194661617279053, + "learning_rate": 0.0001, + "loss": 66.9295, + "step": 2660 + }, + { + "epoch": 43.0650406504065, + "grad_norm": 1.273779273033142, + "learning_rate": 0.0001, + "loss": 60.7906, + "step": 2670 + }, + { + "epoch": 43.22764227642276, + "grad_norm": 1.1498609781265259, + "learning_rate": 0.0001, + "loss": 68.0405, + "step": 2680 + }, + { + "epoch": 43.390243902439025, + "grad_norm": 1.4689130783081055, + "learning_rate": 0.0001, + "loss": 66.2358, + "step": 2690 + }, + { + "epoch": 43.552845528455286, + "grad_norm": 1.4495564699172974, + "learning_rate": 0.0001, + "loss": 67.4267, + "step": 2700 + }, + { + "epoch": 43.71544715447155, + "grad_norm": 2.5769612789154053, + "learning_rate": 0.0001, + "loss": 66.953, + "step": 2710 + }, + { + "epoch": 43.8780487804878, + "grad_norm": 2.1684205532073975, + "learning_rate": 0.0001, + "loss": 65.6393, + "step": 2720 + }, + { + "epoch": 44.03252032520325, + "grad_norm": 1.4968472719192505, + "learning_rate": 0.0001, + "loss": 60.8256, + "step": 2730 + }, + { + "epoch": 44.19512195121951, + "grad_norm": 1.1874353885650635, + "learning_rate": 0.0001, + "loss": 66.818, + "step": 2740 + }, + { + "epoch": 44.357723577235774, + "grad_norm": 2.271235227584839, + "learning_rate": 0.0001, + "loss": 66.5048, + "step": 2750 + }, + { + "epoch": 44.520325203252035, + "grad_norm": 1.20489501953125, + "learning_rate": 0.0001, + "loss": 67.0487, + "step": 2760 + }, + { + "epoch": 44.68292682926829, + "grad_norm": 1.6273691654205322, + "learning_rate": 0.0001, + "loss": 66.2228, + "step": 2770 + }, + { + "epoch": 44.84552845528455, + "grad_norm": 1.4935433864593506, + "learning_rate": 0.0001, + "loss": 67.2039, + "step": 2780 + }, + { + "epoch": 45.0, + "grad_norm": 1.2668659687042236, + "learning_rate": 0.0001, + "loss": 62.7855, + "step": 2790 + }, + { + "epoch": 45.16260162601626, + "grad_norm": 1.6439573764801025, + "learning_rate": 0.0001, + "loss": 67.919, + "step": 2800 + }, + { + "epoch": 45.32520325203252, + "grad_norm": 1.311854600906372, + "learning_rate": 0.0001, + "loss": 65.1797, + "step": 2810 + }, + { + "epoch": 45.48780487804878, + "grad_norm": 1.3596776723861694, + "learning_rate": 0.0001, + "loss": 67.469, + "step": 2820 + }, + { + "epoch": 45.65040650406504, + "grad_norm": 1.5618443489074707, + "learning_rate": 0.0001, + "loss": 66.1808, + "step": 2830 + }, + { + "epoch": 45.8130081300813, + "grad_norm": 1.7551205158233643, + "learning_rate": 0.0001, + "loss": 65.3803, + "step": 2840 + }, + { + "epoch": 45.97560975609756, + "grad_norm": 1.433794379234314, + "learning_rate": 0.0001, + "loss": 67.8314, + "step": 2850 + }, + { + "epoch": 46.13008130081301, + "grad_norm": 1.4634851217269897, + "learning_rate": 0.0001, + "loss": 63.4219, + "step": 2860 + }, + { + "epoch": 46.292682926829265, + "grad_norm": 1.083052396774292, + "learning_rate": 0.0001, + "loss": 68.0719, + "step": 2870 + }, + { + "epoch": 46.45528455284553, + "grad_norm": 1.1784703731536865, + "learning_rate": 0.0001, + "loss": 66.1391, + "step": 2880 + }, + { + "epoch": 46.61788617886179, + "grad_norm": 1.2552465200424194, + "learning_rate": 0.0001, + "loss": 65.6006, + "step": 2890 + }, + { + "epoch": 46.78048780487805, + "grad_norm": 1.4684522151947021, + "learning_rate": 0.0001, + "loss": 66.8314, + "step": 2900 + }, + { + "epoch": 46.94308943089431, + "grad_norm": 1.741514801979065, + "learning_rate": 0.0001, + "loss": 67.5056, + "step": 2910 + }, + { + "epoch": 47.09756097560975, + "grad_norm": 5.696534156799316, + "learning_rate": 0.0001, + "loss": 62.8016, + "step": 2920 + }, + { + "epoch": 47.260162601626014, + "grad_norm": 1.5000969171524048, + "learning_rate": 0.0001, + "loss": 68.0272, + "step": 2930 + }, + { + "epoch": 47.422764227642276, + "grad_norm": 1.1044933795928955, + "learning_rate": 0.0001, + "loss": 67.0905, + "step": 2940 + }, + { + "epoch": 47.58536585365854, + "grad_norm": 0.9933589100837708, + "learning_rate": 0.0001, + "loss": 67.6077, + "step": 2950 + }, + { + "epoch": 47.7479674796748, + "grad_norm": 1.2908599376678467, + "learning_rate": 0.0001, + "loss": 65.869, + "step": 2960 + }, + { + "epoch": 47.91056910569106, + "grad_norm": 1.0470529794692993, + "learning_rate": 0.0001, + "loss": 66.5353, + "step": 2970 + }, + { + "epoch": 48.0650406504065, + "grad_norm": 1.0927976369857788, + "learning_rate": 0.0001, + "loss": 62.7541, + "step": 2980 + }, + { + "epoch": 48.22764227642276, + "grad_norm": 1.0870513916015625, + "learning_rate": 0.0001, + "loss": 64.172, + "step": 2990 + }, + { + "epoch": 48.390243902439025, + "grad_norm": 1.3492090702056885, + "learning_rate": 0.0001, + "loss": 67.6642, + "step": 3000 + }, + { + "epoch": 48.552845528455286, + "grad_norm": 1.4024657011032104, + "learning_rate": 0.0001, + "loss": 65.2722, + "step": 3010 + }, + { + "epoch": 48.71544715447155, + "grad_norm": 1.1165252923965454, + "learning_rate": 0.0001, + "loss": 66.7233, + "step": 3020 + }, + { + "epoch": 48.8780487804878, + "grad_norm": 1.2147928476333618, + "learning_rate": 0.0001, + "loss": 65.7277, + "step": 3030 + }, + { + "epoch": 49.03252032520325, + "grad_norm": 0.925310492515564, + "learning_rate": 0.0001, + "loss": 62.9256, + "step": 3040 + }, + { + "epoch": 49.19512195121951, + "grad_norm": 1.2353442907333374, + "learning_rate": 0.0001, + "loss": 66.1218, + "step": 3050 + }, + { + "epoch": 49.357723577235774, + "grad_norm": 1.2177367210388184, + "learning_rate": 0.0001, + "loss": 66.7978, + "step": 3060 + }, + { + "epoch": 49.520325203252035, + "grad_norm": 1.248044729232788, + "learning_rate": 0.0001, + "loss": 66.2132, + "step": 3070 + }, + { + "epoch": 49.68292682926829, + "grad_norm": 1.366050124168396, + "learning_rate": 0.0001, + "loss": 67.0118, + "step": 3080 + }, + { + "epoch": 49.84552845528455, + "grad_norm": 1.3701151609420776, + "learning_rate": 0.0001, + "loss": 67.6099, + "step": 3090 + }, + { + "epoch": 50.0, + "grad_norm": 1.0420141220092773, + "learning_rate": 0.0001, + "loss": 61.5289, + "step": 3100 + }, + { + "epoch": 50.16260162601626, + "grad_norm": 1.0957746505737305, + "learning_rate": 0.0001, + "loss": 66.7348, + "step": 3110 + }, + { + "epoch": 50.32520325203252, + "grad_norm": 1.108628749847412, + "learning_rate": 0.0001, + "loss": 66.6199, + "step": 3120 + }, + { + "epoch": 50.48780487804878, + "grad_norm": 1.3680156469345093, + "learning_rate": 0.0001, + "loss": 67.0825, + "step": 3130 + }, + { + "epoch": 50.65040650406504, + "grad_norm": 1.169851303100586, + "learning_rate": 0.0001, + "loss": 65.4901, + "step": 3140 + }, + { + "epoch": 50.8130081300813, + "grad_norm": 2.1822028160095215, + "learning_rate": 0.0001, + "loss": 65.6841, + "step": 3150 + }, + { + "epoch": 50.97560975609756, + "grad_norm": 1.053402304649353, + "learning_rate": 0.0001, + "loss": 66.3119, + "step": 3160 + }, + { + "epoch": 51.13008130081301, + "grad_norm": 1.0627789497375488, + "learning_rate": 0.0001, + "loss": 61.3642, + "step": 3170 + }, + { + "epoch": 51.292682926829265, + "grad_norm": 1.926291823387146, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 3180 + }, + { + "epoch": 51.45528455284553, + "grad_norm": 1.5791983604431152, + "learning_rate": 0.0001, + "loss": 63.913, + "step": 3190 + }, + { + "epoch": 51.61788617886179, + "grad_norm": 1.210265040397644, + "learning_rate": 0.0001, + "loss": 64.5261, + "step": 3200 + }, + { + "epoch": 51.78048780487805, + "grad_norm": 1.0720808506011963, + "learning_rate": 0.0001, + "loss": 67.0194, + "step": 3210 + }, + { + "epoch": 51.94308943089431, + "grad_norm": 1.94563889503479, + "learning_rate": 0.0001, + "loss": 68.4662, + "step": 3220 + }, + { + "epoch": 52.09756097560975, + "grad_norm": 1.598980188369751, + "learning_rate": 0.0001, + "loss": 62.4946, + "step": 3230 + }, + { + "epoch": 52.260162601626014, + "grad_norm": 1.0564576387405396, + "learning_rate": 0.0001, + "loss": 65.7912, + "step": 3240 + }, + { + "epoch": 52.422764227642276, + "grad_norm": 1.2189645767211914, + "learning_rate": 0.0001, + "loss": 65.5772, + "step": 3250 + }, + { + "epoch": 52.58536585365854, + "grad_norm": 1.1357049942016602, + "learning_rate": 0.0001, + "loss": 65.1559, + "step": 3260 + }, + { + "epoch": 52.7479674796748, + "grad_norm": 1.316867709159851, + "learning_rate": 0.0001, + "loss": 64.7897, + "step": 3270 + }, + { + "epoch": 52.91056910569106, + "grad_norm": 1.3870197534561157, + "learning_rate": 0.0001, + "loss": 65.5195, + "step": 3280 + }, + { + "epoch": 53.0650406504065, + "grad_norm": 1.2467927932739258, + "learning_rate": 0.0001, + "loss": 61.3392, + "step": 3290 + }, + { + "epoch": 53.22764227642276, + "grad_norm": 1.1661109924316406, + "learning_rate": 0.0001, + "loss": 66.0117, + "step": 3300 + }, + { + "epoch": 53.390243902439025, + "grad_norm": 1.3509628772735596, + "learning_rate": 0.0001, + "loss": 64.7228, + "step": 3310 + }, + { + "epoch": 53.552845528455286, + "grad_norm": 1.1093642711639404, + "learning_rate": 0.0001, + "loss": 65.2534, + "step": 3320 + }, + { + "epoch": 53.71544715447155, + "grad_norm": 1.1801207065582275, + "learning_rate": 0.0001, + "loss": 65.7097, + "step": 3330 + }, + { + "epoch": 53.8780487804878, + "grad_norm": 1.175884485244751, + "learning_rate": 0.0001, + "loss": 64.4368, + "step": 3340 + }, + { + "epoch": 54.03252032520325, + "grad_norm": 1.1727889776229858, + "learning_rate": 0.0001, + "loss": 62.9373, + "step": 3350 + }, + { + "epoch": 54.19512195121951, + "grad_norm": 1.5330945253372192, + "learning_rate": 0.0001, + "loss": 66.7763, + "step": 3360 + }, + { + "epoch": 54.357723577235774, + "grad_norm": 1.2992504835128784, + "learning_rate": 0.0001, + "loss": 65.7854, + "step": 3370 + }, + { + "epoch": 54.520325203252035, + "grad_norm": 1.0185420513153076, + "learning_rate": 0.0001, + "loss": 65.7046, + "step": 3380 + }, + { + "epoch": 54.68292682926829, + "grad_norm": 1.426229476928711, + "learning_rate": 0.0001, + "loss": 64.0286, + "step": 3390 + }, + { + "epoch": 54.84552845528455, + "grad_norm": 1.2600868940353394, + "learning_rate": 0.0001, + "loss": 66.1738, + "step": 3400 + }, + { + "epoch": 55.0, + "grad_norm": 1.917455792427063, + "learning_rate": 0.0001, + "loss": 61.9381, + "step": 3410 + }, + { + "epoch": 55.16260162601626, + "grad_norm": 1.2000917196273804, + "learning_rate": 0.0001, + "loss": 65.5626, + "step": 3420 + }, + { + "epoch": 55.32520325203252, + "grad_norm": 1.362176775932312, + "learning_rate": 0.0001, + "loss": 66.8012, + "step": 3430 + }, + { + "epoch": 55.48780487804878, + "grad_norm": 1.1611402034759521, + "learning_rate": 0.0001, + "loss": 64.5227, + "step": 3440 + }, + { + "epoch": 55.65040650406504, + "grad_norm": 1.2066621780395508, + "learning_rate": 0.0001, + "loss": 63.4351, + "step": 3450 + }, + { + "epoch": 55.8130081300813, + "grad_norm": 1.005340814590454, + "learning_rate": 0.0001, + "loss": 65.9261, + "step": 3460 + }, + { + "epoch": 55.97560975609756, + "grad_norm": 1.0906637907028198, + "learning_rate": 0.0001, + "loss": 64.6635, + "step": 3470 + }, + { + "epoch": 56.13008130081301, + "grad_norm": 1.0608748197555542, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 3480 + }, + { + "epoch": 56.292682926829265, + "grad_norm": 1.025319218635559, + "learning_rate": 0.0001, + "loss": 64.7783, + "step": 3490 + }, + { + "epoch": 56.45528455284553, + "grad_norm": 1.1596784591674805, + "learning_rate": 0.0001, + "loss": 66.4629, + "step": 3500 + }, + { + "epoch": 56.61788617886179, + "grad_norm": 0.9318640232086182, + "learning_rate": 0.0001, + "loss": 65.5533, + "step": 3510 + }, + { + "epoch": 56.78048780487805, + "grad_norm": 1.0977418422698975, + "learning_rate": 0.0001, + "loss": 63.647, + "step": 3520 + }, + { + "epoch": 56.94308943089431, + "grad_norm": 1.3912436962127686, + "learning_rate": 0.0001, + "loss": 62.8273, + "step": 3530 + }, + { + "epoch": 57.09756097560975, + "grad_norm": 1.0921591520309448, + "learning_rate": 0.0001, + "loss": 61.5968, + "step": 3540 + }, + { + "epoch": 57.260162601626014, + "grad_norm": 1.17819344997406, + "learning_rate": 0.0001, + "loss": 64.9381, + "step": 3550 + }, + { + "epoch": 57.422764227642276, + "grad_norm": 1.188829779624939, + "learning_rate": 0.0001, + "loss": 64.6498, + "step": 3560 + }, + { + "epoch": 57.58536585365854, + "grad_norm": 3.664839506149292, + "learning_rate": 0.0001, + "loss": 65.3834, + "step": 3570 + }, + { + "epoch": 57.7479674796748, + "grad_norm": 1.57958984375, + "learning_rate": 0.0001, + "loss": 63.8808, + "step": 3580 + }, + { + "epoch": 57.91056910569106, + "grad_norm": 1.1837552785873413, + "learning_rate": 0.0001, + "loss": 64.1545, + "step": 3590 + }, + { + "epoch": 58.0650406504065, + "grad_norm": 1.5986013412475586, + "learning_rate": 0.0001, + "loss": 60.5742, + "step": 3600 + }, + { + "epoch": 58.22764227642276, + "grad_norm": 1.1551810503005981, + "learning_rate": 0.0001, + "loss": 66.2378, + "step": 3610 + }, + { + "epoch": 58.390243902439025, + "grad_norm": 1.0869863033294678, + "learning_rate": 0.0001, + "loss": 65.3034, + "step": 3620 + }, + { + "epoch": 58.552845528455286, + "grad_norm": 1.2263212203979492, + "learning_rate": 0.0001, + "loss": 63.976, + "step": 3630 + }, + { + "epoch": 58.71544715447155, + "grad_norm": 1.4410817623138428, + "learning_rate": 0.0001, + "loss": 63.4778, + "step": 3640 + }, + { + "epoch": 58.8780487804878, + "grad_norm": 1.9633759260177612, + "learning_rate": 0.0001, + "loss": 65.5946, + "step": 3650 + }, + { + "epoch": 59.03252032520325, + "grad_norm": 1.1729766130447388, + "learning_rate": 0.0001, + "loss": 62.2171, + "step": 3660 + }, + { + "epoch": 59.19512195121951, + "grad_norm": 1.0049556493759155, + "learning_rate": 0.0001, + "loss": 65.1854, + "step": 3670 + }, + { + "epoch": 59.357723577235774, + "grad_norm": 1.1727055311203003, + "learning_rate": 0.0001, + "loss": 65.4839, + "step": 3680 + }, + { + "epoch": 59.520325203252035, + "grad_norm": 1.5394458770751953, + "learning_rate": 0.0001, + "loss": 64.9341, + "step": 3690 + }, + { + "epoch": 59.68292682926829, + "grad_norm": 1.1583774089813232, + "learning_rate": 0.0001, + "loss": 63.9197, + "step": 3700 + }, + { + "epoch": 59.84552845528455, + "grad_norm": 1.0835607051849365, + "learning_rate": 0.0001, + "loss": 64.0494, + "step": 3710 + }, + { + "epoch": 60.0, + "grad_norm": 1.7504246234893799, + "learning_rate": 0.0001, + "loss": 59.4134, + "step": 3720 + }, + { + "epoch": 60.16260162601626, + "grad_norm": 1.1870343685150146, + "learning_rate": 0.0001, + "loss": 63.2158, + "step": 3730 + }, + { + "epoch": 60.32520325203252, + "grad_norm": 1.1983181238174438, + "learning_rate": 0.0001, + "loss": 64.7144, + "step": 3740 + }, + { + "epoch": 60.48780487804878, + "grad_norm": 1.3014581203460693, + "learning_rate": 0.0001, + "loss": 64.6219, + "step": 3750 + }, + { + "epoch": 60.65040650406504, + "grad_norm": 1.4954087734222412, + "learning_rate": 0.0001, + "loss": 63.3785, + "step": 3760 + }, + { + "epoch": 60.8130081300813, + "grad_norm": 1.2236649990081787, + "learning_rate": 0.0001, + "loss": 64.6933, + "step": 3770 + }, + { + "epoch": 60.97560975609756, + "grad_norm": 1.0796610116958618, + "learning_rate": 0.0001, + "loss": 63.2704, + "step": 3780 + }, + { + "epoch": 61.13008130081301, + "grad_norm": 0.9969968795776367, + "learning_rate": 0.0001, + "loss": 62.1838, + "step": 3790 + }, + { + "epoch": 61.292682926829265, + "grad_norm": 1.1624361276626587, + "learning_rate": 0.0001, + "loss": 63.6249, + "step": 3800 + }, + { + "epoch": 61.45528455284553, + "grad_norm": 1.3849223852157593, + "learning_rate": 0.0001, + "loss": 64.5321, + "step": 3810 + }, + { + "epoch": 61.61788617886179, + "grad_norm": 1.6607458591461182, + "learning_rate": 0.0001, + "loss": 64.651, + "step": 3820 + }, + { + "epoch": 61.78048780487805, + "grad_norm": 1.09684419631958, + "learning_rate": 0.0001, + "loss": 64.1129, + "step": 3830 + }, + { + "epoch": 61.94308943089431, + "grad_norm": 1.406246304512024, + "learning_rate": 0.0001, + "loss": 64.3642, + "step": 3840 + }, + { + "epoch": 62.09756097560975, + "grad_norm": 0.9850211143493652, + "learning_rate": 0.0001, + "loss": 60.8703, + "step": 3850 + }, + { + "epoch": 62.260162601626014, + "grad_norm": 1.001344084739685, + "learning_rate": 0.0001, + "loss": 67.5523, + "step": 3860 + }, + { + "epoch": 62.422764227642276, + "grad_norm": 2.435969591140747, + "learning_rate": 0.0001, + "loss": 62.7363, + "step": 3870 + }, + { + "epoch": 62.58536585365854, + "grad_norm": 1.0675824880599976, + "learning_rate": 0.0001, + "loss": 64.8513, + "step": 3880 + }, + { + "epoch": 62.7479674796748, + "grad_norm": 0.9793754816055298, + "learning_rate": 0.0001, + "loss": 64.9904, + "step": 3890 + }, + { + "epoch": 62.91056910569106, + "grad_norm": 1.2187775373458862, + "learning_rate": 0.0001, + "loss": 63.2821, + "step": 3900 + }, + { + "epoch": 63.0650406504065, + "grad_norm": 1.130772590637207, + "learning_rate": 0.0001, + "loss": 61.5957, + "step": 3910 + }, + { + "epoch": 63.22764227642276, + "grad_norm": 1.0573055744171143, + "learning_rate": 0.0001, + "loss": 63.6337, + "step": 3920 + }, + { + "epoch": 63.390243902439025, + "grad_norm": 2.055490255355835, + "learning_rate": 0.0001, + "loss": 63.9598, + "step": 3930 + }, + { + "epoch": 63.552845528455286, + "grad_norm": 1.0780516862869263, + "learning_rate": 0.0001, + "loss": 62.9303, + "step": 3940 + }, + { + "epoch": 63.71544715447155, + "grad_norm": 1.8136590719223022, + "learning_rate": 0.0001, + "loss": 64.4588, + "step": 3950 + }, + { + "epoch": 63.8780487804878, + "grad_norm": 1.0525016784667969, + "learning_rate": 0.0001, + "loss": 64.192, + "step": 3960 + }, + { + "epoch": 64.03252032520325, + "grad_norm": 1.12627375125885, + "learning_rate": 0.0001, + "loss": 59.1823, + "step": 3970 + }, + { + "epoch": 64.1951219512195, + "grad_norm": 1.2558730840682983, + "learning_rate": 0.0001, + "loss": 65.126, + "step": 3980 + }, + { + "epoch": 64.35772357723577, + "grad_norm": 1.2304726839065552, + "learning_rate": 0.0001, + "loss": 62.0868, + "step": 3990 + }, + { + "epoch": 64.52032520325203, + "grad_norm": 1.4179201126098633, + "learning_rate": 0.0001, + "loss": 62.3114, + "step": 4000 + }, + { + "epoch": 64.6829268292683, + "grad_norm": 1.125758409500122, + "learning_rate": 0.0001, + "loss": 63.4103, + "step": 4010 + }, + { + "epoch": 64.84552845528455, + "grad_norm": 2.321124315261841, + "learning_rate": 0.0001, + "loss": 62.9032, + "step": 4020 + }, + { + "epoch": 65.0, + "grad_norm": 1.4433993101119995, + "learning_rate": 0.0001, + "loss": 61.8766, + "step": 4030 + }, + { + "epoch": 65.16260162601625, + "grad_norm": 1.157878041267395, + "learning_rate": 0.0001, + "loss": 62.4833, + "step": 4040 + }, + { + "epoch": 65.32520325203252, + "grad_norm": 1.4127166271209717, + "learning_rate": 0.0001, + "loss": 64.768, + "step": 4050 + }, + { + "epoch": 65.48780487804878, + "grad_norm": 1.2575284242630005, + "learning_rate": 0.0001, + "loss": 62.9401, + "step": 4060 + }, + { + "epoch": 65.65040650406505, + "grad_norm": 1.0957911014556885, + "learning_rate": 0.0001, + "loss": 64.6061, + "step": 4070 + }, + { + "epoch": 65.8130081300813, + "grad_norm": 1.1773414611816406, + "learning_rate": 0.0001, + "loss": 63.9469, + "step": 4080 + }, + { + "epoch": 65.97560975609755, + "grad_norm": 2.1189942359924316, + "learning_rate": 0.0001, + "loss": 65.1422, + "step": 4090 + }, + { + "epoch": 66.130081300813, + "grad_norm": 1.1064234972000122, + "learning_rate": 0.0001, + "loss": 59.6108, + "step": 4100 + }, + { + "epoch": 66.29268292682927, + "grad_norm": 1.1135871410369873, + "learning_rate": 0.0001, + "loss": 63.4036, + "step": 4110 + }, + { + "epoch": 66.45528455284553, + "grad_norm": 4.167781829833984, + "learning_rate": 0.0001, + "loss": 63.1174, + "step": 4120 + }, + { + "epoch": 66.6178861788618, + "grad_norm": 1.1429376602172852, + "learning_rate": 0.0001, + "loss": 64.4085, + "step": 4130 + }, + { + "epoch": 66.78048780487805, + "grad_norm": 1.3188339471817017, + "learning_rate": 0.0001, + "loss": 62.5997, + "step": 4140 + }, + { + "epoch": 66.9430894308943, + "grad_norm": 1.160335659980774, + "learning_rate": 0.0001, + "loss": 64.2911, + "step": 4150 + }, + { + "epoch": 67.09756097560975, + "grad_norm": 0.9261873960494995, + "learning_rate": 0.0001, + "loss": 61.9036, + "step": 4160 + }, + { + "epoch": 67.26016260162602, + "grad_norm": 1.2254258394241333, + "learning_rate": 0.0001, + "loss": 64.1092, + "step": 4170 + }, + { + "epoch": 67.42276422764228, + "grad_norm": 0.9519097208976746, + "learning_rate": 0.0001, + "loss": 61.6428, + "step": 4180 + }, + { + "epoch": 67.58536585365853, + "grad_norm": 1.2290887832641602, + "learning_rate": 0.0001, + "loss": 64.9842, + "step": 4190 + }, + { + "epoch": 67.7479674796748, + "grad_norm": 1.0995723009109497, + "learning_rate": 0.0001, + "loss": 64.0677, + "step": 4200 + }, + { + "epoch": 67.91056910569105, + "grad_norm": 2.727957010269165, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 4210 + }, + { + "epoch": 68.0650406504065, + "grad_norm": 1.1222411394119263, + "learning_rate": 0.0001, + "loss": 61.0693, + "step": 4220 + }, + { + "epoch": 68.22764227642277, + "grad_norm": 1.159303069114685, + "learning_rate": 0.0001, + "loss": 64.2469, + "step": 4230 + }, + { + "epoch": 68.39024390243902, + "grad_norm": 1.2205990552902222, + "learning_rate": 0.0001, + "loss": 63.8831, + "step": 4240 + }, + { + "epoch": 68.55284552845528, + "grad_norm": 1.2351205348968506, + "learning_rate": 0.0001, + "loss": 63.4294, + "step": 4250 + }, + { + "epoch": 68.71544715447155, + "grad_norm": 1.4483757019042969, + "learning_rate": 0.0001, + "loss": 63.5345, + "step": 4260 + }, + { + "epoch": 68.8780487804878, + "grad_norm": 1.0808624029159546, + "learning_rate": 0.0001, + "loss": 63.6885, + "step": 4270 + }, + { + "epoch": 69.03252032520325, + "grad_norm": 1.7775999307632446, + "learning_rate": 0.0001, + "loss": 60.7266, + "step": 4280 + }, + { + "epoch": 69.1951219512195, + "grad_norm": 1.3214768171310425, + "learning_rate": 0.0001, + "loss": 63.3496, + "step": 4290 + }, + { + "epoch": 69.35772357723577, + "grad_norm": 1.1698002815246582, + "learning_rate": 0.0001, + "loss": 63.5284, + "step": 4300 + }, + { + "epoch": 69.52032520325203, + "grad_norm": 1.2050056457519531, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 4310 + }, + { + "epoch": 69.6829268292683, + "grad_norm": 1.0873243808746338, + "learning_rate": 0.0001, + "loss": 65.1709, + "step": 4320 + }, + { + "epoch": 69.84552845528455, + "grad_norm": 1.14133620262146, + "learning_rate": 0.0001, + "loss": 63.1872, + "step": 4330 + }, + { + "epoch": 70.0, + "grad_norm": 0.9684088230133057, + "learning_rate": 0.0001, + "loss": 60.1486, + "step": 4340 + }, + { + "epoch": 70.16260162601625, + "grad_norm": 1.1206653118133545, + "learning_rate": 0.0001, + "loss": 61.117, + "step": 4350 + }, + { + "epoch": 70.32520325203252, + "grad_norm": 1.2754334211349487, + "learning_rate": 0.0001, + "loss": 62.5409, + "step": 4360 + }, + { + "epoch": 70.48780487804878, + "grad_norm": 1.0154718160629272, + "learning_rate": 0.0001, + "loss": 62.9428, + "step": 4370 + }, + { + "epoch": 70.65040650406505, + "grad_norm": 1.0597270727157593, + "learning_rate": 0.0001, + "loss": 63.079, + "step": 4380 + }, + { + "epoch": 70.8130081300813, + "grad_norm": 1.254758596420288, + "learning_rate": 0.0001, + "loss": 63.3035, + "step": 4390 + }, + { + "epoch": 70.97560975609755, + "grad_norm": 1.0185120105743408, + "learning_rate": 0.0001, + "loss": 63.8922, + "step": 4400 + }, + { + "epoch": 71.130081300813, + "grad_norm": 1.007300853729248, + "learning_rate": 0.0001, + "loss": 58.7735, + "step": 4410 + }, + { + "epoch": 71.29268292682927, + "grad_norm": 1.2012463808059692, + "learning_rate": 0.0001, + "loss": 63.1782, + "step": 4420 + }, + { + "epoch": 71.45528455284553, + "grad_norm": 1.0258845090866089, + "learning_rate": 0.0001, + "loss": 63.2483, + "step": 4430 + }, + { + "epoch": 71.6178861788618, + "grad_norm": 1.0611220598220825, + "learning_rate": 0.0001, + "loss": 63.2247, + "step": 4440 + }, + { + "epoch": 71.78048780487805, + "grad_norm": 1.060114860534668, + "learning_rate": 0.0001, + "loss": 61.8352, + "step": 4450 + }, + { + "epoch": 71.9430894308943, + "grad_norm": 1.0540589094161987, + "learning_rate": 0.0001, + "loss": 62.1156, + "step": 4460 + }, + { + "epoch": 72.09756097560975, + "grad_norm": 1.4923726320266724, + "learning_rate": 0.0001, + "loss": 59.3863, + "step": 4470 + }, + { + "epoch": 72.26016260162602, + "grad_norm": 1.13791024684906, + "learning_rate": 0.0001, + "loss": 62.203, + "step": 4480 + }, + { + "epoch": 72.42276422764228, + "grad_norm": 0.9737065434455872, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 4490 + }, + { + "epoch": 72.58536585365853, + "grad_norm": 1.3980190753936768, + "learning_rate": 0.0001, + "loss": 63.5627, + "step": 4500 + }, + { + "epoch": 72.7479674796748, + "grad_norm": 1.0274118185043335, + "learning_rate": 0.0001, + "loss": 63.6129, + "step": 4510 + }, + { + "epoch": 72.91056910569105, + "grad_norm": 1.3148925304412842, + "learning_rate": 0.0001, + "loss": 63.6506, + "step": 4520 + }, + { + "epoch": 73.0650406504065, + "grad_norm": 1.4182451963424683, + "learning_rate": 0.0001, + "loss": 60.3616, + "step": 4530 + }, + { + "epoch": 73.22764227642277, + "grad_norm": 0.9235166907310486, + "learning_rate": 0.0001, + "loss": 63.6465, + "step": 4540 + }, + { + "epoch": 73.39024390243902, + "grad_norm": 0.9293469786643982, + "learning_rate": 0.0001, + "loss": 61.0984, + "step": 4550 + }, + { + "epoch": 73.55284552845528, + "grad_norm": 1.1075892448425293, + "learning_rate": 0.0001, + "loss": 62.9356, + "step": 4560 + }, + { + "epoch": 73.71544715447155, + "grad_norm": 1.4955459833145142, + "learning_rate": 0.0001, + "loss": 61.3811, + "step": 4570 + }, + { + "epoch": 73.8780487804878, + "grad_norm": 1.220595359802246, + "learning_rate": 0.0001, + "loss": 62.4364, + "step": 4580 + }, + { + "epoch": 74.03252032520325, + "grad_norm": 0.9975704550743103, + "learning_rate": 0.0001, + "loss": 58.5194, + "step": 4590 + }, + { + "epoch": 74.1951219512195, + "grad_norm": 1.0160857439041138, + "learning_rate": 0.0001, + "loss": 64.6989, + "step": 4600 + }, + { + "epoch": 74.35772357723577, + "grad_norm": 1.0052287578582764, + "learning_rate": 0.0001, + "loss": 62.526, + "step": 4610 + }, + { + "epoch": 74.52032520325203, + "grad_norm": 1.015514612197876, + "learning_rate": 0.0001, + "loss": 63.8627, + "step": 4620 + }, + { + "epoch": 74.6829268292683, + "grad_norm": 1.139051914215088, + "learning_rate": 0.0001, + "loss": 62.4525, + "step": 4630 + }, + { + "epoch": 74.84552845528455, + "grad_norm": 1.1727714538574219, + "learning_rate": 0.0001, + "loss": 64.0046, + "step": 4640 + }, + { + "epoch": 75.0, + "grad_norm": 1.1611062288284302, + "learning_rate": 0.0001, + "loss": 57.6443, + "step": 4650 + }, + { + "epoch": 75.16260162601625, + "grad_norm": 0.9893108606338501, + "learning_rate": 0.0001, + "loss": 64.1109, + "step": 4660 + }, + { + "epoch": 75.32520325203252, + "grad_norm": 1.190146803855896, + "learning_rate": 0.0001, + "loss": 62.1988, + "step": 4670 + }, + { + "epoch": 75.48780487804878, + "grad_norm": 1.1648579835891724, + "learning_rate": 0.0001, + "loss": 63.1393, + "step": 4680 + }, + { + "epoch": 75.65040650406505, + "grad_norm": 1.3303166627883911, + "learning_rate": 0.0001, + "loss": 63.7, + "step": 4690 + }, + { + "epoch": 75.8130081300813, + "grad_norm": 1.407772421836853, + "learning_rate": 0.0001, + "loss": 60.6798, + "step": 4700 + }, + { + "epoch": 75.97560975609755, + "grad_norm": 1.039741039276123, + "learning_rate": 0.0001, + "loss": 61.6126, + "step": 4710 + }, + { + "epoch": 76.130081300813, + "grad_norm": 1.013515830039978, + "learning_rate": 0.0001, + "loss": 57.8452, + "step": 4720 + }, + { + "epoch": 76.29268292682927, + "grad_norm": 1.1279789209365845, + "learning_rate": 0.0001, + "loss": 63.0592, + "step": 4730 + }, + { + "epoch": 76.45528455284553, + "grad_norm": 1.158605694770813, + "learning_rate": 0.0001, + "loss": 62.0836, + "step": 4740 + }, + { + "epoch": 76.6178861788618, + "grad_norm": 1.0200021266937256, + "learning_rate": 0.0001, + "loss": 62.1775, + "step": 4750 + }, + { + "epoch": 76.78048780487805, + "grad_norm": 1.216412901878357, + "learning_rate": 0.0001, + "loss": 62.6316, + "step": 4760 + }, + { + "epoch": 76.9430894308943, + "grad_norm": 1.0513523817062378, + "learning_rate": 0.0001, + "loss": 60.1037, + "step": 4770 + }, + { + "epoch": 77.09756097560975, + "grad_norm": 1.10075843334198, + "learning_rate": 0.0001, + "loss": 58.7599, + "step": 4780 + }, + { + "epoch": 77.26016260162602, + "grad_norm": 1.1728724241256714, + "learning_rate": 0.0001, + "loss": 63.3476, + "step": 4790 + }, + { + "epoch": 77.42276422764228, + "grad_norm": 1.3511605262756348, + "learning_rate": 0.0001, + "loss": 60.9885, + "step": 4800 + }, + { + "epoch": 77.58536585365853, + "grad_norm": 0.9733042120933533, + "learning_rate": 0.0001, + "loss": 62.4245, + "step": 4810 + }, + { + "epoch": 77.7479674796748, + "grad_norm": 1.0380535125732422, + "learning_rate": 0.0001, + "loss": 64.0755, + "step": 4820 + }, + { + "epoch": 77.91056910569105, + "grad_norm": 1.1594985723495483, + "learning_rate": 0.0001, + "loss": 62.1099, + "step": 4830 + }, + { + "epoch": 78.0650406504065, + "grad_norm": 1.0392569303512573, + "learning_rate": 0.0001, + "loss": 58.3778, + "step": 4840 + }, + { + "epoch": 78.22764227642277, + "grad_norm": 1.0032405853271484, + "learning_rate": 0.0001, + "loss": 62.1384, + "step": 4850 + }, + { + "epoch": 78.39024390243902, + "grad_norm": 1.0576013326644897, + "learning_rate": 0.0001, + "loss": 63.3308, + "step": 4860 + }, + { + "epoch": 78.55284552845528, + "grad_norm": 1.34328031539917, + "learning_rate": 0.0001, + "loss": 62.5351, + "step": 4870 + }, + { + "epoch": 78.71544715447155, + "grad_norm": 1.1040822267532349, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 4880 + }, + { + "epoch": 78.8780487804878, + "grad_norm": 1.1554768085479736, + "learning_rate": 0.0001, + "loss": 62.7308, + "step": 4890 + }, + { + "epoch": 79.03252032520325, + "grad_norm": 1.1864408254623413, + "learning_rate": 0.0001, + "loss": 60.0741, + "step": 4900 + }, + { + "epoch": 79.1951219512195, + "grad_norm": 0.9181821346282959, + "learning_rate": 0.0001, + "loss": 63.1682, + "step": 4910 + }, + { + "epoch": 79.35772357723577, + "grad_norm": 1.1400946378707886, + "learning_rate": 0.0001, + "loss": 62.0736, + "step": 4920 + }, + { + "epoch": 79.52032520325203, + "grad_norm": 1.245202660560608, + "learning_rate": 0.0001, + "loss": 60.51, + "step": 4930 + }, + { + "epoch": 79.6829268292683, + "grad_norm": 1.033347725868225, + "learning_rate": 0.0001, + "loss": 61.7845, + "step": 4940 + }, + { + "epoch": 79.84552845528455, + "grad_norm": 1.6206163167953491, + "learning_rate": 0.0001, + "loss": 61.5232, + "step": 4950 + }, + { + "epoch": 80.0, + "grad_norm": 1.0924937725067139, + "learning_rate": 0.0001, + "loss": 57.24, + "step": 4960 + }, + { + "epoch": 80.16260162601625, + "grad_norm": 1.075985074043274, + "learning_rate": 0.0001, + "loss": 61.2615, + "step": 4970 + }, + { + "epoch": 80.32520325203252, + "grad_norm": 0.9964771866798401, + "learning_rate": 0.0001, + "loss": 62.287, + "step": 4980 + }, + { + "epoch": 80.48780487804878, + "grad_norm": 1.3707735538482666, + "learning_rate": 0.0001, + "loss": 60.9383, + "step": 4990 + }, + { + "epoch": 80.65040650406505, + "grad_norm": 0.9395170211791992, + "learning_rate": 0.0001, + "loss": 61.2697, + "step": 5000 + }, + { + "epoch": 80.8130081300813, + "grad_norm": 2.0208873748779297, + "learning_rate": 0.0001, + "loss": 61.7219, + "step": 5010 + }, + { + "epoch": 80.97560975609755, + "grad_norm": 1.468702793121338, + "learning_rate": 0.0001, + "loss": 62.6967, + "step": 5020 + }, + { + "epoch": 81.130081300813, + "grad_norm": 1.0525966882705688, + "learning_rate": 0.0001, + "loss": 59.9793, + "step": 5030 + }, + { + "epoch": 81.29268292682927, + "grad_norm": 1.0951132774353027, + "learning_rate": 0.0001, + "loss": 60.9807, + "step": 5040 + }, + { + "epoch": 81.45528455284553, + "grad_norm": 1.3457857370376587, + "learning_rate": 0.0001, + "loss": 61.4137, + "step": 5050 + }, + { + "epoch": 81.6178861788618, + "grad_norm": 1.0895649194717407, + "learning_rate": 0.0001, + "loss": 61.2747, + "step": 5060 + }, + { + "epoch": 81.78048780487805, + "grad_norm": 1.0552371740341187, + "learning_rate": 0.0001, + "loss": 62.0947, + "step": 5070 + }, + { + "epoch": 81.9430894308943, + "grad_norm": 1.039038896560669, + "learning_rate": 0.0001, + "loss": 61.2571, + "step": 5080 + }, + { + "epoch": 82.09756097560975, + "grad_norm": 0.970783531665802, + "learning_rate": 0.0001, + "loss": 57.4157, + "step": 5090 + }, + { + "epoch": 82.26016260162602, + "grad_norm": 1.0276020765304565, + "learning_rate": 0.0001, + "loss": 62.1324, + "step": 5100 + }, + { + "epoch": 82.42276422764228, + "grad_norm": 1.2092448472976685, + "learning_rate": 0.0001, + "loss": 60.6448, + "step": 5110 + }, + { + "epoch": 82.58536585365853, + "grad_norm": 0.9352007508277893, + "learning_rate": 0.0001, + "loss": 61.9299, + "step": 5120 + }, + { + "epoch": 82.7479674796748, + "grad_norm": 1.2678358554840088, + "learning_rate": 0.0001, + "loss": 61.6622, + "step": 5130 + }, + { + "epoch": 82.91056910569105, + "grad_norm": 1.200494647026062, + "learning_rate": 0.0001, + "loss": 62.3169, + "step": 5140 + }, + { + "epoch": 83.0650406504065, + "grad_norm": 1.1284401416778564, + "learning_rate": 0.0001, + "loss": 58.7768, + "step": 5150 + }, + { + "epoch": 83.22764227642277, + "grad_norm": 1.1578079462051392, + "learning_rate": 0.0001, + "loss": 62.8423, + "step": 5160 + }, + { + "epoch": 83.39024390243902, + "grad_norm": 1.3426504135131836, + "learning_rate": 0.0001, + "loss": 61.5332, + "step": 5170 + }, + { + "epoch": 83.55284552845528, + "grad_norm": 0.9397301077842712, + "learning_rate": 0.0001, + "loss": 60.1043, + "step": 5180 + }, + { + "epoch": 83.71544715447155, + "grad_norm": 1.0728143453598022, + "learning_rate": 0.0001, + "loss": 60.5927, + "step": 5190 + }, + { + "epoch": 83.8780487804878, + "grad_norm": 1.2620211839675903, + "learning_rate": 0.0001, + "loss": 60.9224, + "step": 5200 + }, + { + "epoch": 84.03252032520325, + "grad_norm": 0.8232231140136719, + "learning_rate": 0.0001, + "loss": 58.8128, + "step": 5210 + }, + { + "epoch": 84.1951219512195, + "grad_norm": 1.0222564935684204, + "learning_rate": 0.0001, + "loss": 59.6995, + "step": 5220 + }, + { + "epoch": 84.35772357723577, + "grad_norm": 0.9078415632247925, + "learning_rate": 0.0001, + "loss": 61.1521, + "step": 5230 + }, + { + "epoch": 84.52032520325203, + "grad_norm": 1.0269399881362915, + "learning_rate": 0.0001, + "loss": 61.0136, + "step": 5240 + }, + { + "epoch": 84.6829268292683, + "grad_norm": 1.2108154296875, + "learning_rate": 0.0001, + "loss": 61.3564, + "step": 5250 + }, + { + "epoch": 84.84552845528455, + "grad_norm": 1.1170622110366821, + "learning_rate": 0.0001, + "loss": 61.7047, + "step": 5260 + }, + { + "epoch": 85.0, + "grad_norm": 1.467298150062561, + "learning_rate": 0.0001, + "loss": 58.1667, + "step": 5270 + }, + { + "epoch": 85.16260162601625, + "grad_norm": 0.908743143081665, + "learning_rate": 0.0001, + "loss": 61.5694, + "step": 5280 + }, + { + "epoch": 85.32520325203252, + "grad_norm": 0.9148766398429871, + "learning_rate": 0.0001, + "loss": 61.5325, + "step": 5290 + }, + { + "epoch": 85.48780487804878, + "grad_norm": 0.940021276473999, + "learning_rate": 0.0001, + "loss": 62.8807, + "step": 5300 + }, + { + "epoch": 85.65040650406505, + "grad_norm": 1.1239454746246338, + "learning_rate": 0.0001, + "loss": 62.361, + "step": 5310 + }, + { + "epoch": 85.8130081300813, + "grad_norm": 0.840437114238739, + "learning_rate": 0.0001, + "loss": 61.9057, + "step": 5320 + }, + { + "epoch": 85.97560975609755, + "grad_norm": 0.9764263033866882, + "learning_rate": 0.0001, + "loss": 61.0075, + "step": 5330 + }, + { + "epoch": 86.130081300813, + "grad_norm": 0.998540997505188, + "learning_rate": 0.0001, + "loss": 57.7146, + "step": 5340 + }, + { + "epoch": 86.29268292682927, + "grad_norm": 1.121551513671875, + "learning_rate": 0.0001, + "loss": 62.2278, + "step": 5350 + }, + { + "epoch": 86.45528455284553, + "grad_norm": 1.101842999458313, + "learning_rate": 0.0001, + "loss": 60.8837, + "step": 5360 + }, + { + "epoch": 86.6178861788618, + "grad_norm": 1.1724908351898193, + "learning_rate": 0.0001, + "loss": 61.6788, + "step": 5370 + }, + { + "epoch": 86.78048780487805, + "grad_norm": 1.3108958005905151, + "learning_rate": 0.0001, + "loss": 60.1461, + "step": 5380 + }, + { + "epoch": 86.9430894308943, + "grad_norm": 1.482298731803894, + "learning_rate": 0.0001, + "loss": 61.0457, + "step": 5390 + }, + { + "epoch": 87.09756097560975, + "grad_norm": 1.1471034288406372, + "learning_rate": 0.0001, + "loss": 57.6835, + "step": 5400 + }, + { + "epoch": 87.26016260162602, + "grad_norm": 0.9349374175071716, + "learning_rate": 0.0001, + "loss": 62.7018, + "step": 5410 + }, + { + "epoch": 87.42276422764228, + "grad_norm": 0.9917856454849243, + "learning_rate": 0.0001, + "loss": 60.2201, + "step": 5420 + }, + { + "epoch": 87.58536585365853, + "grad_norm": 1.0112310647964478, + "learning_rate": 0.0001, + "loss": 62.1185, + "step": 5430 + }, + { + "epoch": 87.7479674796748, + "grad_norm": 1.288894534111023, + "learning_rate": 0.0001, + "loss": 61.4348, + "step": 5440 + }, + { + "epoch": 87.91056910569105, + "grad_norm": 0.8577322363853455, + "learning_rate": 0.0001, + "loss": 60.7867, + "step": 5450 + }, + { + "epoch": 88.0650406504065, + "grad_norm": 0.8162860870361328, + "learning_rate": 0.0001, + "loss": 59.5224, + "step": 5460 + }, + { + "epoch": 88.22764227642277, + "grad_norm": 0.9491421580314636, + "learning_rate": 0.0001, + "loss": 60.4936, + "step": 5470 + }, + { + "epoch": 88.39024390243902, + "grad_norm": 4.164788246154785, + "learning_rate": 0.0001, + "loss": 61.8397, + "step": 5480 + }, + { + "epoch": 88.55284552845528, + "grad_norm": 0.9954016804695129, + "learning_rate": 0.0001, + "loss": 61.286, + "step": 5490 + }, + { + "epoch": 88.71544715447155, + "grad_norm": 1.4991120100021362, + "learning_rate": 0.0001, + "loss": 60.6157, + "step": 5500 + }, + { + "epoch": 88.8780487804878, + "grad_norm": 1.3678480386734009, + "learning_rate": 0.0001, + "loss": 62.5187, + "step": 5510 + }, + { + "epoch": 89.03252032520325, + "grad_norm": 1.057661771774292, + "learning_rate": 0.0001, + "loss": 57.5134, + "step": 5520 + }, + { + "epoch": 89.1951219512195, + "grad_norm": 0.8331280946731567, + "learning_rate": 0.0001, + "loss": 62.007, + "step": 5530 + }, + { + "epoch": 89.35772357723577, + "grad_norm": 0.9842398762702942, + "learning_rate": 0.0001, + "loss": 60.6178, + "step": 5540 + }, + { + "epoch": 89.52032520325203, + "grad_norm": 1.0510483980178833, + "learning_rate": 0.0001, + "loss": 59.6382, + "step": 5550 + }, + { + "epoch": 89.6829268292683, + "grad_norm": 1.013115406036377, + "learning_rate": 0.0001, + "loss": 61.015, + "step": 5560 + }, + { + "epoch": 89.84552845528455, + "grad_norm": 1.1595944166183472, + "learning_rate": 0.0001, + "loss": 62.5374, + "step": 5570 + }, + { + "epoch": 90.0, + "grad_norm": 1.1506270170211792, + "learning_rate": 0.0001, + "loss": 61.0063, + "step": 5580 + }, + { + "epoch": 90.16260162601625, + "grad_norm": 0.9940153360366821, + "learning_rate": 0.0001, + "loss": 62.0944, + "step": 5590 + }, + { + "epoch": 90.32520325203252, + "grad_norm": 1.1302493810653687, + "learning_rate": 0.0001, + "loss": 61.616, + "step": 5600 + }, + { + "epoch": 90.48780487804878, + "grad_norm": 1.2370656728744507, + "learning_rate": 0.0001, + "loss": 61.291, + "step": 5610 + }, + { + "epoch": 90.65040650406505, + "grad_norm": 1.228763461112976, + "learning_rate": 0.0001, + "loss": 60.0629, + "step": 5620 + }, + { + "epoch": 90.8130081300813, + "grad_norm": 1.2230069637298584, + "learning_rate": 0.0001, + "loss": 61.5391, + "step": 5630 + }, + { + "epoch": 90.97560975609755, + "grad_norm": 0.9849008917808533, + "learning_rate": 0.0001, + "loss": 63.1633, + "step": 5640 + }, + { + "epoch": 91.130081300813, + "grad_norm": 1.408632755279541, + "learning_rate": 0.0001, + "loss": 58.4195, + "step": 5650 + }, + { + "epoch": 91.29268292682927, + "grad_norm": 0.929027795791626, + "learning_rate": 0.0001, + "loss": 60.9862, + "step": 5660 + }, + { + "epoch": 91.45528455284553, + "grad_norm": 0.9228377342224121, + "learning_rate": 0.0001, + "loss": 60.8337, + "step": 5670 + }, + { + "epoch": 91.6178861788618, + "grad_norm": 1.0354220867156982, + "learning_rate": 0.0001, + "loss": 59.4667, + "step": 5680 + }, + { + "epoch": 91.78048780487805, + "grad_norm": 1.0338927507400513, + "learning_rate": 0.0001, + "loss": 62.0064, + "step": 5690 + }, + { + "epoch": 91.9430894308943, + "grad_norm": 1.101967215538025, + "learning_rate": 0.0001, + "loss": 61.6317, + "step": 5700 + }, + { + "epoch": 92.09756097560975, + "grad_norm": 1.370689034461975, + "learning_rate": 0.0001, + "loss": 57.2157, + "step": 5710 + }, + { + "epoch": 92.26016260162602, + "grad_norm": 0.9904115796089172, + "learning_rate": 0.0001, + "loss": 61.2786, + "step": 5720 + }, + { + "epoch": 92.42276422764228, + "grad_norm": 1.0545448064804077, + "learning_rate": 0.0001, + "loss": 63.2384, + "step": 5730 + }, + { + "epoch": 92.58536585365853, + "grad_norm": 1.0657731294631958, + "learning_rate": 0.0001, + "loss": 60.582, + "step": 5740 + }, + { + "epoch": 92.7479674796748, + "grad_norm": 0.9333183765411377, + "learning_rate": 0.0001, + "loss": 62.4627, + "step": 5750 + }, + { + "epoch": 92.91056910569105, + "grad_norm": 0.9810492396354675, + "learning_rate": 0.0001, + "loss": 60.0055, + "step": 5760 + }, + { + "epoch": 93.0650406504065, + "grad_norm": 1.6973884105682373, + "learning_rate": 0.0001, + "loss": 57.5381, + "step": 5770 + }, + { + "epoch": 93.22764227642277, + "grad_norm": 1.0075498819351196, + "learning_rate": 0.0001, + "loss": 61.2272, + "step": 5780 + }, + { + "epoch": 93.39024390243902, + "grad_norm": 0.980633556842804, + "learning_rate": 0.0001, + "loss": 60.4794, + "step": 5790 + }, + { + "epoch": 93.55284552845528, + "grad_norm": 1.128214955329895, + "learning_rate": 0.0001, + "loss": 60.4825, + "step": 5800 + }, + { + "epoch": 93.71544715447155, + "grad_norm": 1.302827000617981, + "learning_rate": 0.0001, + "loss": 62.4397, + "step": 5810 + }, + { + "epoch": 93.8780487804878, + "grad_norm": 0.826600193977356, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 5820 + }, + { + "epoch": 94.03252032520325, + "grad_norm": 11.418835639953613, + "learning_rate": 0.0001, + "loss": 58.5383, + "step": 5830 + }, + { + "epoch": 94.1951219512195, + "grad_norm": 1.0686477422714233, + "learning_rate": 0.0001, + "loss": 60.5696, + "step": 5840 + }, + { + "epoch": 94.35772357723577, + "grad_norm": 1.1232597827911377, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 5850 + }, + { + "epoch": 94.52032520325203, + "grad_norm": 1.1051944494247437, + "learning_rate": 0.0001, + "loss": 60.5387, + "step": 5860 + }, + { + "epoch": 94.6829268292683, + "grad_norm": 0.9954739809036255, + "learning_rate": 0.0001, + "loss": 61.3899, + "step": 5870 + }, + { + "epoch": 94.84552845528455, + "grad_norm": 1.9687999486923218, + "learning_rate": 0.0001, + "loss": 60.7711, + "step": 5880 + }, + { + "epoch": 95.0, + "grad_norm": 1.3172541856765747, + "learning_rate": 0.0001, + "loss": 58.5394, + "step": 5890 + }, + { + "epoch": 95.16260162601625, + "grad_norm": 1.2251427173614502, + "learning_rate": 0.0001, + "loss": 61.8319, + "step": 5900 + }, + { + "epoch": 95.32520325203252, + "grad_norm": 1.1701544523239136, + "learning_rate": 0.0001, + "loss": 60.1928, + "step": 5910 + }, + { + "epoch": 95.48780487804878, + "grad_norm": 1.209375262260437, + "learning_rate": 0.0001, + "loss": 61.1347, + "step": 5920 + }, + { + "epoch": 95.65040650406505, + "grad_norm": 0.9307044148445129, + "learning_rate": 0.0001, + "loss": 62.6676, + "step": 5930 + }, + { + "epoch": 95.8130081300813, + "grad_norm": 0.9679144024848938, + "learning_rate": 0.0001, + "loss": 60.7623, + "step": 5940 + }, + { + "epoch": 95.97560975609755, + "grad_norm": 1.398711085319519, + "learning_rate": 0.0001, + "loss": 59.0407, + "step": 5950 + }, + { + "epoch": 96.130081300813, + "grad_norm": 1.9118247032165527, + "learning_rate": 0.0001, + "loss": 59.9417, + "step": 5960 + }, + { + "epoch": 96.29268292682927, + "grad_norm": 1.036737084388733, + "learning_rate": 0.0001, + "loss": 60.6891, + "step": 5970 + }, + { + "epoch": 96.45528455284553, + "grad_norm": 1.5916132926940918, + "learning_rate": 0.0001, + "loss": 59.44, + "step": 5980 + }, + { + "epoch": 96.6178861788618, + "grad_norm": 1.526591181755066, + "learning_rate": 0.0001, + "loss": 58.9081, + "step": 5990 + }, + { + "epoch": 96.78048780487805, + "grad_norm": 0.9556435942649841, + "learning_rate": 0.0001, + "loss": 61.5743, + "step": 6000 + }, + { + "epoch": 96.9430894308943, + "grad_norm": 1.0041887760162354, + "learning_rate": 0.0001, + "loss": 60.0724, + "step": 6010 + }, + { + "epoch": 97.09756097560975, + "grad_norm": 0.9792081117630005, + "learning_rate": 0.0001, + "loss": 60.0613, + "step": 6020 + }, + { + "epoch": 97.26016260162602, + "grad_norm": 1.029634714126587, + "learning_rate": 0.0001, + "loss": 60.3526, + "step": 6030 + }, + { + "epoch": 97.42276422764228, + "grad_norm": 0.9812865257263184, + "learning_rate": 0.0001, + "loss": 60.9818, + "step": 6040 + }, + { + "epoch": 97.58536585365853, + "grad_norm": 0.8826327323913574, + "learning_rate": 0.0001, + "loss": 61.6375, + "step": 6050 + }, + { + "epoch": 97.7479674796748, + "grad_norm": 1.2197599411010742, + "learning_rate": 0.0001, + "loss": 60.2698, + "step": 6060 + }, + { + "epoch": 97.91056910569105, + "grad_norm": 0.9068922400474548, + "learning_rate": 0.0001, + "loss": 58.8698, + "step": 6070 + }, + { + "epoch": 98.0650406504065, + "grad_norm": 0.9008221626281738, + "learning_rate": 0.0001, + "loss": 56.4687, + "step": 6080 + }, + { + "epoch": 98.22764227642277, + "grad_norm": 1.8165878057479858, + "learning_rate": 0.0001, + "loss": 61.4213, + "step": 6090 + }, + { + "epoch": 98.39024390243902, + "grad_norm": 0.9541304111480713, + "learning_rate": 0.0001, + "loss": 62.8381, + "step": 6100 + }, + { + "epoch": 98.55284552845528, + "grad_norm": 1.293872594833374, + "learning_rate": 0.0001, + "loss": 61.3292, + "step": 6110 + }, + { + "epoch": 98.71544715447155, + "grad_norm": 0.9569422006607056, + "learning_rate": 0.0001, + "loss": 61.436, + "step": 6120 + }, + { + "epoch": 98.8780487804878, + "grad_norm": 1.160468578338623, + "learning_rate": 0.0001, + "loss": 60.8419, + "step": 6130 + }, + { + "epoch": 99.03252032520325, + "grad_norm": 1.3102303743362427, + "learning_rate": 0.0001, + "loss": 59.0064, + "step": 6140 + }, + { + "epoch": 99.1951219512195, + "grad_norm": 0.9553463459014893, + "learning_rate": 0.0001, + "loss": 59.925, + "step": 6150 + }, + { + "epoch": 99.35772357723577, + "grad_norm": 1.1993688344955444, + "learning_rate": 0.0001, + "loss": 62.3683, + "step": 6160 + }, + { + "epoch": 99.52032520325203, + "grad_norm": 1.0867350101470947, + "learning_rate": 0.0001, + "loss": 61.8848, + "step": 6170 + }, + { + "epoch": 99.6829268292683, + "grad_norm": 0.9924171566963196, + "learning_rate": 0.0001, + "loss": 60.9243, + "step": 6180 + }, + { + "epoch": 99.84552845528455, + "grad_norm": 0.9548614025115967, + "learning_rate": 0.0001, + "loss": 61.0234, + "step": 6190 + }, + { + "epoch": 100.0, + "grad_norm": 1.0070925951004028, + "learning_rate": 0.0001, + "loss": 56.6554, + "step": 6200 + }, + { + "epoch": 100.16260162601625, + "grad_norm": 1.202168583869934, + "learning_rate": 0.0001, + "loss": 61.3235, + "step": 6210 + }, + { + "epoch": 100.32520325203252, + "grad_norm": 0.9931091070175171, + "learning_rate": 0.0001, + "loss": 59.651, + "step": 6220 + }, + { + "epoch": 100.48780487804878, + "grad_norm": 0.9679157137870789, + "learning_rate": 0.0001, + "loss": 61.5785, + "step": 6230 + }, + { + "epoch": 100.65040650406505, + "grad_norm": 1.049842119216919, + "learning_rate": 0.0001, + "loss": 58.9646, + "step": 6240 + }, + { + "epoch": 100.8130081300813, + "grad_norm": 1.0772044658660889, + "learning_rate": 0.0001, + "loss": 62.3309, + "step": 6250 + }, + { + "epoch": 100.97560975609755, + "grad_norm": 0.9494168162345886, + "learning_rate": 0.0001, + "loss": 61.8663, + "step": 6260 + }, + { + "epoch": 101.130081300813, + "grad_norm": 1.0491962432861328, + "learning_rate": 0.0001, + "loss": 56.5872, + "step": 6270 + }, + { + "epoch": 101.29268292682927, + "grad_norm": 1.118611216545105, + "learning_rate": 0.0001, + "loss": 58.1338, + "step": 6280 + }, + { + "epoch": 101.45528455284553, + "grad_norm": 1.0419825315475464, + "learning_rate": 0.0001, + "loss": 59.6661, + "step": 6290 + }, + { + "epoch": 101.6178861788618, + "grad_norm": 1.2351056337356567, + "learning_rate": 0.0001, + "loss": 61.1697, + "step": 6300 + }, + { + "epoch": 101.78048780487805, + "grad_norm": 1.0556049346923828, + "learning_rate": 0.0001, + "loss": 60.463, + "step": 6310 + }, + { + "epoch": 101.9430894308943, + "grad_norm": 1.090761661529541, + "learning_rate": 0.0001, + "loss": 61.983, + "step": 6320 + }, + { + "epoch": 102.09756097560975, + "grad_norm": 1.4305297136306763, + "learning_rate": 0.0001, + "loss": 55.559, + "step": 6330 + }, + { + "epoch": 102.26016260162602, + "grad_norm": 1.1773794889450073, + "learning_rate": 0.0001, + "loss": 59.5931, + "step": 6340 + }, + { + "epoch": 102.42276422764228, + "grad_norm": 1.0028730630874634, + "learning_rate": 0.0001, + "loss": 60.2714, + "step": 6350 + }, + { + "epoch": 102.58536585365853, + "grad_norm": 0.9494019150733948, + "learning_rate": 0.0001, + "loss": 59.7757, + "step": 6360 + }, + { + "epoch": 102.7479674796748, + "grad_norm": 1.0415751934051514, + "learning_rate": 0.0001, + "loss": 60.7123, + "step": 6370 + }, + { + "epoch": 102.91056910569105, + "grad_norm": 1.2036449909210205, + "learning_rate": 0.0001, + "loss": 60.1201, + "step": 6380 + }, + { + "epoch": 103.0650406504065, + "grad_norm": 0.8801053166389465, + "learning_rate": 0.0001, + "loss": 56.4943, + "step": 6390 + }, + { + "epoch": 103.22764227642277, + "grad_norm": 0.9018735289573669, + "learning_rate": 0.0001, + "loss": 62.4522, + "step": 6400 + }, + { + "epoch": 103.39024390243902, + "grad_norm": 0.971685528755188, + "learning_rate": 0.0001, + "loss": 62.2367, + "step": 6410 + }, + { + "epoch": 103.55284552845528, + "grad_norm": 0.8531002402305603, + "learning_rate": 0.0001, + "loss": 59.5436, + "step": 6420 + }, + { + "epoch": 103.71544715447155, + "grad_norm": 2.123033285140991, + "learning_rate": 0.0001, + "loss": 59.0421, + "step": 6430 + }, + { + "epoch": 103.8780487804878, + "grad_norm": 1.209804654121399, + "learning_rate": 0.0001, + "loss": 60.7791, + "step": 6440 + }, + { + "epoch": 104.03252032520325, + "grad_norm": 0.8219161033630371, + "learning_rate": 0.0001, + "loss": 55.7522, + "step": 6450 + }, + { + "epoch": 104.1951219512195, + "grad_norm": 1.8217495679855347, + "learning_rate": 0.0001, + "loss": 60.6916, + "step": 6460 + }, + { + "epoch": 104.35772357723577, + "grad_norm": 1.060073971748352, + "learning_rate": 0.0001, + "loss": 59.971, + "step": 6470 + }, + { + "epoch": 104.52032520325203, + "grad_norm": 0.9892823696136475, + "learning_rate": 0.0001, + "loss": 61.467, + "step": 6480 + }, + { + "epoch": 104.6829268292683, + "grad_norm": 1.128934621810913, + "learning_rate": 0.0001, + "loss": 62.0451, + "step": 6490 + }, + { + "epoch": 104.84552845528455, + "grad_norm": 0.9627387523651123, + "learning_rate": 0.0001, + "loss": 61.532, + "step": 6500 + }, + { + "epoch": 105.0, + "grad_norm": 1.5344375371932983, + "learning_rate": 0.0001, + "loss": 56.4323, + "step": 6510 + }, + { + "epoch": 105.16260162601625, + "grad_norm": 0.9524087309837341, + "learning_rate": 0.0001, + "loss": 59.6628, + "step": 6520 + }, + { + "epoch": 105.32520325203252, + "grad_norm": 1.0469943284988403, + "learning_rate": 0.0001, + "loss": 59.0605, + "step": 6530 + }, + { + "epoch": 105.48780487804878, + "grad_norm": 2.7462990283966064, + "learning_rate": 0.0001, + "loss": 61.1429, + "step": 6540 + }, + { + "epoch": 105.65040650406505, + "grad_norm": 1.3140382766723633, + "learning_rate": 0.0001, + "loss": 58.2711, + "step": 6550 + }, + { + "epoch": 105.8130081300813, + "grad_norm": 1.1528750658035278, + "learning_rate": 0.0001, + "loss": 59.9937, + "step": 6560 + }, + { + "epoch": 105.97560975609755, + "grad_norm": 1.4449951648712158, + "learning_rate": 0.0001, + "loss": 59.1526, + "step": 6570 + }, + { + "epoch": 106.130081300813, + "grad_norm": 1.1350126266479492, + "learning_rate": 0.0001, + "loss": 55.1142, + "step": 6580 + }, + { + "epoch": 106.29268292682927, + "grad_norm": 0.9332845211029053, + "learning_rate": 0.0001, + "loss": 59.8049, + "step": 6590 + }, + { + "epoch": 106.45528455284553, + "grad_norm": 0.9678266644477844, + "learning_rate": 0.0001, + "loss": 60.9627, + "step": 6600 + }, + { + "epoch": 106.6178861788618, + "grad_norm": 1.1794427633285522, + "learning_rate": 0.0001, + "loss": 59.5372, + "step": 6610 + }, + { + "epoch": 106.78048780487805, + "grad_norm": 1.0404181480407715, + "learning_rate": 0.0001, + "loss": 59.2715, + "step": 6620 + }, + { + "epoch": 106.9430894308943, + "grad_norm": 0.8667073845863342, + "learning_rate": 0.0001, + "loss": 60.2625, + "step": 6630 + }, + { + "epoch": 107.09756097560975, + "grad_norm": 0.9940314888954163, + "learning_rate": 0.0001, + "loss": 55.6906, + "step": 6640 + }, + { + "epoch": 107.26016260162602, + "grad_norm": 1.3880919218063354, + "learning_rate": 0.0001, + "loss": 59.8103, + "step": 6650 + }, + { + "epoch": 107.42276422764228, + "grad_norm": 0.900931179523468, + "learning_rate": 0.0001, + "loss": 59.1929, + "step": 6660 + }, + { + "epoch": 107.58536585365853, + "grad_norm": 0.9374089241027832, + "learning_rate": 0.0001, + "loss": 60.1047, + "step": 6670 + }, + { + "epoch": 107.7479674796748, + "grad_norm": 1.0847140550613403, + "learning_rate": 0.0001, + "loss": 58.6943, + "step": 6680 + }, + { + "epoch": 107.91056910569105, + "grad_norm": 1.086506962776184, + "learning_rate": 0.0001, + "loss": 60.0341, + "step": 6690 + }, + { + "epoch": 108.0650406504065, + "grad_norm": 0.8984777331352234, + "learning_rate": 0.0001, + "loss": 57.0422, + "step": 6700 + }, + { + "epoch": 108.22764227642277, + "grad_norm": 1.187198519706726, + "learning_rate": 0.0001, + "loss": 60.7935, + "step": 6710 + }, + { + "epoch": 108.39024390243902, + "grad_norm": 0.8393696546554565, + "learning_rate": 0.0001, + "loss": 59.7896, + "step": 6720 + }, + { + "epoch": 108.55284552845528, + "grad_norm": 1.0898207426071167, + "learning_rate": 0.0001, + "loss": 60.2579, + "step": 6730 + }, + { + "epoch": 108.71544715447155, + "grad_norm": 1.016926646232605, + "learning_rate": 0.0001, + "loss": 60.9018, + "step": 6740 + }, + { + "epoch": 108.8780487804878, + "grad_norm": 1.3991293907165527, + "learning_rate": 0.0001, + "loss": 60.889, + "step": 6750 + }, + { + "epoch": 109.03252032520325, + "grad_norm": 0.9950843453407288, + "learning_rate": 0.0001, + "loss": 55.1098, + "step": 6760 + }, + { + "epoch": 109.1951219512195, + "grad_norm": 1.0843164920806885, + "learning_rate": 0.0001, + "loss": 60.3208, + "step": 6770 + }, + { + "epoch": 109.35772357723577, + "grad_norm": 0.9067014455795288, + "learning_rate": 0.0001, + "loss": 59.1254, + "step": 6780 + }, + { + "epoch": 109.52032520325203, + "grad_norm": 1.2166807651519775, + "learning_rate": 0.0001, + "loss": 59.9978, + "step": 6790 + }, + { + "epoch": 109.6829268292683, + "grad_norm": 1.256101131439209, + "learning_rate": 0.0001, + "loss": 60.2544, + "step": 6800 + }, + { + "epoch": 109.84552845528455, + "grad_norm": 1.312761664390564, + "learning_rate": 0.0001, + "loss": 59.1564, + "step": 6810 + }, + { + "epoch": 110.0, + "grad_norm": 0.9668724536895752, + "learning_rate": 0.0001, + "loss": 58.067, + "step": 6820 + }, + { + "epoch": 110.16260162601625, + "grad_norm": 1.1048885583877563, + "learning_rate": 0.0001, + "loss": 59.1506, + "step": 6830 + }, + { + "epoch": 110.32520325203252, + "grad_norm": 1.498469352722168, + "learning_rate": 0.0001, + "loss": 59.1366, + "step": 6840 + }, + { + "epoch": 110.48780487804878, + "grad_norm": 1.1747376918792725, + "learning_rate": 0.0001, + "loss": 59.8449, + "step": 6850 + }, + { + "epoch": 110.65040650406505, + "grad_norm": 1.1041772365570068, + "learning_rate": 0.0001, + "loss": 61.0879, + "step": 6860 + }, + { + "epoch": 110.8130081300813, + "grad_norm": 0.9452431797981262, + "learning_rate": 0.0001, + "loss": 60.8911, + "step": 6870 + }, + { + "epoch": 110.97560975609755, + "grad_norm": 1.0108182430267334, + "learning_rate": 0.0001, + "loss": 58.7924, + "step": 6880 + }, + { + "epoch": 111.130081300813, + "grad_norm": 0.9914487600326538, + "learning_rate": 0.0001, + "loss": 57.4749, + "step": 6890 + }, + { + "epoch": 111.29268292682927, + "grad_norm": 1.0404850244522095, + "learning_rate": 0.0001, + "loss": 59.5329, + "step": 6900 + }, + { + "epoch": 111.45528455284553, + "grad_norm": 1.0363966226577759, + "learning_rate": 0.0001, + "loss": 58.8972, + "step": 6910 + }, + { + "epoch": 111.6178861788618, + "grad_norm": 1.0189651250839233, + "learning_rate": 0.0001, + "loss": 59.3613, + "step": 6920 + }, + { + "epoch": 111.78048780487805, + "grad_norm": 0.9696914553642273, + "learning_rate": 0.0001, + "loss": 59.8001, + "step": 6930 + }, + { + "epoch": 111.9430894308943, + "grad_norm": 1.406821370124817, + "learning_rate": 0.0001, + "loss": 58.6405, + "step": 6940 + }, + { + "epoch": 112.09756097560975, + "grad_norm": 1.0969383716583252, + "learning_rate": 0.0001, + "loss": 57.4458, + "step": 6950 + }, + { + "epoch": 112.26016260162602, + "grad_norm": 1.117593765258789, + "learning_rate": 0.0001, + "loss": 62.599, + "step": 6960 + }, + { + "epoch": 112.42276422764228, + "grad_norm": 0.9327741861343384, + "learning_rate": 0.0001, + "loss": 60.632, + "step": 6970 + }, + { + "epoch": 112.58536585365853, + "grad_norm": 1.1198201179504395, + "learning_rate": 0.0001, + "loss": 59.6119, + "step": 6980 + }, + { + "epoch": 112.7479674796748, + "grad_norm": 0.8709354400634766, + "learning_rate": 0.0001, + "loss": 60.831, + "step": 6990 + }, + { + "epoch": 112.91056910569105, + "grad_norm": 0.9589444994926453, + "learning_rate": 0.0001, + "loss": 58.0002, + "step": 7000 + }, + { + "epoch": 113.0650406504065, + "grad_norm": 1.0971899032592773, + "learning_rate": 0.0001, + "loss": 56.8572, + "step": 7010 + }, + { + "epoch": 113.22764227642277, + "grad_norm": 0.9764513969421387, + "learning_rate": 0.0001, + "loss": 59.3038, + "step": 7020 + }, + { + "epoch": 113.39024390243902, + "grad_norm": 0.9050093293190002, + "learning_rate": 0.0001, + "loss": 59.9146, + "step": 7030 + }, + { + "epoch": 113.55284552845528, + "grad_norm": 0.9506774544715881, + "learning_rate": 0.0001, + "loss": 59.7523, + "step": 7040 + }, + { + "epoch": 113.71544715447155, + "grad_norm": 0.9678346514701843, + "learning_rate": 0.0001, + "loss": 61.3103, + "step": 7050 + }, + { + "epoch": 113.8780487804878, + "grad_norm": 0.9859515428543091, + "learning_rate": 0.0001, + "loss": 56.3866, + "step": 7060 + }, + { + "epoch": 114.03252032520325, + "grad_norm": 1.2696164846420288, + "learning_rate": 0.0001, + "loss": 58.16, + "step": 7070 + }, + { + "epoch": 114.1951219512195, + "grad_norm": 0.9323938488960266, + "learning_rate": 0.0001, + "loss": 58.9483, + "step": 7080 + }, + { + "epoch": 114.35772357723577, + "grad_norm": 1.4123785495758057, + "learning_rate": 0.0001, + "loss": 60.4666, + "step": 7090 + }, + { + "epoch": 114.52032520325203, + "grad_norm": 1.009020447731018, + "learning_rate": 0.0001, + "loss": 58.7152, + "step": 7100 + }, + { + "epoch": 114.6829268292683, + "grad_norm": 1.219383955001831, + "learning_rate": 0.0001, + "loss": 59.1494, + "step": 7110 + }, + { + "epoch": 114.84552845528455, + "grad_norm": 0.9579033255577087, + "learning_rate": 0.0001, + "loss": 58.9879, + "step": 7120 + }, + { + "epoch": 115.0, + "grad_norm": 0.899074912071228, + "learning_rate": 0.0001, + "loss": 55.2031, + "step": 7130 + }, + { + "epoch": 115.16260162601625, + "grad_norm": 0.9924830794334412, + "learning_rate": 0.0001, + "loss": 58.583, + "step": 7140 + }, + { + "epoch": 115.32520325203252, + "grad_norm": 1.0280295610427856, + "learning_rate": 0.0001, + "loss": 60.0948, + "step": 7150 + }, + { + "epoch": 115.48780487804878, + "grad_norm": 0.8108797669410706, + "learning_rate": 0.0001, + "loss": 60.9433, + "step": 7160 + }, + { + "epoch": 115.65040650406505, + "grad_norm": 1.2597944736480713, + "learning_rate": 0.0001, + "loss": 59.1905, + "step": 7170 + }, + { + "epoch": 115.8130081300813, + "grad_norm": 1.2565767765045166, + "learning_rate": 0.0001, + "loss": 59.3545, + "step": 7180 + }, + { + "epoch": 115.97560975609755, + "grad_norm": 1.2012240886688232, + "learning_rate": 0.0001, + "loss": 58.348, + "step": 7190 + }, + { + "epoch": 116.130081300813, + "grad_norm": 1.2883665561676025, + "learning_rate": 0.0001, + "loss": 57.462, + "step": 7200 + }, + { + "epoch": 116.29268292682927, + "grad_norm": 0.904152512550354, + "learning_rate": 0.0001, + "loss": 58.958, + "step": 7210 + }, + { + "epoch": 116.45528455284553, + "grad_norm": 1.1928468942642212, + "learning_rate": 0.0001, + "loss": 58.6106, + "step": 7220 + }, + { + "epoch": 116.6178861788618, + "grad_norm": 1.0655683279037476, + "learning_rate": 0.0001, + "loss": 58.4948, + "step": 7230 + }, + { + "epoch": 116.78048780487805, + "grad_norm": 1.0527434349060059, + "learning_rate": 0.0001, + "loss": 60.1771, + "step": 7240 + }, + { + "epoch": 116.9430894308943, + "grad_norm": 1.1439871788024902, + "learning_rate": 0.0001, + "loss": 58.6737, + "step": 7250 + }, + { + "epoch": 117.09756097560975, + "grad_norm": 1.506571650505066, + "learning_rate": 0.0001, + "loss": 58.0901, + "step": 7260 + }, + { + "epoch": 117.26016260162602, + "grad_norm": 1.1419955492019653, + "learning_rate": 0.0001, + "loss": 59.0705, + "step": 7270 + }, + { + "epoch": 117.42276422764228, + "grad_norm": 1.1211196184158325, + "learning_rate": 0.0001, + "loss": 57.801, + "step": 7280 + }, + { + "epoch": 117.58536585365853, + "grad_norm": 1.1424155235290527, + "learning_rate": 0.0001, + "loss": 59.7449, + "step": 7290 + }, + { + "epoch": 117.7479674796748, + "grad_norm": 1.0593281984329224, + "learning_rate": 0.0001, + "loss": 60.1789, + "step": 7300 + }, + { + "epoch": 117.91056910569105, + "grad_norm": 1.3680177927017212, + "learning_rate": 0.0001, + "loss": 58.4225, + "step": 7310 + }, + { + "epoch": 118.0650406504065, + "grad_norm": 0.8391700983047485, + "learning_rate": 0.0001, + "loss": 56.2102, + "step": 7320 + }, + { + "epoch": 118.22764227642277, + "grad_norm": 1.3166592121124268, + "learning_rate": 0.0001, + "loss": 61.0399, + "step": 7330 + }, + { + "epoch": 118.39024390243902, + "grad_norm": 1.929949402809143, + "learning_rate": 0.0001, + "loss": 59.6514, + "step": 7340 + }, + { + "epoch": 118.55284552845528, + "grad_norm": 0.9464254975318909, + "learning_rate": 0.0001, + "loss": 58.6488, + "step": 7350 + }, + { + "epoch": 118.71544715447155, + "grad_norm": 1.1066193580627441, + "learning_rate": 0.0001, + "loss": 61.5959, + "step": 7360 + }, + { + "epoch": 118.8780487804878, + "grad_norm": 1.216125726699829, + "learning_rate": 0.0001, + "loss": 58.8231, + "step": 7370 + }, + { + "epoch": 119.03252032520325, + "grad_norm": 1.1182583570480347, + "learning_rate": 0.0001, + "loss": 56.9496, + "step": 7380 + }, + { + "epoch": 119.1951219512195, + "grad_norm": 1.105729341506958, + "learning_rate": 0.0001, + "loss": 58.4291, + "step": 7390 + }, + { + "epoch": 119.35772357723577, + "grad_norm": 1.017879843711853, + "learning_rate": 0.0001, + "loss": 59.4963, + "step": 7400 + }, + { + "epoch": 119.52032520325203, + "grad_norm": 0.7882587909698486, + "learning_rate": 0.0001, + "loss": 57.7243, + "step": 7410 + }, + { + "epoch": 119.6829268292683, + "grad_norm": 0.8356695175170898, + "learning_rate": 0.0001, + "loss": 60.4962, + "step": 7420 + }, + { + "epoch": 119.84552845528455, + "grad_norm": 1.1358702182769775, + "learning_rate": 0.0001, + "loss": 60.7173, + "step": 7430 + }, + { + "epoch": 120.0, + "grad_norm": 1.116322636604309, + "learning_rate": 0.0001, + "loss": 53.8387, + "step": 7440 + }, + { + "epoch": 120.16260162601625, + "grad_norm": 0.8522642254829407, + "learning_rate": 0.0001, + "loss": 57.9095, + "step": 7450 + }, + { + "epoch": 120.32520325203252, + "grad_norm": 0.890731692314148, + "learning_rate": 0.0001, + "loss": 57.2296, + "step": 7460 + }, + { + "epoch": 120.48780487804878, + "grad_norm": 1.6366336345672607, + "learning_rate": 0.0001, + "loss": 58.9417, + "step": 7470 + }, + { + "epoch": 120.65040650406505, + "grad_norm": 0.9774733781814575, + "learning_rate": 0.0001, + "loss": 58.8446, + "step": 7480 + }, + { + "epoch": 120.8130081300813, + "grad_norm": 1.6188803911209106, + "learning_rate": 0.0001, + "loss": 59.9794, + "step": 7490 + }, + { + "epoch": 120.97560975609755, + "grad_norm": 1.146599292755127, + "learning_rate": 0.0001, + "loss": 59.8794, + "step": 7500 + }, + { + "epoch": 121.130081300813, + "grad_norm": 0.9472863078117371, + "learning_rate": 0.0001, + "loss": 55.2454, + "step": 7510 + }, + { + "epoch": 121.29268292682927, + "grad_norm": 0.8604205250740051, + "learning_rate": 0.0001, + "loss": 57.6314, + "step": 7520 + }, + { + "epoch": 121.45528455284553, + "grad_norm": 1.0755794048309326, + "learning_rate": 0.0001, + "loss": 60.909, + "step": 7530 + }, + { + "epoch": 121.6178861788618, + "grad_norm": 0.7967071533203125, + "learning_rate": 0.0001, + "loss": 60.1755, + "step": 7540 + }, + { + "epoch": 121.78048780487805, + "grad_norm": 1.037655234336853, + "learning_rate": 0.0001, + "loss": 59.283, + "step": 7550 + }, + { + "epoch": 121.9430894308943, + "grad_norm": 1.070142388343811, + "learning_rate": 0.0001, + "loss": 57.0941, + "step": 7560 + }, + { + "epoch": 122.09756097560975, + "grad_norm": 1.4201022386550903, + "learning_rate": 0.0001, + "loss": 56.2891, + "step": 7570 + }, + { + "epoch": 122.26016260162602, + "grad_norm": 1.0909277200698853, + "learning_rate": 0.0001, + "loss": 57.5213, + "step": 7580 + }, + { + "epoch": 122.42276422764228, + "grad_norm": 0.7573383450508118, + "learning_rate": 0.0001, + "loss": 59.0847, + "step": 7590 + }, + { + "epoch": 122.58536585365853, + "grad_norm": 0.8931725025177002, + "learning_rate": 0.0001, + "loss": 58.9706, + "step": 7600 + }, + { + "epoch": 122.7479674796748, + "grad_norm": 0.9459935426712036, + "learning_rate": 0.0001, + "loss": 58.6267, + "step": 7610 + }, + { + "epoch": 122.91056910569105, + "grad_norm": 0.9463319778442383, + "learning_rate": 0.0001, + "loss": 60.8536, + "step": 7620 + }, + { + "epoch": 123.0650406504065, + "grad_norm": 0.9286382794380188, + "learning_rate": 0.0001, + "loss": 55.2711, + "step": 7630 + }, + { + "epoch": 123.22764227642277, + "grad_norm": 0.9317494630813599, + "learning_rate": 0.0001, + "loss": 60.1578, + "step": 7640 + }, + { + "epoch": 123.39024390243902, + "grad_norm": 0.9834237098693848, + "learning_rate": 0.0001, + "loss": 58.8121, + "step": 7650 + }, + { + "epoch": 123.55284552845528, + "grad_norm": 0.9992889761924744, + "learning_rate": 0.0001, + "loss": 59.8061, + "step": 7660 + }, + { + "epoch": 123.71544715447155, + "grad_norm": 1.0354453325271606, + "learning_rate": 0.0001, + "loss": 59.5853, + "step": 7670 + }, + { + "epoch": 123.8780487804878, + "grad_norm": 1.0609121322631836, + "learning_rate": 0.0001, + "loss": 59.6556, + "step": 7680 + }, + { + "epoch": 124.03252032520325, + "grad_norm": 0.9742912650108337, + "learning_rate": 0.0001, + "loss": 57.2294, + "step": 7690 + }, + { + "epoch": 124.1951219512195, + "grad_norm": 0.9758710265159607, + "learning_rate": 0.0001, + "loss": 59.8581, + "step": 7700 + }, + { + "epoch": 124.35772357723577, + "grad_norm": 0.9492778182029724, + "learning_rate": 0.0001, + "loss": 60.0624, + "step": 7710 + }, + { + "epoch": 124.52032520325203, + "grad_norm": 0.8311835527420044, + "learning_rate": 0.0001, + "loss": 60.0849, + "step": 7720 + }, + { + "epoch": 124.6829268292683, + "grad_norm": 0.8652944564819336, + "learning_rate": 0.0001, + "loss": 58.2321, + "step": 7730 + }, + { + "epoch": 124.84552845528455, + "grad_norm": 1.2980835437774658, + "learning_rate": 0.0001, + "loss": 57.5315, + "step": 7740 + }, + { + "epoch": 125.0, + "grad_norm": 1.0550237894058228, + "learning_rate": 0.0001, + "loss": 56.9615, + "step": 7750 + }, + { + "epoch": 125.16260162601625, + "grad_norm": 1.1248353719711304, + "learning_rate": 0.0001, + "loss": 58.4853, + "step": 7760 + }, + { + "epoch": 125.32520325203252, + "grad_norm": 1.338152289390564, + "learning_rate": 0.0001, + "loss": 57.8604, + "step": 7770 + }, + { + "epoch": 125.48780487804878, + "grad_norm": 0.8162221908569336, + "learning_rate": 0.0001, + "loss": 56.6396, + "step": 7780 + }, + { + "epoch": 125.65040650406505, + "grad_norm": 1.0431848764419556, + "learning_rate": 0.0001, + "loss": 58.9896, + "step": 7790 + }, + { + "epoch": 125.8130081300813, + "grad_norm": 0.848052978515625, + "learning_rate": 0.0001, + "loss": 57.7459, + "step": 7800 + }, + { + "epoch": 125.97560975609755, + "grad_norm": 0.8441061973571777, + "learning_rate": 0.0001, + "loss": 57.9488, + "step": 7810 + }, + { + "epoch": 126.130081300813, + "grad_norm": 0.774566650390625, + "learning_rate": 0.0001, + "loss": 55.9949, + "step": 7820 + }, + { + "epoch": 126.29268292682927, + "grad_norm": 1.178572416305542, + "learning_rate": 0.0001, + "loss": 58.3554, + "step": 7830 + }, + { + "epoch": 126.45528455284553, + "grad_norm": 0.9363889098167419, + "learning_rate": 0.0001, + "loss": 60.6901, + "step": 7840 + }, + { + "epoch": 126.6178861788618, + "grad_norm": 1.0010961294174194, + "learning_rate": 0.0001, + "loss": 57.0078, + "step": 7850 + }, + { + "epoch": 126.78048780487805, + "grad_norm": 0.9573602080345154, + "learning_rate": 0.0001, + "loss": 58.0567, + "step": 7860 + }, + { + "epoch": 126.9430894308943, + "grad_norm": 1.1903672218322754, + "learning_rate": 0.0001, + "loss": 58.3699, + "step": 7870 + }, + { + "epoch": 127.09756097560975, + "grad_norm": 1.2158106565475464, + "learning_rate": 0.0001, + "loss": 57.1154, + "step": 7880 + }, + { + "epoch": 127.26016260162602, + "grad_norm": 1.198789119720459, + "learning_rate": 0.0001, + "loss": 59.1004, + "step": 7890 + }, + { + "epoch": 127.42276422764228, + "grad_norm": 0.8893532752990723, + "learning_rate": 0.0001, + "loss": 59.6481, + "step": 7900 + }, + { + "epoch": 127.58536585365853, + "grad_norm": 0.9834772348403931, + "learning_rate": 0.0001, + "loss": 57.4394, + "step": 7910 + }, + { + "epoch": 127.7479674796748, + "grad_norm": 1.1957768201828003, + "learning_rate": 0.0001, + "loss": 59.1622, + "step": 7920 + }, + { + "epoch": 127.91056910569105, + "grad_norm": 0.822515070438385, + "learning_rate": 0.0001, + "loss": 58.4967, + "step": 7930 + }, + { + "epoch": 128.0650406504065, + "grad_norm": 1.0374864339828491, + "learning_rate": 0.0001, + "loss": 53.9391, + "step": 7940 + }, + { + "epoch": 128.22764227642276, + "grad_norm": 1.1767654418945312, + "learning_rate": 0.0001, + "loss": 59.4305, + "step": 7950 + }, + { + "epoch": 128.390243902439, + "grad_norm": 1.0048503875732422, + "learning_rate": 0.0001, + "loss": 59.7954, + "step": 7960 + }, + { + "epoch": 128.5528455284553, + "grad_norm": 0.9039503931999207, + "learning_rate": 0.0001, + "loss": 58.6945, + "step": 7970 + }, + { + "epoch": 128.71544715447155, + "grad_norm": 1.047418236732483, + "learning_rate": 0.0001, + "loss": 58.6324, + "step": 7980 + }, + { + "epoch": 128.8780487804878, + "grad_norm": 1.7829573154449463, + "learning_rate": 0.0001, + "loss": 57.3699, + "step": 7990 + }, + { + "epoch": 129.03252032520325, + "grad_norm": 0.8874698281288147, + "learning_rate": 0.0001, + "loss": 57.8226, + "step": 8000 + }, + { + "epoch": 129.1951219512195, + "grad_norm": 0.8585357069969177, + "learning_rate": 0.0001, + "loss": 59.1828, + "step": 8010 + }, + { + "epoch": 129.35772357723576, + "grad_norm": 0.7905164957046509, + "learning_rate": 0.0001, + "loss": 58.4257, + "step": 8020 + }, + { + "epoch": 129.52032520325204, + "grad_norm": 1.15513014793396, + "learning_rate": 0.0001, + "loss": 57.712, + "step": 8030 + }, + { + "epoch": 129.6829268292683, + "grad_norm": 1.4509782791137695, + "learning_rate": 0.0001, + "loss": 59.965, + "step": 8040 + }, + { + "epoch": 129.84552845528455, + "grad_norm": 0.8998057246208191, + "learning_rate": 0.0001, + "loss": 58.6614, + "step": 8050 + }, + { + "epoch": 130.0, + "grad_norm": 1.3381586074829102, + "learning_rate": 0.0001, + "loss": 55.2756, + "step": 8060 + }, + { + "epoch": 130.16260162601625, + "grad_norm": 0.9927449226379395, + "learning_rate": 0.0001, + "loss": 59.6809, + "step": 8070 + }, + { + "epoch": 130.3252032520325, + "grad_norm": 0.9900999665260315, + "learning_rate": 0.0001, + "loss": 60.0017, + "step": 8080 + }, + { + "epoch": 130.4878048780488, + "grad_norm": 0.8317362070083618, + "learning_rate": 0.0001, + "loss": 58.2334, + "step": 8090 + }, + { + "epoch": 130.65040650406505, + "grad_norm": 1.0413036346435547, + "learning_rate": 0.0001, + "loss": 58.987, + "step": 8100 + }, + { + "epoch": 130.8130081300813, + "grad_norm": 1.0915170907974243, + "learning_rate": 0.0001, + "loss": 58.9502, + "step": 8110 + }, + { + "epoch": 130.97560975609755, + "grad_norm": 0.9717564582824707, + "learning_rate": 0.0001, + "loss": 58.6578, + "step": 8120 + }, + { + "epoch": 131.130081300813, + "grad_norm": 0.9847933650016785, + "learning_rate": 0.0001, + "loss": 55.0067, + "step": 8130 + }, + { + "epoch": 131.29268292682926, + "grad_norm": 1.2512580156326294, + "learning_rate": 0.0001, + "loss": 56.0223, + "step": 8140 + }, + { + "epoch": 131.45528455284554, + "grad_norm": 0.8859831094741821, + "learning_rate": 0.0001, + "loss": 57.8123, + "step": 8150 + }, + { + "epoch": 131.6178861788618, + "grad_norm": 1.1545639038085938, + "learning_rate": 0.0001, + "loss": 60.2219, + "step": 8160 + }, + { + "epoch": 131.78048780487805, + "grad_norm": 1.3861064910888672, + "learning_rate": 0.0001, + "loss": 60.2156, + "step": 8170 + }, + { + "epoch": 131.9430894308943, + "grad_norm": 1.2621673345565796, + "learning_rate": 0.0001, + "loss": 58.6422, + "step": 8180 + }, + { + "epoch": 132.09756097560975, + "grad_norm": 1.2075496912002563, + "learning_rate": 0.0001, + "loss": 55.4202, + "step": 8190 + }, + { + "epoch": 132.260162601626, + "grad_norm": 1.1960432529449463, + "learning_rate": 0.0001, + "loss": 59.6405, + "step": 8200 + }, + { + "epoch": 132.4227642276423, + "grad_norm": 1.683369755744934, + "learning_rate": 0.0001, + "loss": 60.1361, + "step": 8210 + }, + { + "epoch": 132.58536585365854, + "grad_norm": 0.9033818244934082, + "learning_rate": 0.0001, + "loss": 58.2383, + "step": 8220 + }, + { + "epoch": 132.7479674796748, + "grad_norm": 1.0529452562332153, + "learning_rate": 0.0001, + "loss": 58.9644, + "step": 8230 + }, + { + "epoch": 132.91056910569105, + "grad_norm": 1.0898828506469727, + "learning_rate": 0.0001, + "loss": 57.0898, + "step": 8240 + }, + { + "epoch": 133.0650406504065, + "grad_norm": 1.0203288793563843, + "learning_rate": 0.0001, + "loss": 56.8206, + "step": 8250 + }, + { + "epoch": 133.22764227642276, + "grad_norm": 0.9973819255828857, + "learning_rate": 0.0001, + "loss": 59.2097, + "step": 8260 + }, + { + "epoch": 133.390243902439, + "grad_norm": 0.9728502035140991, + "learning_rate": 0.0001, + "loss": 57.9959, + "step": 8270 + }, + { + "epoch": 133.5528455284553, + "grad_norm": 0.8645833134651184, + "learning_rate": 0.0001, + "loss": 58.9452, + "step": 8280 + }, + { + "epoch": 133.71544715447155, + "grad_norm": 1.0179742574691772, + "learning_rate": 0.0001, + "loss": 57.8577, + "step": 8290 + }, + { + "epoch": 133.8780487804878, + "grad_norm": 0.8542449474334717, + "learning_rate": 0.0001, + "loss": 57.5798, + "step": 8300 + }, + { + "epoch": 134.03252032520325, + "grad_norm": 0.9958596229553223, + "learning_rate": 0.0001, + "loss": 56.6277, + "step": 8310 + }, + { + "epoch": 134.1951219512195, + "grad_norm": 1.1476788520812988, + "learning_rate": 0.0001, + "loss": 60.8397, + "step": 8320 + }, + { + "epoch": 134.35772357723576, + "grad_norm": 0.9098069071769714, + "learning_rate": 0.0001, + "loss": 59.6647, + "step": 8330 + }, + { + "epoch": 134.52032520325204, + "grad_norm": 0.8126556277275085, + "learning_rate": 0.0001, + "loss": 57.147, + "step": 8340 + }, + { + "epoch": 134.6829268292683, + "grad_norm": 1.4846469163894653, + "learning_rate": 0.0001, + "loss": 58.1245, + "step": 8350 + }, + { + "epoch": 134.84552845528455, + "grad_norm": 1.0117210149765015, + "learning_rate": 0.0001, + "loss": 58.7183, + "step": 8360 + }, + { + "epoch": 135.0, + "grad_norm": 0.9517931938171387, + "learning_rate": 0.0001, + "loss": 57.0423, + "step": 8370 + }, + { + "epoch": 135.16260162601625, + "grad_norm": 1.0269144773483276, + "learning_rate": 0.0001, + "loss": 59.6931, + "step": 8380 + }, + { + "epoch": 135.3252032520325, + "grad_norm": 1.1544915437698364, + "learning_rate": 0.0001, + "loss": 57.4079, + "step": 8390 + }, + { + "epoch": 135.4878048780488, + "grad_norm": 1.1180702447891235, + "learning_rate": 0.0001, + "loss": 57.855, + "step": 8400 + }, + { + "epoch": 135.65040650406505, + "grad_norm": 1.2864829301834106, + "learning_rate": 0.0001, + "loss": 57.677, + "step": 8410 + }, + { + "epoch": 135.8130081300813, + "grad_norm": 1.1786772012710571, + "learning_rate": 0.0001, + "loss": 59.5238, + "step": 8420 + }, + { + "epoch": 135.97560975609755, + "grad_norm": 1.8596229553222656, + "learning_rate": 0.0001, + "loss": 57.1596, + "step": 8430 + }, + { + "epoch": 136.130081300813, + "grad_norm": 0.8087555170059204, + "learning_rate": 0.0001, + "loss": 55.8342, + "step": 8440 + }, + { + "epoch": 136.29268292682926, + "grad_norm": 0.8107356429100037, + "learning_rate": 0.0001, + "loss": 58.5965, + "step": 8450 + }, + { + "epoch": 136.45528455284554, + "grad_norm": 1.2416011095046997, + "learning_rate": 0.0001, + "loss": 57.5909, + "step": 8460 + }, + { + "epoch": 136.6178861788618, + "grad_norm": 0.9763500690460205, + "learning_rate": 0.0001, + "loss": 58.1822, + "step": 8470 + }, + { + "epoch": 136.78048780487805, + "grad_norm": 1.1418349742889404, + "learning_rate": 0.0001, + "loss": 59.3942, + "step": 8480 + }, + { + "epoch": 136.9430894308943, + "grad_norm": 1.206878900527954, + "learning_rate": 0.0001, + "loss": 58.4128, + "step": 8490 + }, + { + "epoch": 137.09756097560975, + "grad_norm": 0.861142098903656, + "learning_rate": 0.0001, + "loss": 55.8831, + "step": 8500 + }, + { + "epoch": 137.260162601626, + "grad_norm": 1.0507313013076782, + "learning_rate": 0.0001, + "loss": 59.755, + "step": 8510 + }, + { + "epoch": 137.4227642276423, + "grad_norm": 0.7070153951644897, + "learning_rate": 0.0001, + "loss": 55.6448, + "step": 8520 + }, + { + "epoch": 137.58536585365854, + "grad_norm": 0.9338926672935486, + "learning_rate": 0.0001, + "loss": 58.4242, + "step": 8530 + }, + { + "epoch": 137.7479674796748, + "grad_norm": 1.0600488185882568, + "learning_rate": 0.0001, + "loss": 59.4947, + "step": 8540 + }, + { + "epoch": 137.91056910569105, + "grad_norm": 1.069604754447937, + "learning_rate": 0.0001, + "loss": 57.5164, + "step": 8550 + }, + { + "epoch": 138.0650406504065, + "grad_norm": 0.872332751750946, + "learning_rate": 0.0001, + "loss": 56.453, + "step": 8560 + }, + { + "epoch": 138.22764227642276, + "grad_norm": 0.9670246243476868, + "learning_rate": 0.0001, + "loss": 59.6467, + "step": 8570 + }, + { + "epoch": 138.390243902439, + "grad_norm": 1.1045281887054443, + "learning_rate": 0.0001, + "loss": 59.3334, + "step": 8580 + }, + { + "epoch": 138.5528455284553, + "grad_norm": 1.0553908348083496, + "learning_rate": 0.0001, + "loss": 59.9081, + "step": 8590 + }, + { + "epoch": 138.71544715447155, + "grad_norm": 0.7342197895050049, + "learning_rate": 0.0001, + "loss": 60.2319, + "step": 8600 + }, + { + "epoch": 138.8780487804878, + "grad_norm": 0.9574354290962219, + "learning_rate": 0.0001, + "loss": 56.7984, + "step": 8610 + }, + { + "epoch": 139.03252032520325, + "grad_norm": 0.9238156676292419, + "learning_rate": 0.0001, + "loss": 56.0007, + "step": 8620 + }, + { + "epoch": 139.1951219512195, + "grad_norm": 1.0704458951950073, + "learning_rate": 0.0001, + "loss": 59.4573, + "step": 8630 + }, + { + "epoch": 139.35772357723576, + "grad_norm": 0.8392401337623596, + "learning_rate": 0.0001, + "loss": 57.7589, + "step": 8640 + }, + { + "epoch": 139.52032520325204, + "grad_norm": 0.9042978286743164, + "learning_rate": 0.0001, + "loss": 59.0548, + "step": 8650 + }, + { + "epoch": 139.6829268292683, + "grad_norm": 2.024853467941284, + "learning_rate": 0.0001, + "loss": 58.0019, + "step": 8660 + }, + { + "epoch": 139.84552845528455, + "grad_norm": 1.9823323488235474, + "learning_rate": 0.0001, + "loss": 57.4398, + "step": 8670 + }, + { + "epoch": 140.0, + "grad_norm": 1.626451849937439, + "learning_rate": 0.0001, + "loss": 54.969, + "step": 8680 + }, + { + "epoch": 140.16260162601625, + "grad_norm": 2.435973644256592, + "learning_rate": 0.0001, + "loss": 57.8945, + "step": 8690 + }, + { + "epoch": 140.3252032520325, + "grad_norm": 1.0417332649230957, + "learning_rate": 0.0001, + "loss": 59.9172, + "step": 8700 + }, + { + "epoch": 140.4878048780488, + "grad_norm": 1.0246574878692627, + "learning_rate": 0.0001, + "loss": 57.7337, + "step": 8710 + }, + { + "epoch": 140.65040650406505, + "grad_norm": 0.9492414593696594, + "learning_rate": 0.0001, + "loss": 57.6049, + "step": 8720 + }, + { + "epoch": 140.8130081300813, + "grad_norm": 1.054689645767212, + "learning_rate": 0.0001, + "loss": 56.4394, + "step": 8730 + }, + { + "epoch": 140.97560975609755, + "grad_norm": 0.7142108082771301, + "learning_rate": 0.0001, + "loss": 58.0006, + "step": 8740 + }, + { + "epoch": 141.130081300813, + "grad_norm": 1.1387385129928589, + "learning_rate": 0.0001, + "loss": 55.2639, + "step": 8750 + }, + { + "epoch": 141.29268292682926, + "grad_norm": 2.105823040008545, + "learning_rate": 0.0001, + "loss": 59.3938, + "step": 8760 + }, + { + "epoch": 141.45528455284554, + "grad_norm": 0.9188179969787598, + "learning_rate": 0.0001, + "loss": 58.731, + "step": 8770 + }, + { + "epoch": 141.6178861788618, + "grad_norm": 1.043778896331787, + "learning_rate": 0.0001, + "loss": 56.2486, + "step": 8780 + }, + { + "epoch": 141.78048780487805, + "grad_norm": 1.438591718673706, + "learning_rate": 0.0001, + "loss": 58.4228, + "step": 8790 + }, + { + "epoch": 141.9430894308943, + "grad_norm": 1.116957187652588, + "learning_rate": 0.0001, + "loss": 59.2317, + "step": 8800 + }, + { + "epoch": 142.09756097560975, + "grad_norm": 0.8644933104515076, + "learning_rate": 0.0001, + "loss": 55.3905, + "step": 8810 + }, + { + "epoch": 142.260162601626, + "grad_norm": 1.0249485969543457, + "learning_rate": 0.0001, + "loss": 59.0372, + "step": 8820 + }, + { + "epoch": 142.4227642276423, + "grad_norm": 0.9939082264900208, + "learning_rate": 0.0001, + "loss": 57.3947, + "step": 8830 + }, + { + "epoch": 142.58536585365854, + "grad_norm": 0.9836716055870056, + "learning_rate": 0.0001, + "loss": 57.9229, + "step": 8840 + }, + { + "epoch": 142.7479674796748, + "grad_norm": 0.9596876502037048, + "learning_rate": 0.0001, + "loss": 57.0285, + "step": 8850 + }, + { + "epoch": 142.91056910569105, + "grad_norm": 1.052899956703186, + "learning_rate": 0.0001, + "loss": 60.1384, + "step": 8860 + }, + { + "epoch": 143.0650406504065, + "grad_norm": 1.1220297813415527, + "learning_rate": 0.0001, + "loss": 55.3928, + "step": 8870 + }, + { + "epoch": 143.22764227642276, + "grad_norm": 0.8395596742630005, + "learning_rate": 0.0001, + "loss": 57.8185, + "step": 8880 + }, + { + "epoch": 143.390243902439, + "grad_norm": 1.1940062046051025, + "learning_rate": 0.0001, + "loss": 58.0322, + "step": 8890 + }, + { + "epoch": 143.5528455284553, + "grad_norm": 1.1662484407424927, + "learning_rate": 0.0001, + "loss": 58.2746, + "step": 8900 + }, + { + "epoch": 143.71544715447155, + "grad_norm": 0.9028235077857971, + "learning_rate": 0.0001, + "loss": 56.5817, + "step": 8910 + }, + { + "epoch": 143.8780487804878, + "grad_norm": 1.0597296953201294, + "learning_rate": 0.0001, + "loss": 56.8426, + "step": 8920 + }, + { + "epoch": 144.03252032520325, + "grad_norm": 0.9968701004981995, + "learning_rate": 0.0001, + "loss": 54.9727, + "step": 8930 + }, + { + "epoch": 144.1951219512195, + "grad_norm": 0.9306491613388062, + "learning_rate": 0.0001, + "loss": 56.5321, + "step": 8940 + }, + { + "epoch": 144.35772357723576, + "grad_norm": 1.1079609394073486, + "learning_rate": 0.0001, + "loss": 56.9714, + "step": 8950 + }, + { + "epoch": 144.52032520325204, + "grad_norm": 0.9644935131072998, + "learning_rate": 0.0001, + "loss": 58.613, + "step": 8960 + }, + { + "epoch": 144.6829268292683, + "grad_norm": 1.2037341594696045, + "learning_rate": 0.0001, + "loss": 56.7768, + "step": 8970 + }, + { + "epoch": 144.84552845528455, + "grad_norm": 0.7347294092178345, + "learning_rate": 0.0001, + "loss": 58.5653, + "step": 8980 + }, + { + "epoch": 145.0, + "grad_norm": 0.8877835273742676, + "learning_rate": 0.0001, + "loss": 53.7768, + "step": 8990 + }, + { + "epoch": 145.16260162601625, + "grad_norm": 0.9649038910865784, + "learning_rate": 0.0001, + "loss": 57.7391, + "step": 9000 + }, + { + "epoch": 145.3252032520325, + "grad_norm": 1.1804879903793335, + "learning_rate": 0.0001, + "loss": 60.3727, + "step": 9010 + }, + { + "epoch": 145.4878048780488, + "grad_norm": 0.8406301140785217, + "learning_rate": 0.0001, + "loss": 57.9417, + "step": 9020 + }, + { + "epoch": 145.65040650406505, + "grad_norm": 0.9555912613868713, + "learning_rate": 0.0001, + "loss": 58.0936, + "step": 9030 + }, + { + "epoch": 145.8130081300813, + "grad_norm": 1.20796537399292, + "learning_rate": 0.0001, + "loss": 57.0784, + "step": 9040 + }, + { + "epoch": 145.97560975609755, + "grad_norm": 1.0527915954589844, + "learning_rate": 0.0001, + "loss": 56.752, + "step": 9050 + }, + { + "epoch": 146.130081300813, + "grad_norm": 1.0104299783706665, + "learning_rate": 0.0001, + "loss": 56.2158, + "step": 9060 + }, + { + "epoch": 146.29268292682926, + "grad_norm": 1.4884357452392578, + "learning_rate": 0.0001, + "loss": 57.8631, + "step": 9070 + }, + { + "epoch": 146.45528455284554, + "grad_norm": 0.9095257520675659, + "learning_rate": 0.0001, + "loss": 57.6492, + "step": 9080 + }, + { + "epoch": 146.6178861788618, + "grad_norm": 0.8465835452079773, + "learning_rate": 0.0001, + "loss": 57.78, + "step": 9090 + }, + { + "epoch": 146.78048780487805, + "grad_norm": 1.0254603624343872, + "learning_rate": 0.0001, + "loss": 57.7678, + "step": 9100 + }, + { + "epoch": 146.9430894308943, + "grad_norm": 1.0137311220169067, + "learning_rate": 0.0001, + "loss": 58.2813, + "step": 9110 + }, + { + "epoch": 147.09756097560975, + "grad_norm": 0.9816886186599731, + "learning_rate": 0.0001, + "loss": 54.4789, + "step": 9120 + }, + { + "epoch": 147.260162601626, + "grad_norm": 0.9038514494895935, + "learning_rate": 0.0001, + "loss": 58.5366, + "step": 9130 + }, + { + "epoch": 147.4227642276423, + "grad_norm": 0.7649261355400085, + "learning_rate": 0.0001, + "loss": 57.2898, + "step": 9140 + }, + { + "epoch": 147.58536585365854, + "grad_norm": 0.8841894865036011, + "learning_rate": 0.0001, + "loss": 57.9109, + "step": 9150 + }, + { + "epoch": 147.7479674796748, + "grad_norm": 1.0480542182922363, + "learning_rate": 0.0001, + "loss": 56.9625, + "step": 9160 + }, + { + "epoch": 147.91056910569105, + "grad_norm": 0.8710455894470215, + "learning_rate": 0.0001, + "loss": 57.1721, + "step": 9170 + }, + { + "epoch": 148.0650406504065, + "grad_norm": 1.374118447303772, + "learning_rate": 0.0001, + "loss": 55.3243, + "step": 9180 + }, + { + "epoch": 148.22764227642276, + "grad_norm": 1.0879178047180176, + "learning_rate": 0.0001, + "loss": 57.1631, + "step": 9190 + }, + { + "epoch": 148.390243902439, + "grad_norm": 0.8242596387863159, + "learning_rate": 0.0001, + "loss": 56.9993, + "step": 9200 + }, + { + "epoch": 148.5528455284553, + "grad_norm": 1.02505362033844, + "learning_rate": 0.0001, + "loss": 57.9201, + "step": 9210 + }, + { + "epoch": 148.71544715447155, + "grad_norm": 0.7904821038246155, + "learning_rate": 0.0001, + "loss": 59.2553, + "step": 9220 + }, + { + "epoch": 148.8780487804878, + "grad_norm": 1.4904934167861938, + "learning_rate": 0.0001, + "loss": 59.498, + "step": 9230 + }, + { + "epoch": 149.03252032520325, + "grad_norm": 1.4203144311904907, + "learning_rate": 0.0001, + "loss": 54.6357, + "step": 9240 + }, + { + "epoch": 149.1951219512195, + "grad_norm": 0.8256629705429077, + "learning_rate": 0.0001, + "loss": 57.5996, + "step": 9250 + }, + { + "epoch": 149.35772357723576, + "grad_norm": 0.8736788630485535, + "learning_rate": 0.0001, + "loss": 56.8112, + "step": 9260 + }, + { + "epoch": 149.52032520325204, + "grad_norm": 3.20312762260437, + "learning_rate": 0.0001, + "loss": 55.8763, + "step": 9270 + }, + { + "epoch": 149.6829268292683, + "grad_norm": 0.7431173920631409, + "learning_rate": 0.0001, + "loss": 56.7385, + "step": 9280 + }, + { + "epoch": 149.84552845528455, + "grad_norm": 0.8165793418884277, + "learning_rate": 0.0001, + "loss": 57.8441, + "step": 9290 + }, + { + "epoch": 150.0, + "grad_norm": 1.2387259006500244, + "learning_rate": 0.0001, + "loss": 53.47, + "step": 9300 + }, + { + "epoch": 150.16260162601625, + "grad_norm": 0.8952508568763733, + "learning_rate": 0.0001, + "loss": 57.7642, + "step": 9310 + }, + { + "epoch": 150.3252032520325, + "grad_norm": 0.9837163686752319, + "learning_rate": 0.0001, + "loss": 57.2288, + "step": 9320 + }, + { + "epoch": 150.4878048780488, + "grad_norm": 0.9019481539726257, + "learning_rate": 0.0001, + "loss": 57.4239, + "step": 9330 + }, + { + "epoch": 150.65040650406505, + "grad_norm": 1.0404818058013916, + "learning_rate": 0.0001, + "loss": 57.7678, + "step": 9340 + }, + { + "epoch": 150.8130081300813, + "grad_norm": 1.0074576139450073, + "learning_rate": 0.0001, + "loss": 58.1508, + "step": 9350 + }, + { + "epoch": 150.97560975609755, + "grad_norm": 0.9951609969139099, + "learning_rate": 0.0001, + "loss": 56.1586, + "step": 9360 + }, + { + "epoch": 151.130081300813, + "grad_norm": 1.0799294710159302, + "learning_rate": 0.0001, + "loss": 52.2597, + "step": 9370 + }, + { + "epoch": 151.29268292682926, + "grad_norm": 0.9001291394233704, + "learning_rate": 0.0001, + "loss": 58.1814, + "step": 9380 + }, + { + "epoch": 151.45528455284554, + "grad_norm": 1.0388239622116089, + "learning_rate": 0.0001, + "loss": 57.1812, + "step": 9390 + }, + { + "epoch": 151.6178861788618, + "grad_norm": 0.7662557363510132, + "learning_rate": 0.0001, + "loss": 56.087, + "step": 9400 + }, + { + "epoch": 151.78048780487805, + "grad_norm": 1.0790860652923584, + "learning_rate": 0.0001, + "loss": 55.9247, + "step": 9410 + }, + { + "epoch": 151.9430894308943, + "grad_norm": 0.8173624277114868, + "learning_rate": 0.0001, + "loss": 57.5506, + "step": 9420 + }, + { + "epoch": 152.09756097560975, + "grad_norm": 1.0954891443252563, + "learning_rate": 0.0001, + "loss": 54.6799, + "step": 9430 + }, + { + "epoch": 152.260162601626, + "grad_norm": 1.0103733539581299, + "learning_rate": 0.0001, + "loss": 58.5454, + "step": 9440 + }, + { + "epoch": 152.4227642276423, + "grad_norm": 0.6966420412063599, + "learning_rate": 0.0001, + "loss": 57.561, + "step": 9450 + }, + { + "epoch": 152.58536585365854, + "grad_norm": 0.9085037708282471, + "learning_rate": 0.0001, + "loss": 57.1746, + "step": 9460 + }, + { + "epoch": 152.7479674796748, + "grad_norm": 1.0009453296661377, + "learning_rate": 0.0001, + "loss": 56.8, + "step": 9470 + }, + { + "epoch": 152.91056910569105, + "grad_norm": 0.9139633178710938, + "learning_rate": 0.0001, + "loss": 56.8229, + "step": 9480 + }, + { + "epoch": 153.0650406504065, + "grad_norm": 1.4209275245666504, + "learning_rate": 0.0001, + "loss": 55.3074, + "step": 9490 + }, + { + "epoch": 153.22764227642276, + "grad_norm": 1.3912067413330078, + "learning_rate": 0.0001, + "loss": 58.1401, + "step": 9500 + }, + { + "epoch": 153.390243902439, + "grad_norm": 0.8208648562431335, + "learning_rate": 0.0001, + "loss": 55.2981, + "step": 9510 + }, + { + "epoch": 153.5528455284553, + "grad_norm": 1.1563907861709595, + "learning_rate": 0.0001, + "loss": 59.0262, + "step": 9520 + }, + { + "epoch": 153.71544715447155, + "grad_norm": 1.1142042875289917, + "learning_rate": 0.0001, + "loss": 55.5979, + "step": 9530 + }, + { + "epoch": 153.8780487804878, + "grad_norm": 0.8886147141456604, + "learning_rate": 0.0001, + "loss": 58.5592, + "step": 9540 + }, + { + "epoch": 154.03252032520325, + "grad_norm": 0.9479674696922302, + "learning_rate": 0.0001, + "loss": 53.956, + "step": 9550 + }, + { + "epoch": 154.1951219512195, + "grad_norm": 1.0572675466537476, + "learning_rate": 0.0001, + "loss": 59.5914, + "step": 9560 + }, + { + "epoch": 154.35772357723576, + "grad_norm": 0.8284531831741333, + "learning_rate": 0.0001, + "loss": 56.4313, + "step": 9570 + }, + { + "epoch": 154.52032520325204, + "grad_norm": 0.8882488012313843, + "learning_rate": 0.0001, + "loss": 57.707, + "step": 9580 + }, + { + "epoch": 154.6829268292683, + "grad_norm": 0.9115352630615234, + "learning_rate": 0.0001, + "loss": 58.0544, + "step": 9590 + }, + { + "epoch": 154.84552845528455, + "grad_norm": 1.0006799697875977, + "learning_rate": 0.0001, + "loss": 59.4528, + "step": 9600 + }, + { + "epoch": 155.0, + "grad_norm": 0.9925333261489868, + "learning_rate": 0.0001, + "loss": 52.977, + "step": 9610 + }, + { + "epoch": 155.16260162601625, + "grad_norm": 1.423391580581665, + "learning_rate": 0.0001, + "loss": 57.0083, + "step": 9620 + }, + { + "epoch": 155.3252032520325, + "grad_norm": 1.0822738409042358, + "learning_rate": 0.0001, + "loss": 58.187, + "step": 9630 + }, + { + "epoch": 155.4878048780488, + "grad_norm": 0.932689905166626, + "learning_rate": 0.0001, + "loss": 55.6032, + "step": 9640 + }, + { + "epoch": 155.65040650406505, + "grad_norm": 0.895504355430603, + "learning_rate": 0.0001, + "loss": 57.1324, + "step": 9650 + }, + { + "epoch": 155.8130081300813, + "grad_norm": 1.0883748531341553, + "learning_rate": 0.0001, + "loss": 58.8422, + "step": 9660 + }, + { + "epoch": 155.97560975609755, + "grad_norm": 1.0161819458007812, + "learning_rate": 0.0001, + "loss": 56.8333, + "step": 9670 + }, + { + "epoch": 156.130081300813, + "grad_norm": 0.9483168721199036, + "learning_rate": 0.0001, + "loss": 52.8434, + "step": 9680 + }, + { + "epoch": 156.29268292682926, + "grad_norm": 1.0677295923233032, + "learning_rate": 0.0001, + "loss": 57.749, + "step": 9690 + }, + { + "epoch": 156.45528455284554, + "grad_norm": 0.8123661875724792, + "learning_rate": 0.0001, + "loss": 56.0896, + "step": 9700 + }, + { + "epoch": 156.6178861788618, + "grad_norm": 0.7926080822944641, + "learning_rate": 0.0001, + "loss": 57.8899, + "step": 9710 + }, + { + "epoch": 156.78048780487805, + "grad_norm": 1.0433725118637085, + "learning_rate": 0.0001, + "loss": 59.6728, + "step": 9720 + }, + { + "epoch": 156.9430894308943, + "grad_norm": 0.8284909129142761, + "learning_rate": 0.0001, + "loss": 55.6093, + "step": 9730 + }, + { + "epoch": 157.09756097560975, + "grad_norm": 0.9702232480049133, + "learning_rate": 0.0001, + "loss": 55.3595, + "step": 9740 + }, + { + "epoch": 157.260162601626, + "grad_norm": 0.8909348249435425, + "learning_rate": 0.0001, + "loss": 57.6344, + "step": 9750 + }, + { + "epoch": 157.4227642276423, + "grad_norm": 0.9539676308631897, + "learning_rate": 0.0001, + "loss": 55.8826, + "step": 9760 + }, + { + "epoch": 157.58536585365854, + "grad_norm": 1.1801178455352783, + "learning_rate": 0.0001, + "loss": 55.7141, + "step": 9770 + }, + { + "epoch": 157.7479674796748, + "grad_norm": 0.8674126863479614, + "learning_rate": 0.0001, + "loss": 56.0571, + "step": 9780 + }, + { + "epoch": 157.91056910569105, + "grad_norm": 1.0690264701843262, + "learning_rate": 0.0001, + "loss": 56.1485, + "step": 9790 + }, + { + "epoch": 158.0650406504065, + "grad_norm": 0.8807869553565979, + "learning_rate": 0.0001, + "loss": 56.9874, + "step": 9800 + }, + { + "epoch": 158.22764227642276, + "grad_norm": 0.9477714896202087, + "learning_rate": 0.0001, + "loss": 58.2035, + "step": 9810 + }, + { + "epoch": 158.390243902439, + "grad_norm": 1.4875513315200806, + "learning_rate": 0.0001, + "loss": 59.0302, + "step": 9820 + }, + { + "epoch": 158.5528455284553, + "grad_norm": 0.7979354858398438, + "learning_rate": 0.0001, + "loss": 57.2801, + "step": 9830 + }, + { + "epoch": 158.71544715447155, + "grad_norm": 0.8941402435302734, + "learning_rate": 0.0001, + "loss": 56.3615, + "step": 9840 + }, + { + "epoch": 158.8780487804878, + "grad_norm": 0.8475911021232605, + "learning_rate": 0.0001, + "loss": 57.266, + "step": 9850 + }, + { + "epoch": 159.03252032520325, + "grad_norm": 0.8971277475357056, + "learning_rate": 0.0001, + "loss": 52.792, + "step": 9860 + }, + { + "epoch": 159.1951219512195, + "grad_norm": 0.930777907371521, + "learning_rate": 0.0001, + "loss": 57.4098, + "step": 9870 + }, + { + "epoch": 159.35772357723576, + "grad_norm": 0.9232887625694275, + "learning_rate": 0.0001, + "loss": 56.0267, + "step": 9880 + }, + { + "epoch": 159.52032520325204, + "grad_norm": 0.8904425501823425, + "learning_rate": 0.0001, + "loss": 57.6255, + "step": 9890 + }, + { + "epoch": 159.6829268292683, + "grad_norm": 0.9250755906105042, + "learning_rate": 0.0001, + "loss": 56.5248, + "step": 9900 + }, + { + "epoch": 159.84552845528455, + "grad_norm": 1.3746178150177002, + "learning_rate": 0.0001, + "loss": 56.7843, + "step": 9910 + }, + { + "epoch": 160.0, + "grad_norm": 0.9464873671531677, + "learning_rate": 0.0001, + "loss": 54.1325, + "step": 9920 + }, + { + "epoch": 160.16260162601625, + "grad_norm": 0.9019768238067627, + "learning_rate": 0.0001, + "loss": 58.3641, + "step": 9930 + }, + { + "epoch": 160.3252032520325, + "grad_norm": 0.9282474517822266, + "learning_rate": 0.0001, + "loss": 57.9474, + "step": 9940 + }, + { + "epoch": 160.4878048780488, + "grad_norm": 1.1150450706481934, + "learning_rate": 0.0001, + "loss": 58.2999, + "step": 9950 + }, + { + "epoch": 160.65040650406505, + "grad_norm": 0.9780807495117188, + "learning_rate": 0.0001, + "loss": 55.6409, + "step": 9960 + }, + { + "epoch": 160.8130081300813, + "grad_norm": 0.9708426594734192, + "learning_rate": 0.0001, + "loss": 58.651, + "step": 9970 + }, + { + "epoch": 160.97560975609755, + "grad_norm": 0.9891226887702942, + "learning_rate": 0.0001, + "loss": 57.3798, + "step": 9980 + }, + { + "epoch": 161.130081300813, + "grad_norm": 0.8949639797210693, + "learning_rate": 0.0001, + "loss": 53.4708, + "step": 9990 + }, + { + "epoch": 161.29268292682926, + "grad_norm": 0.831688404083252, + "learning_rate": 0.0001, + "loss": 57.1608, + "step": 10000 + }, + { + "epoch": 161.45528455284554, + "grad_norm": 1.1002815961837769, + "learning_rate": 0.0001, + "loss": 57.5943, + "step": 10010 + }, + { + "epoch": 161.6178861788618, + "grad_norm": 0.823810338973999, + "learning_rate": 0.0001, + "loss": 56.3465, + "step": 10020 + }, + { + "epoch": 161.78048780487805, + "grad_norm": 0.8624166250228882, + "learning_rate": 0.0001, + "loss": 58.2043, + "step": 10030 + }, + { + "epoch": 161.9430894308943, + "grad_norm": 0.9643419981002808, + "learning_rate": 0.0001, + "loss": 58.6286, + "step": 10040 + }, + { + "epoch": 162.09756097560975, + "grad_norm": 0.8477599024772644, + "learning_rate": 0.0001, + "loss": 54.0391, + "step": 10050 + }, + { + "epoch": 162.260162601626, + "grad_norm": 0.932232677936554, + "learning_rate": 0.0001, + "loss": 57.3955, + "step": 10060 + }, + { + "epoch": 162.4227642276423, + "grad_norm": 0.9273334741592407, + "learning_rate": 0.0001, + "loss": 57.3421, + "step": 10070 + }, + { + "epoch": 162.58536585365854, + "grad_norm": 0.746417224407196, + "learning_rate": 0.0001, + "loss": 56.1001, + "step": 10080 + }, + { + "epoch": 162.7479674796748, + "grad_norm": 0.9870802760124207, + "learning_rate": 0.0001, + "loss": 57.1234, + "step": 10090 + }, + { + "epoch": 162.91056910569105, + "grad_norm": 1.2378058433532715, + "learning_rate": 0.0001, + "loss": 58.035, + "step": 10100 + }, + { + "epoch": 163.0650406504065, + "grad_norm": 0.8810584545135498, + "learning_rate": 0.0001, + "loss": 52.1406, + "step": 10110 + }, + { + "epoch": 163.22764227642276, + "grad_norm": 1.1010875701904297, + "learning_rate": 0.0001, + "loss": 57.2534, + "step": 10120 + }, + { + "epoch": 163.390243902439, + "grad_norm": 1.1182267665863037, + "learning_rate": 0.0001, + "loss": 58.0887, + "step": 10130 + }, + { + "epoch": 163.5528455284553, + "grad_norm": 0.8677687048912048, + "learning_rate": 0.0001, + "loss": 56.3502, + "step": 10140 + }, + { + "epoch": 163.71544715447155, + "grad_norm": 1.3890044689178467, + "learning_rate": 0.0001, + "loss": 57.1057, + "step": 10150 + }, + { + "epoch": 163.8780487804878, + "grad_norm": 1.340419054031372, + "learning_rate": 0.0001, + "loss": 56.9196, + "step": 10160 + }, + { + "epoch": 164.03252032520325, + "grad_norm": 0.8187946081161499, + "learning_rate": 0.0001, + "loss": 53.388, + "step": 10170 + }, + { + "epoch": 164.1951219512195, + "grad_norm": 0.7856727838516235, + "learning_rate": 0.0001, + "loss": 57.0134, + "step": 10180 + }, + { + "epoch": 164.35772357723576, + "grad_norm": 0.8677143454551697, + "learning_rate": 0.0001, + "loss": 56.1385, + "step": 10190 + }, + { + "epoch": 164.52032520325204, + "grad_norm": 0.8334061503410339, + "learning_rate": 0.0001, + "loss": 58.4353, + "step": 10200 + }, + { + "epoch": 164.6829268292683, + "grad_norm": 0.9447886943817139, + "learning_rate": 0.0001, + "loss": 56.2007, + "step": 10210 + }, + { + "epoch": 164.84552845528455, + "grad_norm": 0.7787997722625732, + "learning_rate": 0.0001, + "loss": 56.7128, + "step": 10220 + }, + { + "epoch": 165.0, + "grad_norm": 0.864169716835022, + "learning_rate": 0.0001, + "loss": 53.6559, + "step": 10230 + }, + { + "epoch": 165.16260162601625, + "grad_norm": 0.9252000451087952, + "learning_rate": 0.0001, + "loss": 57.1013, + "step": 10240 + }, + { + "epoch": 165.3252032520325, + "grad_norm": 0.9552235007286072, + "learning_rate": 0.0001, + "loss": 58.1834, + "step": 10250 + }, + { + "epoch": 165.4878048780488, + "grad_norm": 1.0003581047058105, + "learning_rate": 0.0001, + "loss": 56.3756, + "step": 10260 + }, + { + "epoch": 165.65040650406505, + "grad_norm": 1.1489440202713013, + "learning_rate": 0.0001, + "loss": 54.8963, + "step": 10270 + }, + { + "epoch": 165.8130081300813, + "grad_norm": 0.9719933867454529, + "learning_rate": 0.0001, + "loss": 59.2853, + "step": 10280 + }, + { + "epoch": 165.97560975609755, + "grad_norm": 0.888495147228241, + "learning_rate": 0.0001, + "loss": 56.7793, + "step": 10290 + }, + { + "epoch": 166.130081300813, + "grad_norm": 0.793580174446106, + "learning_rate": 0.0001, + "loss": 53.6792, + "step": 10300 + }, + { + "epoch": 166.29268292682926, + "grad_norm": 0.928779125213623, + "learning_rate": 0.0001, + "loss": 57.5329, + "step": 10310 + }, + { + "epoch": 166.45528455284554, + "grad_norm": 1.2024686336517334, + "learning_rate": 0.0001, + "loss": 58.08, + "step": 10320 + }, + { + "epoch": 166.6178861788618, + "grad_norm": 1.421026349067688, + "learning_rate": 0.0001, + "loss": 57.5318, + "step": 10330 + }, + { + "epoch": 166.78048780487805, + "grad_norm": 0.9189603328704834, + "learning_rate": 0.0001, + "loss": 57.1355, + "step": 10340 + }, + { + "epoch": 166.9430894308943, + "grad_norm": 1.0614054203033447, + "learning_rate": 0.0001, + "loss": 56.0604, + "step": 10350 + }, + { + "epoch": 167.09756097560975, + "grad_norm": 1.0266008377075195, + "learning_rate": 0.0001, + "loss": 55.0443, + "step": 10360 + }, + { + "epoch": 167.260162601626, + "grad_norm": 0.8291250467300415, + "learning_rate": 0.0001, + "loss": 57.6836, + "step": 10370 + }, + { + "epoch": 167.4227642276423, + "grad_norm": 1.0415455102920532, + "learning_rate": 0.0001, + "loss": 56.1081, + "step": 10380 + }, + { + "epoch": 167.58536585365854, + "grad_norm": 0.8045644164085388, + "learning_rate": 0.0001, + "loss": 54.8943, + "step": 10390 + }, + { + "epoch": 167.7479674796748, + "grad_norm": 0.9025611281394958, + "learning_rate": 0.0001, + "loss": 56.5143, + "step": 10400 + }, + { + "epoch": 167.91056910569105, + "grad_norm": 0.93849116563797, + "learning_rate": 0.0001, + "loss": 57.4787, + "step": 10410 + }, + { + "epoch": 168.0650406504065, + "grad_norm": 0.8948699235916138, + "learning_rate": 0.0001, + "loss": 54.9249, + "step": 10420 + }, + { + "epoch": 168.22764227642276, + "grad_norm": 0.7900148630142212, + "learning_rate": 0.0001, + "loss": 55.9813, + "step": 10430 + }, + { + "epoch": 168.390243902439, + "grad_norm": 1.1853744983673096, + "learning_rate": 0.0001, + "loss": 54.9089, + "step": 10440 + }, + { + "epoch": 168.5528455284553, + "grad_norm": 1.225123643875122, + "learning_rate": 0.0001, + "loss": 58.238, + "step": 10450 + }, + { + "epoch": 168.71544715447155, + "grad_norm": 0.8648725748062134, + "learning_rate": 0.0001, + "loss": 55.4905, + "step": 10460 + }, + { + "epoch": 168.8780487804878, + "grad_norm": 0.9054272770881653, + "learning_rate": 0.0001, + "loss": 58.2697, + "step": 10470 + }, + { + "epoch": 169.03252032520325, + "grad_norm": 0.946158230304718, + "learning_rate": 0.0001, + "loss": 53.3644, + "step": 10480 + }, + { + "epoch": 169.1951219512195, + "grad_norm": 0.8751044273376465, + "learning_rate": 0.0001, + "loss": 54.3469, + "step": 10490 + }, + { + "epoch": 169.35772357723576, + "grad_norm": 0.9814269542694092, + "learning_rate": 0.0001, + "loss": 56.5358, + "step": 10500 + }, + { + "epoch": 169.52032520325204, + "grad_norm": 0.8131198883056641, + "learning_rate": 0.0001, + "loss": 56.2025, + "step": 10510 + }, + { + "epoch": 169.6829268292683, + "grad_norm": 0.7442125678062439, + "learning_rate": 0.0001, + "loss": 54.7679, + "step": 10520 + }, + { + "epoch": 169.84552845528455, + "grad_norm": 0.8504229187965393, + "learning_rate": 0.0001, + "loss": 59.024, + "step": 10530 + }, + { + "epoch": 170.0, + "grad_norm": 0.8483893871307373, + "learning_rate": 0.0001, + "loss": 53.8021, + "step": 10540 + }, + { + "epoch": 170.16260162601625, + "grad_norm": 0.8889706134796143, + "learning_rate": 0.0001, + "loss": 55.1988, + "step": 10550 + }, + { + "epoch": 170.3252032520325, + "grad_norm": 0.9083985686302185, + "learning_rate": 0.0001, + "loss": 57.092, + "step": 10560 + }, + { + "epoch": 170.4878048780488, + "grad_norm": 27.54094123840332, + "learning_rate": 0.0001, + "loss": 56.945, + "step": 10570 + }, + { + "epoch": 170.65040650406505, + "grad_norm": 0.6707919836044312, + "learning_rate": 0.0001, + "loss": 55.8705, + "step": 10580 + }, + { + "epoch": 170.8130081300813, + "grad_norm": 1.0104289054870605, + "learning_rate": 0.0001, + "loss": 56.6519, + "step": 10590 + }, + { + "epoch": 170.97560975609755, + "grad_norm": 1.221200704574585, + "learning_rate": 0.0001, + "loss": 57.0875, + "step": 10600 + }, + { + "epoch": 171.130081300813, + "grad_norm": 1.3986961841583252, + "learning_rate": 0.0001, + "loss": 53.9815, + "step": 10610 + }, + { + "epoch": 171.29268292682926, + "grad_norm": 1.0881519317626953, + "learning_rate": 0.0001, + "loss": 55.8683, + "step": 10620 + }, + { + "epoch": 171.45528455284554, + "grad_norm": 1.172620415687561, + "learning_rate": 0.0001, + "loss": 56.7678, + "step": 10630 + }, + { + "epoch": 171.6178861788618, + "grad_norm": 0.867314875125885, + "learning_rate": 0.0001, + "loss": 55.0647, + "step": 10640 + }, + { + "epoch": 171.78048780487805, + "grad_norm": 1.415421962738037, + "learning_rate": 0.0001, + "loss": 57.5599, + "step": 10650 + }, + { + "epoch": 171.9430894308943, + "grad_norm": 1.0275741815567017, + "learning_rate": 0.0001, + "loss": 55.4741, + "step": 10660 + }, + { + "epoch": 172.09756097560975, + "grad_norm": 0.9037189483642578, + "learning_rate": 0.0001, + "loss": 51.9268, + "step": 10670 + }, + { + "epoch": 172.260162601626, + "grad_norm": 0.9657711386680603, + "learning_rate": 0.0001, + "loss": 57.3183, + "step": 10680 + }, + { + "epoch": 172.4227642276423, + "grad_norm": 0.9031921029090881, + "learning_rate": 0.0001, + "loss": 58.6986, + "step": 10690 + }, + { + "epoch": 172.58536585365854, + "grad_norm": 0.9555232524871826, + "learning_rate": 0.0001, + "loss": 55.7602, + "step": 10700 + }, + { + "epoch": 172.7479674796748, + "grad_norm": 0.985213041305542, + "learning_rate": 0.0001, + "loss": 55.3734, + "step": 10710 + }, + { + "epoch": 172.91056910569105, + "grad_norm": 1.1574981212615967, + "learning_rate": 0.0001, + "loss": 57.8304, + "step": 10720 + }, + { + "epoch": 173.0650406504065, + "grad_norm": 1.0298277139663696, + "learning_rate": 0.0001, + "loss": 54.2185, + "step": 10730 + }, + { + "epoch": 173.22764227642276, + "grad_norm": 1.0708621740341187, + "learning_rate": 0.0001, + "loss": 57.1618, + "step": 10740 + }, + { + "epoch": 173.390243902439, + "grad_norm": 1.0337039232254028, + "learning_rate": 0.0001, + "loss": 56.8093, + "step": 10750 + }, + { + "epoch": 173.5528455284553, + "grad_norm": 1.1115336418151855, + "learning_rate": 0.0001, + "loss": 56.1233, + "step": 10760 + }, + { + "epoch": 173.71544715447155, + "grad_norm": 0.9465245008468628, + "learning_rate": 0.0001, + "loss": 57.79, + "step": 10770 + }, + { + "epoch": 173.8780487804878, + "grad_norm": 1.0282483100891113, + "learning_rate": 0.0001, + "loss": 56.7261, + "step": 10780 + }, + { + "epoch": 174.03252032520325, + "grad_norm": 0.8371811509132385, + "learning_rate": 0.0001, + "loss": 54.3208, + "step": 10790 + }, + { + "epoch": 174.1951219512195, + "grad_norm": 0.75701504945755, + "learning_rate": 0.0001, + "loss": 58.4083, + "step": 10800 + }, + { + "epoch": 174.35772357723576, + "grad_norm": 0.8716368675231934, + "learning_rate": 0.0001, + "loss": 56.3668, + "step": 10810 + }, + { + "epoch": 174.52032520325204, + "grad_norm": 0.8204325437545776, + "learning_rate": 0.0001, + "loss": 57.5088, + "step": 10820 + }, + { + "epoch": 174.6829268292683, + "grad_norm": 0.912240743637085, + "learning_rate": 0.0001, + "loss": 54.392, + "step": 10830 + }, + { + "epoch": 174.84552845528455, + "grad_norm": 1.0001140832901, + "learning_rate": 0.0001, + "loss": 56.2384, + "step": 10840 + }, + { + "epoch": 175.0, + "grad_norm": 1.1657922267913818, + "learning_rate": 0.0001, + "loss": 53.9591, + "step": 10850 + }, + { + "epoch": 175.16260162601625, + "grad_norm": 0.9120311141014099, + "learning_rate": 0.0001, + "loss": 56.7532, + "step": 10860 + }, + { + "epoch": 175.3252032520325, + "grad_norm": 0.821774423122406, + "learning_rate": 0.0001, + "loss": 56.1124, + "step": 10870 + }, + { + "epoch": 175.4878048780488, + "grad_norm": 0.8408815264701843, + "learning_rate": 0.0001, + "loss": 56.8947, + "step": 10880 + }, + { + "epoch": 175.65040650406505, + "grad_norm": 1.2977283000946045, + "learning_rate": 0.0001, + "loss": 55.476, + "step": 10890 + }, + { + "epoch": 175.8130081300813, + "grad_norm": 0.8232496380805969, + "learning_rate": 0.0001, + "loss": 56.0607, + "step": 10900 + }, + { + "epoch": 175.97560975609755, + "grad_norm": 0.8235589265823364, + "learning_rate": 0.0001, + "loss": 56.3365, + "step": 10910 + }, + { + "epoch": 176.130081300813, + "grad_norm": 1.0942591428756714, + "learning_rate": 0.0001, + "loss": 53.065, + "step": 10920 + }, + { + "epoch": 176.29268292682926, + "grad_norm": 0.9026378989219666, + "learning_rate": 0.0001, + "loss": 55.6453, + "step": 10930 + }, + { + "epoch": 176.45528455284554, + "grad_norm": 0.9491869211196899, + "learning_rate": 0.0001, + "loss": 56.4853, + "step": 10940 + }, + { + "epoch": 176.6178861788618, + "grad_norm": 1.072492241859436, + "learning_rate": 0.0001, + "loss": 55.7488, + "step": 10950 + }, + { + "epoch": 176.78048780487805, + "grad_norm": 0.9384275674819946, + "learning_rate": 0.0001, + "loss": 56.8056, + "step": 10960 + }, + { + "epoch": 176.9430894308943, + "grad_norm": 0.9458582997322083, + "learning_rate": 0.0001, + "loss": 57.187, + "step": 10970 + }, + { + "epoch": 177.09756097560975, + "grad_norm": 0.9112198352813721, + "learning_rate": 0.0001, + "loss": 52.9576, + "step": 10980 + }, + { + "epoch": 177.260162601626, + "grad_norm": 1.0807278156280518, + "learning_rate": 0.0001, + "loss": 56.0773, + "step": 10990 + }, + { + "epoch": 177.4227642276423, + "grad_norm": 0.8122154474258423, + "learning_rate": 0.0001, + "loss": 56.0297, + "step": 11000 + }, + { + "epoch": 177.58536585365854, + "grad_norm": 0.7836926579475403, + "learning_rate": 0.0001, + "loss": 56.74, + "step": 11010 + }, + { + "epoch": 177.7479674796748, + "grad_norm": 1.0483779907226562, + "learning_rate": 0.0001, + "loss": 55.1174, + "step": 11020 + }, + { + "epoch": 177.91056910569105, + "grad_norm": 0.8454721570014954, + "learning_rate": 0.0001, + "loss": 55.3519, + "step": 11030 + }, + { + "epoch": 178.0650406504065, + "grad_norm": 1.2100988626480103, + "learning_rate": 0.0001, + "loss": 54.5955, + "step": 11040 + }, + { + "epoch": 178.22764227642276, + "grad_norm": 1.0322265625, + "learning_rate": 0.0001, + "loss": 57.2623, + "step": 11050 + }, + { + "epoch": 178.390243902439, + "grad_norm": 0.8880140781402588, + "learning_rate": 0.0001, + "loss": 56.5935, + "step": 11060 + }, + { + "epoch": 178.5528455284553, + "grad_norm": 0.8864359259605408, + "learning_rate": 0.0001, + "loss": 56.802, + "step": 11070 + }, + { + "epoch": 178.71544715447155, + "grad_norm": 1.1736643314361572, + "learning_rate": 0.0001, + "loss": 56.8208, + "step": 11080 + }, + { + "epoch": 178.8780487804878, + "grad_norm": 0.975029706954956, + "learning_rate": 0.0001, + "loss": 57.5741, + "step": 11090 + }, + { + "epoch": 179.03252032520325, + "grad_norm": 0.852942168712616, + "learning_rate": 0.0001, + "loss": 54.327, + "step": 11100 + }, + { + "epoch": 179.1951219512195, + "grad_norm": 1.1381421089172363, + "learning_rate": 0.0001, + "loss": 56.0875, + "step": 11110 + }, + { + "epoch": 179.35772357723576, + "grad_norm": 1.0923271179199219, + "learning_rate": 0.0001, + "loss": 58.1302, + "step": 11120 + }, + { + "epoch": 179.52032520325204, + "grad_norm": 0.9422796964645386, + "learning_rate": 0.0001, + "loss": 56.2919, + "step": 11130 + }, + { + "epoch": 179.6829268292683, + "grad_norm": 0.9511571526527405, + "learning_rate": 0.0001, + "loss": 55.1329, + "step": 11140 + }, + { + "epoch": 179.84552845528455, + "grad_norm": 0.9221625328063965, + "learning_rate": 0.0001, + "loss": 56.6727, + "step": 11150 + }, + { + "epoch": 180.0, + "grad_norm": 0.8903642296791077, + "learning_rate": 0.0001, + "loss": 53.7472, + "step": 11160 + }, + { + "epoch": 180.16260162601625, + "grad_norm": 0.9752726554870605, + "learning_rate": 0.0001, + "loss": 56.2322, + "step": 11170 + }, + { + "epoch": 180.3252032520325, + "grad_norm": 0.8058391809463501, + "learning_rate": 0.0001, + "loss": 57.7904, + "step": 11180 + }, + { + "epoch": 180.4878048780488, + "grad_norm": 0.9203750491142273, + "learning_rate": 0.0001, + "loss": 55.3608, + "step": 11190 + }, + { + "epoch": 180.65040650406505, + "grad_norm": 1.4366613626480103, + "learning_rate": 0.0001, + "loss": 54.5204, + "step": 11200 + }, + { + "epoch": 180.8130081300813, + "grad_norm": 1.040142297744751, + "learning_rate": 0.0001, + "loss": 56.3904, + "step": 11210 + }, + { + "epoch": 180.97560975609755, + "grad_norm": 1.1629812717437744, + "learning_rate": 0.0001, + "loss": 55.4976, + "step": 11220 + }, + { + "epoch": 181.130081300813, + "grad_norm": 0.847724437713623, + "learning_rate": 0.0001, + "loss": 54.4277, + "step": 11230 + }, + { + "epoch": 181.29268292682926, + "grad_norm": 1.0196763277053833, + "learning_rate": 0.0001, + "loss": 58.1868, + "step": 11240 + }, + { + "epoch": 181.45528455284554, + "grad_norm": 1.2344937324523926, + "learning_rate": 0.0001, + "loss": 56.1104, + "step": 11250 + }, + { + "epoch": 181.6178861788618, + "grad_norm": 0.8804627060890198, + "learning_rate": 0.0001, + "loss": 55.728, + "step": 11260 + }, + { + "epoch": 181.78048780487805, + "grad_norm": 0.8948118686676025, + "learning_rate": 0.0001, + "loss": 56.9878, + "step": 11270 + }, + { + "epoch": 181.9430894308943, + "grad_norm": 0.6648201942443848, + "learning_rate": 0.0001, + "loss": 56.5425, + "step": 11280 + }, + { + "epoch": 182.09756097560975, + "grad_norm": 0.8076626658439636, + "learning_rate": 0.0001, + "loss": 53.4703, + "step": 11290 + }, + { + "epoch": 182.260162601626, + "grad_norm": 0.9063478708267212, + "learning_rate": 0.0001, + "loss": 57.6056, + "step": 11300 + }, + { + "epoch": 182.4227642276423, + "grad_norm": 0.8387530446052551, + "learning_rate": 0.0001, + "loss": 56.0165, + "step": 11310 + }, + { + "epoch": 182.58536585365854, + "grad_norm": 0.8866046667098999, + "learning_rate": 0.0001, + "loss": 57.774, + "step": 11320 + }, + { + "epoch": 182.7479674796748, + "grad_norm": 0.9022682905197144, + "learning_rate": 0.0001, + "loss": 57.5811, + "step": 11330 + }, + { + "epoch": 182.91056910569105, + "grad_norm": 0.8135976195335388, + "learning_rate": 0.0001, + "loss": 56.048, + "step": 11340 + }, + { + "epoch": 183.0650406504065, + "grad_norm": 1.0014373064041138, + "learning_rate": 0.0001, + "loss": 52.5195, + "step": 11350 + }, + { + "epoch": 183.22764227642276, + "grad_norm": 1.0275895595550537, + "learning_rate": 0.0001, + "loss": 57.1421, + "step": 11360 + }, + { + "epoch": 183.390243902439, + "grad_norm": 0.8826107382774353, + "learning_rate": 0.0001, + "loss": 57.5582, + "step": 11370 + }, + { + "epoch": 183.5528455284553, + "grad_norm": 0.9023060202598572, + "learning_rate": 0.0001, + "loss": 56.6695, + "step": 11380 + }, + { + "epoch": 183.71544715447155, + "grad_norm": 0.7782049179077148, + "learning_rate": 0.0001, + "loss": 55.4362, + "step": 11390 + }, + { + "epoch": 183.8780487804878, + "grad_norm": 0.9904818534851074, + "learning_rate": 0.0001, + "loss": 56.7164, + "step": 11400 + }, + { + "epoch": 184.03252032520325, + "grad_norm": 0.9218631982803345, + "learning_rate": 0.0001, + "loss": 53.8536, + "step": 11410 + }, + { + "epoch": 184.1951219512195, + "grad_norm": 0.7807294130325317, + "learning_rate": 0.0001, + "loss": 57.1502, + "step": 11420 + }, + { + "epoch": 184.35772357723576, + "grad_norm": 0.8353742957115173, + "learning_rate": 0.0001, + "loss": 56.8276, + "step": 11430 + }, + { + "epoch": 184.52032520325204, + "grad_norm": 0.7909665107727051, + "learning_rate": 0.0001, + "loss": 56.4164, + "step": 11440 + }, + { + "epoch": 184.6829268292683, + "grad_norm": 1.0313756465911865, + "learning_rate": 0.0001, + "loss": 56.1275, + "step": 11450 + }, + { + "epoch": 184.84552845528455, + "grad_norm": 0.9377022385597229, + "learning_rate": 0.0001, + "loss": 55.9829, + "step": 11460 + }, + { + "epoch": 185.0, + "grad_norm": 1.2855385541915894, + "learning_rate": 0.0001, + "loss": 53.6264, + "step": 11470 + }, + { + "epoch": 185.16260162601625, + "grad_norm": 1.7889217138290405, + "learning_rate": 0.0001, + "loss": 55.1279, + "step": 11480 + }, + { + "epoch": 185.3252032520325, + "grad_norm": 0.8415203094482422, + "learning_rate": 0.0001, + "loss": 57.1529, + "step": 11490 + }, + { + "epoch": 185.4878048780488, + "grad_norm": 1.1245211362838745, + "learning_rate": 0.0001, + "loss": 56.2543, + "step": 11500 + }, + { + "epoch": 185.65040650406505, + "grad_norm": 1.607924222946167, + "learning_rate": 0.0001, + "loss": 55.9007, + "step": 11510 + }, + { + "epoch": 185.8130081300813, + "grad_norm": 0.8068832755088806, + "learning_rate": 0.0001, + "loss": 56.5567, + "step": 11520 + }, + { + "epoch": 185.97560975609755, + "grad_norm": 1.109636902809143, + "learning_rate": 0.0001, + "loss": 55.8676, + "step": 11530 + }, + { + "epoch": 186.130081300813, + "grad_norm": 0.9668437242507935, + "learning_rate": 0.0001, + "loss": 52.9045, + "step": 11540 + }, + { + "epoch": 186.29268292682926, + "grad_norm": 1.1277968883514404, + "learning_rate": 0.0001, + "loss": 55.0794, + "step": 11550 + }, + { + "epoch": 186.45528455284554, + "grad_norm": 0.6413527727127075, + "learning_rate": 0.0001, + "loss": 55.4634, + "step": 11560 + }, + { + "epoch": 186.6178861788618, + "grad_norm": 1.039292812347412, + "learning_rate": 0.0001, + "loss": 56.2108, + "step": 11570 + }, + { + "epoch": 186.78048780487805, + "grad_norm": 0.9112428426742554, + "learning_rate": 0.0001, + "loss": 56.4036, + "step": 11580 + }, + { + "epoch": 186.9430894308943, + "grad_norm": 0.8918914794921875, + "learning_rate": 0.0001, + "loss": 54.9626, + "step": 11590 + }, + { + "epoch": 187.09756097560975, + "grad_norm": 0.9715940952301025, + "learning_rate": 0.0001, + "loss": 54.0208, + "step": 11600 + }, + { + "epoch": 187.260162601626, + "grad_norm": 0.8578653931617737, + "learning_rate": 0.0001, + "loss": 56.4479, + "step": 11610 + }, + { + "epoch": 187.4227642276423, + "grad_norm": 1.7716244459152222, + "learning_rate": 0.0001, + "loss": 55.3543, + "step": 11620 + }, + { + "epoch": 187.58536585365854, + "grad_norm": 0.9152880311012268, + "learning_rate": 0.0001, + "loss": 56.8464, + "step": 11630 + }, + { + "epoch": 187.7479674796748, + "grad_norm": 0.8824559450149536, + "learning_rate": 0.0001, + "loss": 55.9501, + "step": 11640 + }, + { + "epoch": 187.91056910569105, + "grad_norm": 0.780864953994751, + "learning_rate": 0.0001, + "loss": 57.3594, + "step": 11650 + }, + { + "epoch": 188.0650406504065, + "grad_norm": 0.7242524027824402, + "learning_rate": 0.0001, + "loss": 53.9958, + "step": 11660 + }, + { + "epoch": 188.22764227642276, + "grad_norm": 0.9025135636329651, + "learning_rate": 0.0001, + "loss": 54.6309, + "step": 11670 + }, + { + "epoch": 188.390243902439, + "grad_norm": 0.7995520830154419, + "learning_rate": 0.0001, + "loss": 56.7882, + "step": 11680 + }, + { + "epoch": 188.5528455284553, + "grad_norm": 0.8525217175483704, + "learning_rate": 0.0001, + "loss": 54.7559, + "step": 11690 + }, + { + "epoch": 188.71544715447155, + "grad_norm": 0.9954654574394226, + "learning_rate": 0.0001, + "loss": 56.4574, + "step": 11700 + }, + { + "epoch": 188.8780487804878, + "grad_norm": 0.851827085018158, + "learning_rate": 0.0001, + "loss": 55.996, + "step": 11710 + }, + { + "epoch": 189.03252032520325, + "grad_norm": 0.9404662847518921, + "learning_rate": 0.0001, + "loss": 53.8035, + "step": 11720 + }, + { + "epoch": 189.1951219512195, + "grad_norm": 0.8355186581611633, + "learning_rate": 0.0001, + "loss": 54.9002, + "step": 11730 + }, + { + "epoch": 189.35772357723576, + "grad_norm": 1.0156077146530151, + "learning_rate": 0.0001, + "loss": 55.8483, + "step": 11740 + }, + { + "epoch": 189.52032520325204, + "grad_norm": 0.7925308346748352, + "learning_rate": 0.0001, + "loss": 54.1772, + "step": 11750 + }, + { + "epoch": 189.6829268292683, + "grad_norm": 0.8137376308441162, + "learning_rate": 0.0001, + "loss": 53.6278, + "step": 11760 + }, + { + "epoch": 189.84552845528455, + "grad_norm": 0.7871745824813843, + "learning_rate": 0.0001, + "loss": 57.7457, + "step": 11770 + }, + { + "epoch": 190.0, + "grad_norm": 0.7344806790351868, + "learning_rate": 0.0001, + "loss": 55.6082, + "step": 11780 + }, + { + "epoch": 190.16260162601625, + "grad_norm": 0.9094588756561279, + "learning_rate": 0.0001, + "loss": 57.2329, + "step": 11790 + }, + { + "epoch": 190.3252032520325, + "grad_norm": 1.2261676788330078, + "learning_rate": 0.0001, + "loss": 56.2071, + "step": 11800 + }, + { + "epoch": 190.4878048780488, + "grad_norm": 0.9519370794296265, + "learning_rate": 0.0001, + "loss": 56.2488, + "step": 11810 + }, + { + "epoch": 190.65040650406505, + "grad_norm": 0.8949872851371765, + "learning_rate": 0.0001, + "loss": 57.9283, + "step": 11820 + }, + { + "epoch": 190.8130081300813, + "grad_norm": 0.8764474391937256, + "learning_rate": 0.0001, + "loss": 55.2962, + "step": 11830 + }, + { + "epoch": 190.97560975609755, + "grad_norm": 0.9472342729568481, + "learning_rate": 0.0001, + "loss": 56.2262, + "step": 11840 + }, + { + "epoch": 191.130081300813, + "grad_norm": 0.7477512359619141, + "learning_rate": 0.0001, + "loss": 51.2613, + "step": 11850 + }, + { + "epoch": 191.29268292682926, + "grad_norm": 1.0346648693084717, + "learning_rate": 0.0001, + "loss": 56.0406, + "step": 11860 + }, + { + "epoch": 191.45528455284554, + "grad_norm": 1.0473452806472778, + "learning_rate": 0.0001, + "loss": 56.3583, + "step": 11870 + }, + { + "epoch": 191.6178861788618, + "grad_norm": 1.0167906284332275, + "learning_rate": 0.0001, + "loss": 56.3803, + "step": 11880 + }, + { + "epoch": 191.78048780487805, + "grad_norm": 0.8768972754478455, + "learning_rate": 0.0001, + "loss": 57.2783, + "step": 11890 + }, + { + "epoch": 191.9430894308943, + "grad_norm": 0.9370661973953247, + "learning_rate": 0.0001, + "loss": 55.9976, + "step": 11900 + }, + { + "epoch": 192.09756097560975, + "grad_norm": 0.7853850722312927, + "learning_rate": 0.0001, + "loss": 51.9341, + "step": 11910 + }, + { + "epoch": 192.260162601626, + "grad_norm": 1.122916579246521, + "learning_rate": 0.0001, + "loss": 55.832, + "step": 11920 + }, + { + "epoch": 192.4227642276423, + "grad_norm": 0.982596755027771, + "learning_rate": 0.0001, + "loss": 54.2484, + "step": 11930 + }, + { + "epoch": 192.58536585365854, + "grad_norm": 1.0893633365631104, + "learning_rate": 0.0001, + "loss": 54.4806, + "step": 11940 + }, + { + "epoch": 192.7479674796748, + "grad_norm": 0.8936525583267212, + "learning_rate": 0.0001, + "loss": 55.1085, + "step": 11950 + }, + { + "epoch": 192.91056910569105, + "grad_norm": 0.9169464111328125, + "learning_rate": 0.0001, + "loss": 56.8134, + "step": 11960 + }, + { + "epoch": 193.0650406504065, + "grad_norm": 1.2202202081680298, + "learning_rate": 0.0001, + "loss": 53.3595, + "step": 11970 + }, + { + "epoch": 193.22764227642276, + "grad_norm": 0.7786313891410828, + "learning_rate": 0.0001, + "loss": 57.6186, + "step": 11980 + }, + { + "epoch": 193.390243902439, + "grad_norm": 0.9757220149040222, + "learning_rate": 0.0001, + "loss": 54.5559, + "step": 11990 + }, + { + "epoch": 193.5528455284553, + "grad_norm": 0.8599177002906799, + "learning_rate": 0.0001, + "loss": 54.9894, + "step": 12000 + }, + { + "epoch": 193.71544715447155, + "grad_norm": 1.614792823791504, + "learning_rate": 0.0001, + "loss": 56.6223, + "step": 12010 + }, + { + "epoch": 193.8780487804878, + "grad_norm": 0.9011794924736023, + "learning_rate": 0.0001, + "loss": 54.9678, + "step": 12020 + }, + { + "epoch": 194.03252032520325, + "grad_norm": 1.1813637018203735, + "learning_rate": 0.0001, + "loss": 51.7197, + "step": 12030 + }, + { + "epoch": 194.1951219512195, + "grad_norm": 1.420372724533081, + "learning_rate": 0.0001, + "loss": 57.0948, + "step": 12040 + }, + { + "epoch": 194.35772357723576, + "grad_norm": 0.8253830671310425, + "learning_rate": 0.0001, + "loss": 54.8281, + "step": 12050 + }, + { + "epoch": 194.52032520325204, + "grad_norm": 1.055162787437439, + "learning_rate": 0.0001, + "loss": 56.3061, + "step": 12060 + }, + { + "epoch": 194.6829268292683, + "grad_norm": 1.0420011281967163, + "learning_rate": 0.0001, + "loss": 54.7829, + "step": 12070 + }, + { + "epoch": 194.84552845528455, + "grad_norm": 0.7156410813331604, + "learning_rate": 0.0001, + "loss": 55.1366, + "step": 12080 + }, + { + "epoch": 195.0, + "grad_norm": 1.2018579244613647, + "learning_rate": 0.0001, + "loss": 55.2568, + "step": 12090 + }, + { + "epoch": 195.16260162601625, + "grad_norm": 0.9234516620635986, + "learning_rate": 0.0001, + "loss": 55.6524, + "step": 12100 + }, + { + "epoch": 195.3252032520325, + "grad_norm": 0.7889829874038696, + "learning_rate": 0.0001, + "loss": 54.6211, + "step": 12110 + }, + { + "epoch": 195.4878048780488, + "grad_norm": 0.6909085512161255, + "learning_rate": 0.0001, + "loss": 55.161, + "step": 12120 + }, + { + "epoch": 195.65040650406505, + "grad_norm": 1.0910391807556152, + "learning_rate": 0.0001, + "loss": 54.289, + "step": 12130 + }, + { + "epoch": 195.8130081300813, + "grad_norm": 0.8114144206047058, + "learning_rate": 0.0001, + "loss": 57.1045, + "step": 12140 + }, + { + "epoch": 195.97560975609755, + "grad_norm": 0.8750683069229126, + "learning_rate": 0.0001, + "loss": 56.6573, + "step": 12150 + }, + { + "epoch": 196.130081300813, + "grad_norm": 0.9273776412010193, + "learning_rate": 0.0001, + "loss": 52.4588, + "step": 12160 + }, + { + "epoch": 196.29268292682926, + "grad_norm": 0.8890953063964844, + "learning_rate": 0.0001, + "loss": 56.2728, + "step": 12170 + }, + { + "epoch": 196.45528455284554, + "grad_norm": 0.7885172367095947, + "learning_rate": 0.0001, + "loss": 56.9814, + "step": 12180 + }, + { + "epoch": 196.6178861788618, + "grad_norm": 1.0934451818466187, + "learning_rate": 0.0001, + "loss": 54.8799, + "step": 12190 + }, + { + "epoch": 196.78048780487805, + "grad_norm": 0.7452919483184814, + "learning_rate": 0.0001, + "loss": 54.8566, + "step": 12200 + }, + { + "epoch": 196.9430894308943, + "grad_norm": 1.2472970485687256, + "learning_rate": 0.0001, + "loss": 56.9789, + "step": 12210 + }, + { + "epoch": 197.09756097560975, + "grad_norm": 1.0619820356369019, + "learning_rate": 0.0001, + "loss": 52.0264, + "step": 12220 + }, + { + "epoch": 197.260162601626, + "grad_norm": 0.9423636198043823, + "learning_rate": 0.0001, + "loss": 54.6033, + "step": 12230 + }, + { + "epoch": 197.4227642276423, + "grad_norm": 0.6707811951637268, + "learning_rate": 0.0001, + "loss": 54.5683, + "step": 12240 + }, + { + "epoch": 197.58536585365854, + "grad_norm": 1.0682896375656128, + "learning_rate": 0.0001, + "loss": 53.6539, + "step": 12250 + }, + { + "epoch": 197.7479674796748, + "grad_norm": 0.9057245254516602, + "learning_rate": 0.0001, + "loss": 57.1636, + "step": 12260 + }, + { + "epoch": 197.91056910569105, + "grad_norm": 1.0249336957931519, + "learning_rate": 0.0001, + "loss": 54.9638, + "step": 12270 + }, + { + "epoch": 198.0650406504065, + "grad_norm": 0.8086504936218262, + "learning_rate": 0.0001, + "loss": 53.5776, + "step": 12280 + }, + { + "epoch": 198.22764227642276, + "grad_norm": 0.8970116376876831, + "learning_rate": 0.0001, + "loss": 55.9899, + "step": 12290 + }, + { + "epoch": 198.390243902439, + "grad_norm": 0.9402612447738647, + "learning_rate": 0.0001, + "loss": 56.7494, + "step": 12300 + }, + { + "epoch": 198.5528455284553, + "grad_norm": 0.865734338760376, + "learning_rate": 0.0001, + "loss": 56.3205, + "step": 12310 + }, + { + "epoch": 198.71544715447155, + "grad_norm": 0.9791702628135681, + "learning_rate": 0.0001, + "loss": 55.1574, + "step": 12320 + }, + { + "epoch": 198.8780487804878, + "grad_norm": 1.0380010604858398, + "learning_rate": 0.0001, + "loss": 56.5315, + "step": 12330 + }, + { + "epoch": 199.03252032520325, + "grad_norm": 1.14964759349823, + "learning_rate": 0.0001, + "loss": 52.6714, + "step": 12340 + }, + { + "epoch": 199.1951219512195, + "grad_norm": 0.9895046949386597, + "learning_rate": 0.0001, + "loss": 55.0279, + "step": 12350 + }, + { + "epoch": 199.35772357723576, + "grad_norm": 0.8557648062705994, + "learning_rate": 0.0001, + "loss": 54.9455, + "step": 12360 + }, + { + "epoch": 199.52032520325204, + "grad_norm": 0.9442970752716064, + "learning_rate": 0.0001, + "loss": 56.2179, + "step": 12370 + }, + { + "epoch": 199.6829268292683, + "grad_norm": 1.2164298295974731, + "learning_rate": 0.0001, + "loss": 54.7891, + "step": 12380 + }, + { + "epoch": 199.84552845528455, + "grad_norm": 1.1152207851409912, + "learning_rate": 0.0001, + "loss": 53.7524, + "step": 12390 + }, + { + "epoch": 200.0, + "grad_norm": 0.9624357223510742, + "learning_rate": 0.0001, + "loss": 52.4309, + "step": 12400 + }, + { + "epoch": 200.16260162601625, + "grad_norm": 1.1027910709381104, + "learning_rate": 0.0001, + "loss": 55.8449, + "step": 12410 + }, + { + "epoch": 200.3252032520325, + "grad_norm": 0.9644309878349304, + "learning_rate": 0.0001, + "loss": 54.6539, + "step": 12420 + }, + { + "epoch": 200.4878048780488, + "grad_norm": 0.8299253582954407, + "learning_rate": 0.0001, + "loss": 53.8338, + "step": 12430 + }, + { + "epoch": 200.65040650406505, + "grad_norm": 1.0003494024276733, + "learning_rate": 0.0001, + "loss": 56.5585, + "step": 12440 + }, + { + "epoch": 200.8130081300813, + "grad_norm": 0.9802242517471313, + "learning_rate": 0.0001, + "loss": 56.4943, + "step": 12450 + }, + { + "epoch": 200.97560975609755, + "grad_norm": 0.8739794492721558, + "learning_rate": 0.0001, + "loss": 55.1041, + "step": 12460 + }, + { + "epoch": 201.130081300813, + "grad_norm": 1.8339481353759766, + "learning_rate": 0.0001, + "loss": 52.89, + "step": 12470 + }, + { + "epoch": 201.29268292682926, + "grad_norm": 1.3706036806106567, + "learning_rate": 0.0001, + "loss": 55.5897, + "step": 12480 + }, + { + "epoch": 201.45528455284554, + "grad_norm": 1.2508513927459717, + "learning_rate": 0.0001, + "loss": 56.3578, + "step": 12490 + }, + { + "epoch": 201.6178861788618, + "grad_norm": 0.7036287188529968, + "learning_rate": 0.0001, + "loss": 54.2583, + "step": 12500 + }, + { + "epoch": 201.78048780487805, + "grad_norm": 0.9160637259483337, + "learning_rate": 0.0001, + "loss": 56.3668, + "step": 12510 + }, + { + "epoch": 201.9430894308943, + "grad_norm": 0.8451681137084961, + "learning_rate": 0.0001, + "loss": 55.0297, + "step": 12520 + }, + { + "epoch": 202.09756097560975, + "grad_norm": 1.103285789489746, + "learning_rate": 0.0001, + "loss": 52.6796, + "step": 12530 + }, + { + "epoch": 202.260162601626, + "grad_norm": 0.9927880167961121, + "learning_rate": 0.0001, + "loss": 55.9175, + "step": 12540 + }, + { + "epoch": 202.4227642276423, + "grad_norm": 0.6602478623390198, + "learning_rate": 0.0001, + "loss": 56.0773, + "step": 12550 + }, + { + "epoch": 202.58536585365854, + "grad_norm": 0.8949015140533447, + "learning_rate": 0.0001, + "loss": 55.9995, + "step": 12560 + }, + { + "epoch": 202.7479674796748, + "grad_norm": 0.7111349701881409, + "learning_rate": 0.0001, + "loss": 55.4216, + "step": 12570 + }, + { + "epoch": 202.91056910569105, + "grad_norm": 0.8465375900268555, + "learning_rate": 0.0001, + "loss": 57.2378, + "step": 12580 + }, + { + "epoch": 203.0650406504065, + "grad_norm": 0.7122918367385864, + "learning_rate": 0.0001, + "loss": 54.5047, + "step": 12590 + }, + { + "epoch": 203.22764227642276, + "grad_norm": 0.9927072525024414, + "learning_rate": 0.0001, + "loss": 53.9752, + "step": 12600 + }, + { + "epoch": 203.390243902439, + "grad_norm": 0.7084205746650696, + "learning_rate": 0.0001, + "loss": 53.7208, + "step": 12610 + }, + { + "epoch": 203.5528455284553, + "grad_norm": 0.7796747088432312, + "learning_rate": 0.0001, + "loss": 54.5118, + "step": 12620 + }, + { + "epoch": 203.71544715447155, + "grad_norm": 0.7925731539726257, + "learning_rate": 0.0001, + "loss": 55.8361, + "step": 12630 + }, + { + "epoch": 203.8780487804878, + "grad_norm": 0.8884345889091492, + "learning_rate": 0.0001, + "loss": 52.6655, + "step": 12640 + }, + { + "epoch": 204.03252032520325, + "grad_norm": 0.7525721192359924, + "learning_rate": 0.0001, + "loss": 52.0389, + "step": 12650 + }, + { + "epoch": 204.1951219512195, + "grad_norm": 1.071750521659851, + "learning_rate": 0.0001, + "loss": 55.6427, + "step": 12660 + }, + { + "epoch": 204.35772357723576, + "grad_norm": 0.7978437542915344, + "learning_rate": 0.0001, + "loss": 55.1445, + "step": 12670 + }, + { + "epoch": 204.52032520325204, + "grad_norm": 0.6743281483650208, + "learning_rate": 0.0001, + "loss": 56.5749, + "step": 12680 + }, + { + "epoch": 204.6829268292683, + "grad_norm": 0.9658812284469604, + "learning_rate": 0.0001, + "loss": 55.2514, + "step": 12690 + }, + { + "epoch": 204.84552845528455, + "grad_norm": 0.8930355310440063, + "learning_rate": 0.0001, + "loss": 56.5771, + "step": 12700 + }, + { + "epoch": 205.0, + "grad_norm": 2.1135165691375732, + "learning_rate": 0.0001, + "loss": 53.9625, + "step": 12710 + }, + { + "epoch": 205.16260162601625, + "grad_norm": 1.0068354606628418, + "learning_rate": 0.0001, + "loss": 55.2095, + "step": 12720 + }, + { + "epoch": 205.3252032520325, + "grad_norm": 0.8000341653823853, + "learning_rate": 0.0001, + "loss": 56.155, + "step": 12730 + }, + { + "epoch": 205.4878048780488, + "grad_norm": 1.201411485671997, + "learning_rate": 0.0001, + "loss": 56.8318, + "step": 12740 + }, + { + "epoch": 205.65040650406505, + "grad_norm": 1.005187749862671, + "learning_rate": 0.0001, + "loss": 55.8241, + "step": 12750 + }, + { + "epoch": 205.8130081300813, + "grad_norm": 1.0376118421554565, + "learning_rate": 0.0001, + "loss": 55.3489, + "step": 12760 + }, + { + "epoch": 205.97560975609755, + "grad_norm": 0.8955312967300415, + "learning_rate": 0.0001, + "loss": 55.583, + "step": 12770 + }, + { + "epoch": 206.130081300813, + "grad_norm": 0.9010198712348938, + "learning_rate": 0.0001, + "loss": 52.4093, + "step": 12780 + }, + { + "epoch": 206.29268292682926, + "grad_norm": 0.758160412311554, + "learning_rate": 0.0001, + "loss": 56.1256, + "step": 12790 + }, + { + "epoch": 206.45528455284554, + "grad_norm": 0.7211036086082458, + "learning_rate": 0.0001, + "loss": 55.0504, + "step": 12800 + }, + { + "epoch": 206.6178861788618, + "grad_norm": 1.065086841583252, + "learning_rate": 0.0001, + "loss": 55.7009, + "step": 12810 + }, + { + "epoch": 206.78048780487805, + "grad_norm": 0.6722949743270874, + "learning_rate": 0.0001, + "loss": 54.0928, + "step": 12820 + }, + { + "epoch": 206.9430894308943, + "grad_norm": 0.9325671792030334, + "learning_rate": 0.0001, + "loss": 55.6211, + "step": 12830 + }, + { + "epoch": 207.09756097560975, + "grad_norm": 0.9299114942550659, + "learning_rate": 0.0001, + "loss": 51.8944, + "step": 12840 + }, + { + "epoch": 207.260162601626, + "grad_norm": 0.8544942140579224, + "learning_rate": 0.0001, + "loss": 55.1983, + "step": 12850 + }, + { + "epoch": 207.4227642276423, + "grad_norm": 1.0352020263671875, + "learning_rate": 0.0001, + "loss": 53.4288, + "step": 12860 + }, + { + "epoch": 207.58536585365854, + "grad_norm": 1.1637558937072754, + "learning_rate": 0.0001, + "loss": 54.5078, + "step": 12870 + }, + { + "epoch": 207.7479674796748, + "grad_norm": 1.0634640455245972, + "learning_rate": 0.0001, + "loss": 53.8032, + "step": 12880 + }, + { + "epoch": 207.91056910569105, + "grad_norm": 1.0478582382202148, + "learning_rate": 0.0001, + "loss": 54.774, + "step": 12890 + }, + { + "epoch": 208.0650406504065, + "grad_norm": 0.772788405418396, + "learning_rate": 0.0001, + "loss": 52.4807, + "step": 12900 + }, + { + "epoch": 208.22764227642276, + "grad_norm": 1.0519691705703735, + "learning_rate": 0.0001, + "loss": 54.8709, + "step": 12910 + }, + { + "epoch": 208.390243902439, + "grad_norm": 0.9291022419929504, + "learning_rate": 0.0001, + "loss": 55.8279, + "step": 12920 + }, + { + "epoch": 208.5528455284553, + "grad_norm": 0.9870265126228333, + "learning_rate": 0.0001, + "loss": 56.3803, + "step": 12930 + }, + { + "epoch": 208.71544715447155, + "grad_norm": 0.787817656993866, + "learning_rate": 0.0001, + "loss": 55.1475, + "step": 12940 + }, + { + "epoch": 208.8780487804878, + "grad_norm": 1.1020783185958862, + "learning_rate": 0.0001, + "loss": 56.1078, + "step": 12950 + }, + { + "epoch": 209.03252032520325, + "grad_norm": 0.8097794055938721, + "learning_rate": 0.0001, + "loss": 53.1355, + "step": 12960 + }, + { + "epoch": 209.1951219512195, + "grad_norm": 1.1928120851516724, + "learning_rate": 0.0001, + "loss": 53.1578, + "step": 12970 + }, + { + "epoch": 209.35772357723576, + "grad_norm": 1.131969928741455, + "learning_rate": 0.0001, + "loss": 56.2361, + "step": 12980 + }, + { + "epoch": 209.52032520325204, + "grad_norm": 0.7544846534729004, + "learning_rate": 0.0001, + "loss": 55.9766, + "step": 12990 + }, + { + "epoch": 209.6829268292683, + "grad_norm": 1.2758643627166748, + "learning_rate": 0.0001, + "loss": 53.5174, + "step": 13000 + }, + { + "epoch": 209.84552845528455, + "grad_norm": 0.8507160544395447, + "learning_rate": 0.0001, + "loss": 53.9673, + "step": 13010 + }, + { + "epoch": 210.0, + "grad_norm": 1.2480005025863647, + "learning_rate": 0.0001, + "loss": 51.9134, + "step": 13020 + }, + { + "epoch": 210.16260162601625, + "grad_norm": 1.0838913917541504, + "learning_rate": 0.0001, + "loss": 52.4845, + "step": 13030 + }, + { + "epoch": 210.3252032520325, + "grad_norm": 0.8026198744773865, + "learning_rate": 0.0001, + "loss": 54.2063, + "step": 13040 + }, + { + "epoch": 210.4878048780488, + "grad_norm": 0.7196829319000244, + "learning_rate": 0.0001, + "loss": 54.91, + "step": 13050 + }, + { + "epoch": 210.65040650406505, + "grad_norm": 0.8283985257148743, + "learning_rate": 0.0001, + "loss": 56.3195, + "step": 13060 + }, + { + "epoch": 210.8130081300813, + "grad_norm": 0.9254607558250427, + "learning_rate": 0.0001, + "loss": 55.7841, + "step": 13070 + }, + { + "epoch": 210.97560975609755, + "grad_norm": 1.0719525814056396, + "learning_rate": 0.0001, + "loss": 55.4097, + "step": 13080 + }, + { + "epoch": 211.130081300813, + "grad_norm": 0.7990654110908508, + "learning_rate": 0.0001, + "loss": 50.4808, + "step": 13090 + }, + { + "epoch": 211.29268292682926, + "grad_norm": 0.7976981401443481, + "learning_rate": 0.0001, + "loss": 53.8439, + "step": 13100 + }, + { + "epoch": 211.45528455284554, + "grad_norm": 0.8668096661567688, + "learning_rate": 0.0001, + "loss": 55.9312, + "step": 13110 + }, + { + "epoch": 211.6178861788618, + "grad_norm": 0.7176859974861145, + "learning_rate": 0.0001, + "loss": 53.5898, + "step": 13120 + }, + { + "epoch": 211.78048780487805, + "grad_norm": 0.6893277168273926, + "learning_rate": 0.0001, + "loss": 54.099, + "step": 13130 + }, + { + "epoch": 211.9430894308943, + "grad_norm": 0.7272772192955017, + "learning_rate": 0.0001, + "loss": 55.8779, + "step": 13140 + }, + { + "epoch": 212.09756097560975, + "grad_norm": 0.9501060843467712, + "learning_rate": 0.0001, + "loss": 52.049, + "step": 13150 + }, + { + "epoch": 212.260162601626, + "grad_norm": 0.8938133716583252, + "learning_rate": 0.0001, + "loss": 54.9783, + "step": 13160 + }, + { + "epoch": 212.4227642276423, + "grad_norm": 1.3081871271133423, + "learning_rate": 0.0001, + "loss": 54.1436, + "step": 13170 + }, + { + "epoch": 212.58536585365854, + "grad_norm": 1.182673454284668, + "learning_rate": 0.0001, + "loss": 55.5235, + "step": 13180 + }, + { + "epoch": 212.7479674796748, + "grad_norm": 0.7060847878456116, + "learning_rate": 0.0001, + "loss": 54.8775, + "step": 13190 + }, + { + "epoch": 212.91056910569105, + "grad_norm": 0.6047799587249756, + "learning_rate": 0.0001, + "loss": 56.6604, + "step": 13200 + }, + { + "epoch": 213.0650406504065, + "grad_norm": 0.8285370469093323, + "learning_rate": 0.0001, + "loss": 52.9922, + "step": 13210 + }, + { + "epoch": 213.22764227642276, + "grad_norm": 0.8437235951423645, + "learning_rate": 0.0001, + "loss": 54.9914, + "step": 13220 + }, + { + "epoch": 213.390243902439, + "grad_norm": 0.7426273822784424, + "learning_rate": 0.0001, + "loss": 54.7659, + "step": 13230 + }, + { + "epoch": 213.5528455284553, + "grad_norm": 0.8756979703903198, + "learning_rate": 0.0001, + "loss": 53.2097, + "step": 13240 + }, + { + "epoch": 213.71544715447155, + "grad_norm": 0.9625734090805054, + "learning_rate": 0.0001, + "loss": 55.2452, + "step": 13250 + }, + { + "epoch": 213.8780487804878, + "grad_norm": 1.5846962928771973, + "learning_rate": 0.0001, + "loss": 54.1381, + "step": 13260 + }, + { + "epoch": 214.03252032520325, + "grad_norm": 0.6845040321350098, + "learning_rate": 0.0001, + "loss": 50.8792, + "step": 13270 + }, + { + "epoch": 214.1951219512195, + "grad_norm": 1.0069342851638794, + "learning_rate": 0.0001, + "loss": 56.4215, + "step": 13280 + }, + { + "epoch": 214.35772357723576, + "grad_norm": 0.8575270771980286, + "learning_rate": 0.0001, + "loss": 54.7543, + "step": 13290 + }, + { + "epoch": 214.52032520325204, + "grad_norm": 1.1226451396942139, + "learning_rate": 0.0001, + "loss": 53.5045, + "step": 13300 + }, + { + "epoch": 214.6829268292683, + "grad_norm": 0.8359237313270569, + "learning_rate": 0.0001, + "loss": 54.7099, + "step": 13310 + }, + { + "epoch": 214.84552845528455, + "grad_norm": 0.823088526725769, + "learning_rate": 0.0001, + "loss": 55.2504, + "step": 13320 + }, + { + "epoch": 215.0, + "grad_norm": 0.9530013799667358, + "learning_rate": 0.0001, + "loss": 51.2342, + "step": 13330 + }, + { + "epoch": 215.16260162601625, + "grad_norm": 1.024066686630249, + "learning_rate": 0.0001, + "loss": 54.8409, + "step": 13340 + }, + { + "epoch": 215.3252032520325, + "grad_norm": 0.7555123567581177, + "learning_rate": 0.0001, + "loss": 54.6994, + "step": 13350 + }, + { + "epoch": 215.4878048780488, + "grad_norm": 0.8845812678337097, + "learning_rate": 0.0001, + "loss": 55.1865, + "step": 13360 + }, + { + "epoch": 215.65040650406505, + "grad_norm": 0.8159278035163879, + "learning_rate": 0.0001, + "loss": 54.4797, + "step": 13370 + }, + { + "epoch": 215.8130081300813, + "grad_norm": 0.8099329471588135, + "learning_rate": 0.0001, + "loss": 53.4086, + "step": 13380 + }, + { + "epoch": 215.97560975609755, + "grad_norm": 0.7204589247703552, + "learning_rate": 0.0001, + "loss": 54.1444, + "step": 13390 + }, + { + "epoch": 216.130081300813, + "grad_norm": 1.0179848670959473, + "learning_rate": 0.0001, + "loss": 52.6701, + "step": 13400 + }, + { + "epoch": 216.29268292682926, + "grad_norm": 0.7029995322227478, + "learning_rate": 0.0001, + "loss": 55.2125, + "step": 13410 + }, + { + "epoch": 216.45528455284554, + "grad_norm": 0.7679967880249023, + "learning_rate": 0.0001, + "loss": 55.5851, + "step": 13420 + }, + { + "epoch": 216.6178861788618, + "grad_norm": 0.925176739692688, + "learning_rate": 0.0001, + "loss": 54.6698, + "step": 13430 + }, + { + "epoch": 216.78048780487805, + "grad_norm": 1.0775889158248901, + "learning_rate": 0.0001, + "loss": 53.8535, + "step": 13440 + }, + { + "epoch": 216.9430894308943, + "grad_norm": 1.9590994119644165, + "learning_rate": 0.0001, + "loss": 53.3539, + "step": 13450 + }, + { + "epoch": 217.09756097560975, + "grad_norm": 1.1416126489639282, + "learning_rate": 0.0001, + "loss": 53.07, + "step": 13460 + }, + { + "epoch": 217.260162601626, + "grad_norm": 0.6063092947006226, + "learning_rate": 0.0001, + "loss": 55.2222, + "step": 13470 + }, + { + "epoch": 217.4227642276423, + "grad_norm": 1.082871913909912, + "learning_rate": 0.0001, + "loss": 54.9424, + "step": 13480 + }, + { + "epoch": 217.58536585365854, + "grad_norm": 1.0098278522491455, + "learning_rate": 0.0001, + "loss": 55.5704, + "step": 13490 + }, + { + "epoch": 217.7479674796748, + "grad_norm": 0.7821612358093262, + "learning_rate": 0.0001, + "loss": 55.6112, + "step": 13500 + }, + { + "epoch": 217.91056910569105, + "grad_norm": 0.7899888753890991, + "learning_rate": 0.0001, + "loss": 54.2449, + "step": 13510 + }, + { + "epoch": 218.0650406504065, + "grad_norm": 0.8772909045219421, + "learning_rate": 0.0001, + "loss": 51.3149, + "step": 13520 + }, + { + "epoch": 218.22764227642276, + "grad_norm": 0.835057258605957, + "learning_rate": 0.0001, + "loss": 54.9729, + "step": 13530 + }, + { + "epoch": 218.390243902439, + "grad_norm": 0.9645805954933167, + "learning_rate": 0.0001, + "loss": 53.1176, + "step": 13540 + }, + { + "epoch": 218.5528455284553, + "grad_norm": 1.1908318996429443, + "learning_rate": 0.0001, + "loss": 54.3461, + "step": 13550 + }, + { + "epoch": 218.71544715447155, + "grad_norm": 0.8490066528320312, + "learning_rate": 0.0001, + "loss": 56.1804, + "step": 13560 + }, + { + "epoch": 218.8780487804878, + "grad_norm": 0.855278491973877, + "learning_rate": 0.0001, + "loss": 54.6583, + "step": 13570 + }, + { + "epoch": 219.03252032520325, + "grad_norm": 0.8656384944915771, + "learning_rate": 0.0001, + "loss": 51.4255, + "step": 13580 + }, + { + "epoch": 219.1951219512195, + "grad_norm": 0.8291656970977783, + "learning_rate": 0.0001, + "loss": 55.1752, + "step": 13590 + }, + { + "epoch": 219.35772357723576, + "grad_norm": 0.9894065260887146, + "learning_rate": 0.0001, + "loss": 52.9551, + "step": 13600 + }, + { + "epoch": 219.52032520325204, + "grad_norm": 0.7791743278503418, + "learning_rate": 0.0001, + "loss": 53.2005, + "step": 13610 + }, + { + "epoch": 219.6829268292683, + "grad_norm": 0.8251843452453613, + "learning_rate": 0.0001, + "loss": 54.3272, + "step": 13620 + }, + { + "epoch": 219.84552845528455, + "grad_norm": 0.6909898519515991, + "learning_rate": 0.0001, + "loss": 53.4259, + "step": 13630 + }, + { + "epoch": 220.0, + "grad_norm": 0.7680906057357788, + "learning_rate": 0.0001, + "loss": 53.6018, + "step": 13640 + }, + { + "epoch": 220.16260162601625, + "grad_norm": 1.1677294969558716, + "learning_rate": 0.0001, + "loss": 53.7904, + "step": 13650 + }, + { + "epoch": 220.3252032520325, + "grad_norm": 0.8123244047164917, + "learning_rate": 0.0001, + "loss": 55.255, + "step": 13660 + }, + { + "epoch": 220.4878048780488, + "grad_norm": 0.8476759791374207, + "learning_rate": 0.0001, + "loss": 56.5699, + "step": 13670 + }, + { + "epoch": 220.65040650406505, + "grad_norm": 0.9571487903594971, + "learning_rate": 0.0001, + "loss": 54.228, + "step": 13680 + }, + { + "epoch": 220.8130081300813, + "grad_norm": 0.7796895503997803, + "learning_rate": 0.0001, + "loss": 54.8423, + "step": 13690 + }, + { + "epoch": 220.97560975609755, + "grad_norm": 0.9055293798446655, + "learning_rate": 0.0001, + "loss": 53.7139, + "step": 13700 + }, + { + "epoch": 221.130081300813, + "grad_norm": 0.7490646839141846, + "learning_rate": 0.0001, + "loss": 52.4622, + "step": 13710 + }, + { + "epoch": 221.29268292682926, + "grad_norm": 0.7626569271087646, + "learning_rate": 0.0001, + "loss": 55.8454, + "step": 13720 + }, + { + "epoch": 221.45528455284554, + "grad_norm": 0.9692214131355286, + "learning_rate": 0.0001, + "loss": 55.0342, + "step": 13730 + }, + { + "epoch": 221.6178861788618, + "grad_norm": 0.8987938761711121, + "learning_rate": 0.0001, + "loss": 54.8382, + "step": 13740 + }, + { + "epoch": 221.78048780487805, + "grad_norm": 0.7676286101341248, + "learning_rate": 0.0001, + "loss": 55.5017, + "step": 13750 + }, + { + "epoch": 221.9430894308943, + "grad_norm": 0.8445277214050293, + "learning_rate": 0.0001, + "loss": 55.331, + "step": 13760 + }, + { + "epoch": 222.09756097560975, + "grad_norm": 0.7058215737342834, + "learning_rate": 0.0001, + "loss": 52.9581, + "step": 13770 + }, + { + "epoch": 222.260162601626, + "grad_norm": 0.7497864365577698, + "learning_rate": 0.0001, + "loss": 54.9104, + "step": 13780 + }, + { + "epoch": 222.4227642276423, + "grad_norm": 0.9665900468826294, + "learning_rate": 0.0001, + "loss": 55.9701, + "step": 13790 + }, + { + "epoch": 222.58536585365854, + "grad_norm": 0.9786218404769897, + "learning_rate": 0.0001, + "loss": 55.2512, + "step": 13800 + }, + { + "epoch": 222.7479674796748, + "grad_norm": 1.1519272327423096, + "learning_rate": 0.0001, + "loss": 56.0712, + "step": 13810 + }, + { + "epoch": 222.91056910569105, + "grad_norm": 0.8277210593223572, + "learning_rate": 0.0001, + "loss": 56.1046, + "step": 13820 + }, + { + "epoch": 223.0650406504065, + "grad_norm": 0.816068708896637, + "learning_rate": 0.0001, + "loss": 52.8655, + "step": 13830 + }, + { + "epoch": 223.22764227642276, + "grad_norm": 0.6874851584434509, + "learning_rate": 0.0001, + "loss": 54.6701, + "step": 13840 + }, + { + "epoch": 223.390243902439, + "grad_norm": 0.8549659848213196, + "learning_rate": 0.0001, + "loss": 55.4961, + "step": 13850 + }, + { + "epoch": 223.5528455284553, + "grad_norm": 0.9988876581192017, + "learning_rate": 0.0001, + "loss": 54.9155, + "step": 13860 + }, + { + "epoch": 223.71544715447155, + "grad_norm": 0.8321837782859802, + "learning_rate": 0.0001, + "loss": 54.0151, + "step": 13870 + }, + { + "epoch": 223.8780487804878, + "grad_norm": 0.730671763420105, + "learning_rate": 0.0001, + "loss": 53.601, + "step": 13880 + }, + { + "epoch": 224.03252032520325, + "grad_norm": 1.1248708963394165, + "learning_rate": 0.0001, + "loss": 51.4019, + "step": 13890 + }, + { + "epoch": 224.1951219512195, + "grad_norm": 1.034224033355713, + "learning_rate": 0.0001, + "loss": 54.3903, + "step": 13900 + }, + { + "epoch": 224.35772357723576, + "grad_norm": 1.0652328729629517, + "learning_rate": 0.0001, + "loss": 56.968, + "step": 13910 + }, + { + "epoch": 224.52032520325204, + "grad_norm": 0.8998368978500366, + "learning_rate": 0.0001, + "loss": 54.6571, + "step": 13920 + }, + { + "epoch": 224.6829268292683, + "grad_norm": 1.0750027894973755, + "learning_rate": 0.0001, + "loss": 53.7169, + "step": 13930 + }, + { + "epoch": 224.84552845528455, + "grad_norm": 1.1613131761550903, + "learning_rate": 0.0001, + "loss": 55.1695, + "step": 13940 + }, + { + "epoch": 225.0, + "grad_norm": 1.035953402519226, + "learning_rate": 0.0001, + "loss": 53.4469, + "step": 13950 + }, + { + "epoch": 225.16260162601625, + "grad_norm": 0.7781218886375427, + "learning_rate": 0.0001, + "loss": 55.1178, + "step": 13960 + }, + { + "epoch": 225.3252032520325, + "grad_norm": 0.9615271091461182, + "learning_rate": 0.0001, + "loss": 54.1518, + "step": 13970 + }, + { + "epoch": 225.4878048780488, + "grad_norm": 0.8908602595329285, + "learning_rate": 0.0001, + "loss": 56.4974, + "step": 13980 + }, + { + "epoch": 225.65040650406505, + "grad_norm": 1.3588823080062866, + "learning_rate": 0.0001, + "loss": 54.4211, + "step": 13990 + }, + { + "epoch": 225.8130081300813, + "grad_norm": 0.8439602255821228, + "learning_rate": 0.0001, + "loss": 53.8013, + "step": 14000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 323, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.7005260166209208e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c7f229ebe1034caaf3e92006c8c7528c21af93d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517586849d044483051c7a59cec7c6ce389951878a7357ef837edb1cbc9983c8 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-14000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/latest new file mode 100644 index 0000000000000000000000000000000000000000..76800d601529826c2ec8d7453b614b895cf74f2a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/latest @@ -0,0 +1 @@ +global_step16000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..296bda947cb39cf3e5396bf3c5fff1b3a74b5ecb --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d162c07125a2d1f37ccfe8310905f206f79572014e148e1102e3d5dec3930186 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..7aeb6ad0e0964795499e3eb4b3357e0d594a5198 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:767c80345fe58cf29f0c42c7fcf08829166bf50154cd857f5ca1d84f799f6f9b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..73ae5b31162d396e7aa7e7eaa83c8bae3408c708 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2885724ae36ef06c67ed8c60f381bd0c9105a30bb38c2ab10ea8483ffa64834 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..4a333271e4bd0c089fd1eab3a98ed92874b44c43 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b15b5100a9f42672ba7e8d6fd457ebdbf546ef5d70f656275e006cb4f9ba7f04 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..c98e0a34bcceacfedc0070d7b90b167e153969b6 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:459173f34df8407620ff047e5df449e69a836728c6254252b0dc79a27990f9c4 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..d37a59513bad683cb6f0dab8f2960bc9d751bd9b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe6c2eadcd322540739c84ce30a7c2338717c70990a382f3672756ffa8f72fe4 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..069ef0fe18daeb6933c284aa66f4fdb54e5d1cbf --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beab9758bb0a877c6c3653bf37194d8a10c027f945454c73d62fd332f91503b8 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..a6ae49c9b38a8f0d0fbc6902221af9cb560e3ab4 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9f7ae6b69350e122b364061a8d959edc122f563f0aa8ce0e26705b70b81e0a1 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..eefab360b9b4f47050497dc38c948f075048ef36 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb5bf0b211467d2f07e5bcd63f186d68692f406ed697a4f00511cf0674c23f28 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..40b93c36797b53b455f19ea4c1bc67022918d62f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a427e77702ae9121e994cdda9f4f553997fc61824162d506f30f54b748c208a5 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0c249b657b26e8b8da4db6eeb386bc66ec224d78 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/trainer_state.json @@ -0,0 +1,11234 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 258.0650406504065, + "eval_steps": 2000, + "global_step": 16000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.16260162601626016, + "grad_norm": 23.72986602783203, + "learning_rate": 0.0001, + "loss": 271.6598, + "step": 10 + }, + { + "epoch": 0.3252032520325203, + "grad_norm": 18.99613380432129, + "learning_rate": 0.0001, + "loss": 233.029, + "step": 20 + }, + { + "epoch": 0.4878048780487805, + "grad_norm": 3.9815304279327393, + "learning_rate": 0.0001, + "loss": 231.0776, + "step": 30 + }, + { + "epoch": 0.6504065040650406, + "grad_norm": 4.641753196716309, + "learning_rate": 0.0001, + "loss": 228.6894, + "step": 40 + }, + { + "epoch": 0.8130081300813008, + "grad_norm": 3.000852108001709, + "learning_rate": 0.0001, + "loss": 228.4694, + "step": 50 + }, + { + "epoch": 0.975609756097561, + "grad_norm": 12.213018417358398, + "learning_rate": 0.0001, + "loss": 226.4258, + "step": 60 + }, + { + "epoch": 1.1300813008130082, + "grad_norm": 2.535187005996704, + "learning_rate": 0.0001, + "loss": 209.9772, + "step": 70 + }, + { + "epoch": 1.2926829268292683, + "grad_norm": 2.0984582901000977, + "learning_rate": 0.0001, + "loss": 217.2421, + "step": 80 + }, + { + "epoch": 1.4552845528455285, + "grad_norm": 3.4802305698394775, + "learning_rate": 0.0001, + "loss": 210.9241, + "step": 90 + }, + { + "epoch": 1.6178861788617886, + "grad_norm": 3.813626527786255, + "learning_rate": 0.0001, + "loss": 205.7566, + "step": 100 + }, + { + "epoch": 1.7804878048780488, + "grad_norm": 2.6378512382507324, + "learning_rate": 0.0001, + "loss": 201.8789, + "step": 110 + }, + { + "epoch": 1.943089430894309, + "grad_norm": 2.7502875328063965, + "learning_rate": 0.0001, + "loss": 196.373, + "step": 120 + }, + { + "epoch": 2.097560975609756, + "grad_norm": 2.659212112426758, + "learning_rate": 0.0001, + "loss": 183.1456, + "step": 130 + }, + { + "epoch": 2.2601626016260163, + "grad_norm": 2.513044595718384, + "learning_rate": 0.0001, + "loss": 190.1661, + "step": 140 + }, + { + "epoch": 2.4227642276422765, + "grad_norm": 2.3192145824432373, + "learning_rate": 0.0001, + "loss": 184.9997, + "step": 150 + }, + { + "epoch": 2.5853658536585367, + "grad_norm": 3.056708574295044, + "learning_rate": 0.0001, + "loss": 182.0164, + "step": 160 + }, + { + "epoch": 2.747967479674797, + "grad_norm": 2.9687108993530273, + "learning_rate": 0.0001, + "loss": 179.6539, + "step": 170 + }, + { + "epoch": 2.910569105691057, + "grad_norm": 2.1498093605041504, + "learning_rate": 0.0001, + "loss": 176.7159, + "step": 180 + }, + { + "epoch": 3.065040650406504, + "grad_norm": 3.0103771686553955, + "learning_rate": 0.0001, + "loss": 166.1703, + "step": 190 + }, + { + "epoch": 3.227642276422764, + "grad_norm": 2.992462635040283, + "learning_rate": 0.0001, + "loss": 171.124, + "step": 200 + }, + { + "epoch": 3.3902439024390243, + "grad_norm": 4.8617119789123535, + "learning_rate": 0.0001, + "loss": 168.2646, + "step": 210 + }, + { + "epoch": 3.5528455284552845, + "grad_norm": 3.1047444343566895, + "learning_rate": 0.0001, + "loss": 163.0889, + "step": 220 + }, + { + "epoch": 3.7154471544715446, + "grad_norm": 6.473227500915527, + "learning_rate": 0.0001, + "loss": 151.3667, + "step": 230 + }, + { + "epoch": 3.8780487804878048, + "grad_norm": 6.16747522354126, + "learning_rate": 0.0001, + "loss": 137.5915, + "step": 240 + }, + { + "epoch": 4.032520325203252, + "grad_norm": 7.994515419006348, + "learning_rate": 0.0001, + "loss": 118.8875, + "step": 250 + }, + { + "epoch": 4.195121951219512, + "grad_norm": 5.882101058959961, + "learning_rate": 0.0001, + "loss": 119.4793, + "step": 260 + }, + { + "epoch": 4.357723577235772, + "grad_norm": 12.32968521118164, + "learning_rate": 0.0001, + "loss": 113.1209, + "step": 270 + }, + { + "epoch": 4.520325203252033, + "grad_norm": 5.697225570678711, + "learning_rate": 0.0001, + "loss": 107.51, + "step": 280 + }, + { + "epoch": 4.682926829268292, + "grad_norm": 4.654167175292969, + "learning_rate": 0.0001, + "loss": 107.0172, + "step": 290 + }, + { + "epoch": 4.845528455284553, + "grad_norm": 2.9595484733581543, + "learning_rate": 0.0001, + "loss": 100.9164, + "step": 300 + }, + { + "epoch": 5.0, + "grad_norm": 21.242591857910156, + "learning_rate": 0.0001, + "loss": 94.9884, + "step": 310 + }, + { + "epoch": 5.16260162601626, + "grad_norm": 5.760314464569092, + "learning_rate": 0.0001, + "loss": 97.5901, + "step": 320 + }, + { + "epoch": 5.32520325203252, + "grad_norm": 3.1373329162597656, + "learning_rate": 0.0001, + "loss": 94.8485, + "step": 330 + }, + { + "epoch": 5.487804878048781, + "grad_norm": 2.3777568340301514, + "learning_rate": 0.0001, + "loss": 96.9982, + "step": 340 + }, + { + "epoch": 5.650406504065041, + "grad_norm": 3.0607948303222656, + "learning_rate": 0.0001, + "loss": 96.1586, + "step": 350 + }, + { + "epoch": 5.8130081300813, + "grad_norm": 4.168269634246826, + "learning_rate": 0.0001, + "loss": 94.5743, + "step": 360 + }, + { + "epoch": 5.975609756097561, + "grad_norm": 2.4383699893951416, + "learning_rate": 0.0001, + "loss": 95.5197, + "step": 370 + }, + { + "epoch": 6.130081300813008, + "grad_norm": 2.302259683609009, + "learning_rate": 0.0001, + "loss": 85.6599, + "step": 380 + }, + { + "epoch": 6.2926829268292686, + "grad_norm": 2.552753210067749, + "learning_rate": 0.0001, + "loss": 91.2999, + "step": 390 + }, + { + "epoch": 6.455284552845528, + "grad_norm": 2.2092080116271973, + "learning_rate": 0.0001, + "loss": 90.7041, + "step": 400 + }, + { + "epoch": 6.617886178861789, + "grad_norm": 2.2935690879821777, + "learning_rate": 0.0001, + "loss": 89.9949, + "step": 410 + }, + { + "epoch": 6.780487804878049, + "grad_norm": 2.487510919570923, + "learning_rate": 0.0001, + "loss": 86.9056, + "step": 420 + }, + { + "epoch": 6.943089430894309, + "grad_norm": 2.8630545139312744, + "learning_rate": 0.0001, + "loss": 88.3347, + "step": 430 + }, + { + "epoch": 7.097560975609756, + "grad_norm": 5.87056303024292, + "learning_rate": 0.0001, + "loss": 84.9632, + "step": 440 + }, + { + "epoch": 7.260162601626016, + "grad_norm": 2.587661027908325, + "learning_rate": 0.0001, + "loss": 86.2024, + "step": 450 + }, + { + "epoch": 7.4227642276422765, + "grad_norm": 3.5671255588531494, + "learning_rate": 0.0001, + "loss": 86.218, + "step": 460 + }, + { + "epoch": 7.585365853658536, + "grad_norm": 2.5036680698394775, + "learning_rate": 0.0001, + "loss": 86.5914, + "step": 470 + }, + { + "epoch": 7.747967479674797, + "grad_norm": 3.2656471729278564, + "learning_rate": 0.0001, + "loss": 82.6081, + "step": 480 + }, + { + "epoch": 7.9105691056910565, + "grad_norm": 2.5053915977478027, + "learning_rate": 0.0001, + "loss": 84.6116, + "step": 490 + }, + { + "epoch": 8.065040650406504, + "grad_norm": 2.1669957637786865, + "learning_rate": 0.0001, + "loss": 80.4354, + "step": 500 + }, + { + "epoch": 8.227642276422765, + "grad_norm": 4.2003703117370605, + "learning_rate": 0.0001, + "loss": 85.2286, + "step": 510 + }, + { + "epoch": 8.390243902439025, + "grad_norm": 3.1909608840942383, + "learning_rate": 0.0001, + "loss": 85.0358, + "step": 520 + }, + { + "epoch": 8.552845528455284, + "grad_norm": 1.8420121669769287, + "learning_rate": 0.0001, + "loss": 85.4647, + "step": 530 + }, + { + "epoch": 8.715447154471544, + "grad_norm": 1.9822801351547241, + "learning_rate": 0.0001, + "loss": 84.7134, + "step": 540 + }, + { + "epoch": 8.878048780487806, + "grad_norm": 2.936758518218994, + "learning_rate": 0.0001, + "loss": 83.8068, + "step": 550 + }, + { + "epoch": 9.032520325203253, + "grad_norm": 1.5713881254196167, + "learning_rate": 0.0001, + "loss": 80.0695, + "step": 560 + }, + { + "epoch": 9.195121951219512, + "grad_norm": 2.0584616661071777, + "learning_rate": 0.0001, + "loss": 84.5158, + "step": 570 + }, + { + "epoch": 9.357723577235772, + "grad_norm": 2.101391553878784, + "learning_rate": 0.0001, + "loss": 82.4719, + "step": 580 + }, + { + "epoch": 9.520325203252032, + "grad_norm": 1.9956543445587158, + "learning_rate": 0.0001, + "loss": 81.6715, + "step": 590 + }, + { + "epoch": 9.682926829268293, + "grad_norm": 2.7622063159942627, + "learning_rate": 0.0001, + "loss": 85.4178, + "step": 600 + }, + { + "epoch": 9.845528455284553, + "grad_norm": 2.2084789276123047, + "learning_rate": 0.0001, + "loss": 84.5702, + "step": 610 + }, + { + "epoch": 10.0, + "grad_norm": 1.9635663032531738, + "learning_rate": 0.0001, + "loss": 77.3838, + "step": 620 + }, + { + "epoch": 10.16260162601626, + "grad_norm": 2.5895581245422363, + "learning_rate": 0.0001, + "loss": 81.723, + "step": 630 + }, + { + "epoch": 10.32520325203252, + "grad_norm": 2.743727445602417, + "learning_rate": 0.0001, + "loss": 83.1332, + "step": 640 + }, + { + "epoch": 10.487804878048781, + "grad_norm": 2.024585008621216, + "learning_rate": 0.0001, + "loss": 81.5968, + "step": 650 + }, + { + "epoch": 10.65040650406504, + "grad_norm": 1.8254153728485107, + "learning_rate": 0.0001, + "loss": 78.9589, + "step": 660 + }, + { + "epoch": 10.8130081300813, + "grad_norm": 3.2015740871429443, + "learning_rate": 0.0001, + "loss": 79.1829, + "step": 670 + }, + { + "epoch": 10.975609756097562, + "grad_norm": 1.788453459739685, + "learning_rate": 0.0001, + "loss": 81.527, + "step": 680 + }, + { + "epoch": 11.130081300813009, + "grad_norm": 1.5744423866271973, + "learning_rate": 0.0001, + "loss": 74.9869, + "step": 690 + }, + { + "epoch": 11.292682926829269, + "grad_norm": 2.006002187728882, + "learning_rate": 0.0001, + "loss": 80.2677, + "step": 700 + }, + { + "epoch": 11.455284552845528, + "grad_norm": 1.8847148418426514, + "learning_rate": 0.0001, + "loss": 78.8709, + "step": 710 + }, + { + "epoch": 11.617886178861788, + "grad_norm": 1.8384478092193604, + "learning_rate": 0.0001, + "loss": 79.008, + "step": 720 + }, + { + "epoch": 11.78048780487805, + "grad_norm": 1.9486687183380127, + "learning_rate": 0.0001, + "loss": 76.1525, + "step": 730 + }, + { + "epoch": 11.94308943089431, + "grad_norm": 1.8644076585769653, + "learning_rate": 0.0001, + "loss": 77.5802, + "step": 740 + }, + { + "epoch": 12.097560975609756, + "grad_norm": 1.6035499572753906, + "learning_rate": 0.0001, + "loss": 75.2422, + "step": 750 + }, + { + "epoch": 12.260162601626016, + "grad_norm": 1.6935622692108154, + "learning_rate": 0.0001, + "loss": 77.5838, + "step": 760 + }, + { + "epoch": 12.422764227642276, + "grad_norm": 1.9327987432479858, + "learning_rate": 0.0001, + "loss": 78.3237, + "step": 770 + }, + { + "epoch": 12.585365853658537, + "grad_norm": 1.7903852462768555, + "learning_rate": 0.0001, + "loss": 78.7459, + "step": 780 + }, + { + "epoch": 12.747967479674797, + "grad_norm": 1.4962575435638428, + "learning_rate": 0.0001, + "loss": 78.9193, + "step": 790 + }, + { + "epoch": 12.910569105691057, + "grad_norm": 42.10435104370117, + "learning_rate": 0.0001, + "loss": 80.1429, + "step": 800 + }, + { + "epoch": 13.065040650406504, + "grad_norm": 1.904065728187561, + "learning_rate": 0.0001, + "loss": 73.7798, + "step": 810 + }, + { + "epoch": 13.227642276422765, + "grad_norm": 1.8768481016159058, + "learning_rate": 0.0001, + "loss": 76.8692, + "step": 820 + }, + { + "epoch": 13.390243902439025, + "grad_norm": 16.140565872192383, + "learning_rate": 0.0001, + "loss": 78.122, + "step": 830 + }, + { + "epoch": 13.552845528455284, + "grad_norm": 2.160288095474243, + "learning_rate": 0.0001, + "loss": 76.2169, + "step": 840 + }, + { + "epoch": 13.715447154471544, + "grad_norm": 27.682231903076172, + "learning_rate": 0.0001, + "loss": 76.6266, + "step": 850 + }, + { + "epoch": 13.878048780487806, + "grad_norm": 1.779772162437439, + "learning_rate": 0.0001, + "loss": 80.0434, + "step": 860 + }, + { + "epoch": 14.032520325203253, + "grad_norm": 1.6516485214233398, + "learning_rate": 0.0001, + "loss": 73.3943, + "step": 870 + }, + { + "epoch": 14.195121951219512, + "grad_norm": 1.846498966217041, + "learning_rate": 0.0001, + "loss": 77.5157, + "step": 880 + }, + { + "epoch": 14.357723577235772, + "grad_norm": 1.942946434020996, + "learning_rate": 0.0001, + "loss": 77.6944, + "step": 890 + }, + { + "epoch": 14.520325203252032, + "grad_norm": 2.0929224491119385, + "learning_rate": 0.0001, + "loss": 76.9411, + "step": 900 + }, + { + "epoch": 14.682926829268293, + "grad_norm": 3.4498767852783203, + "learning_rate": 0.0001, + "loss": 74.9234, + "step": 910 + }, + { + "epoch": 14.845528455284553, + "grad_norm": 1.6880728006362915, + "learning_rate": 0.0001, + "loss": 75.3617, + "step": 920 + }, + { + "epoch": 15.0, + "grad_norm": 1.8403468132019043, + "learning_rate": 0.0001, + "loss": 73.2543, + "step": 930 + }, + { + "epoch": 15.16260162601626, + "grad_norm": 1.4368852376937866, + "learning_rate": 0.0001, + "loss": 76.6271, + "step": 940 + }, + { + "epoch": 15.32520325203252, + "grad_norm": 2.9467780590057373, + "learning_rate": 0.0001, + "loss": 75.8744, + "step": 950 + }, + { + "epoch": 15.487804878048781, + "grad_norm": 1.567134141921997, + "learning_rate": 0.0001, + "loss": 76.6629, + "step": 960 + }, + { + "epoch": 15.65040650406504, + "grad_norm": 1.4503871202468872, + "learning_rate": 0.0001, + "loss": 76.3525, + "step": 970 + }, + { + "epoch": 15.8130081300813, + "grad_norm": 1.8871147632598877, + "learning_rate": 0.0001, + "loss": 77.4684, + "step": 980 + }, + { + "epoch": 15.975609756097562, + "grad_norm": 1.6707026958465576, + "learning_rate": 0.0001, + "loss": 76.3063, + "step": 990 + }, + { + "epoch": 16.130081300813007, + "grad_norm": 3.6300716400146484, + "learning_rate": 0.0001, + "loss": 73.0586, + "step": 1000 + }, + { + "epoch": 16.29268292682927, + "grad_norm": 1.6669234037399292, + "learning_rate": 0.0001, + "loss": 75.8027, + "step": 1010 + }, + { + "epoch": 16.45528455284553, + "grad_norm": 1.6920552253723145, + "learning_rate": 0.0001, + "loss": 74.7252, + "step": 1020 + }, + { + "epoch": 16.617886178861788, + "grad_norm": 1.8184431791305542, + "learning_rate": 0.0001, + "loss": 75.3625, + "step": 1030 + }, + { + "epoch": 16.78048780487805, + "grad_norm": 1.5828496217727661, + "learning_rate": 0.0001, + "loss": 78.0302, + "step": 1040 + }, + { + "epoch": 16.943089430894307, + "grad_norm": 1.7372915744781494, + "learning_rate": 0.0001, + "loss": 78.6949, + "step": 1050 + }, + { + "epoch": 17.097560975609756, + "grad_norm": 1.837161898612976, + "learning_rate": 0.0001, + "loss": 74.4726, + "step": 1060 + }, + { + "epoch": 17.260162601626018, + "grad_norm": 3.8733808994293213, + "learning_rate": 0.0001, + "loss": 76.1711, + "step": 1070 + }, + { + "epoch": 17.422764227642276, + "grad_norm": 1.5907851457595825, + "learning_rate": 0.0001, + "loss": 75.8292, + "step": 1080 + }, + { + "epoch": 17.585365853658537, + "grad_norm": 1.8271747827529907, + "learning_rate": 0.0001, + "loss": 74.5316, + "step": 1090 + }, + { + "epoch": 17.747967479674795, + "grad_norm": 1.827980875968933, + "learning_rate": 0.0001, + "loss": 74.7281, + "step": 1100 + }, + { + "epoch": 17.910569105691057, + "grad_norm": 1.4184074401855469, + "learning_rate": 0.0001, + "loss": 75.736, + "step": 1110 + }, + { + "epoch": 18.065040650406505, + "grad_norm": 2.3459699153900146, + "learning_rate": 0.0001, + "loss": 71.9586, + "step": 1120 + }, + { + "epoch": 18.227642276422763, + "grad_norm": 47.673641204833984, + "learning_rate": 0.0001, + "loss": 75.6649, + "step": 1130 + }, + { + "epoch": 18.390243902439025, + "grad_norm": 1.677398443222046, + "learning_rate": 0.0001, + "loss": 76.0574, + "step": 1140 + }, + { + "epoch": 18.552845528455286, + "grad_norm": 1.4180850982666016, + "learning_rate": 0.0001, + "loss": 74.3077, + "step": 1150 + }, + { + "epoch": 18.715447154471544, + "grad_norm": 2.0041465759277344, + "learning_rate": 0.0001, + "loss": 73.5916, + "step": 1160 + }, + { + "epoch": 18.878048780487806, + "grad_norm": 1.9098238945007324, + "learning_rate": 0.0001, + "loss": 74.5002, + "step": 1170 + }, + { + "epoch": 19.03252032520325, + "grad_norm": 1.405555248260498, + "learning_rate": 0.0001, + "loss": 69.8115, + "step": 1180 + }, + { + "epoch": 19.195121951219512, + "grad_norm": 1.424198865890503, + "learning_rate": 0.0001, + "loss": 73.7289, + "step": 1190 + }, + { + "epoch": 19.357723577235774, + "grad_norm": 7.542550563812256, + "learning_rate": 0.0001, + "loss": 73.7696, + "step": 1200 + }, + { + "epoch": 19.520325203252032, + "grad_norm": 1.425329566001892, + "learning_rate": 0.0001, + "loss": 72.1261, + "step": 1210 + }, + { + "epoch": 19.682926829268293, + "grad_norm": 1.525724172592163, + "learning_rate": 0.0001, + "loss": 73.6578, + "step": 1220 + }, + { + "epoch": 19.84552845528455, + "grad_norm": 1.6387193202972412, + "learning_rate": 0.0001, + "loss": 74.3027, + "step": 1230 + }, + { + "epoch": 20.0, + "grad_norm": 1.8429840803146362, + "learning_rate": 0.0001, + "loss": 69.5056, + "step": 1240 + }, + { + "epoch": 20.16260162601626, + "grad_norm": 2.5166006088256836, + "learning_rate": 0.0001, + "loss": 73.5965, + "step": 1250 + }, + { + "epoch": 20.32520325203252, + "grad_norm": 1.6006672382354736, + "learning_rate": 0.0001, + "loss": 72.4581, + "step": 1260 + }, + { + "epoch": 20.48780487804878, + "grad_norm": 1.6722878217697144, + "learning_rate": 0.0001, + "loss": 72.3097, + "step": 1270 + }, + { + "epoch": 20.65040650406504, + "grad_norm": 1.8715728521347046, + "learning_rate": 0.0001, + "loss": 73.4752, + "step": 1280 + }, + { + "epoch": 20.8130081300813, + "grad_norm": 1.3280900716781616, + "learning_rate": 0.0001, + "loss": 73.2145, + "step": 1290 + }, + { + "epoch": 20.975609756097562, + "grad_norm": 1.941451907157898, + "learning_rate": 0.0001, + "loss": 73.2085, + "step": 1300 + }, + { + "epoch": 21.130081300813007, + "grad_norm": 4.277497291564941, + "learning_rate": 0.0001, + "loss": 68.8305, + "step": 1310 + }, + { + "epoch": 21.29268292682927, + "grad_norm": 1.9949846267700195, + "learning_rate": 0.0001, + "loss": 72.6689, + "step": 1320 + }, + { + "epoch": 21.45528455284553, + "grad_norm": 1.524044394493103, + "learning_rate": 0.0001, + "loss": 72.9705, + "step": 1330 + }, + { + "epoch": 21.617886178861788, + "grad_norm": 1.384194016456604, + "learning_rate": 0.0001, + "loss": 72.5465, + "step": 1340 + }, + { + "epoch": 21.78048780487805, + "grad_norm": 1.724225640296936, + "learning_rate": 0.0001, + "loss": 72.9203, + "step": 1350 + }, + { + "epoch": 21.943089430894307, + "grad_norm": 2.0856704711914062, + "learning_rate": 0.0001, + "loss": 72.4034, + "step": 1360 + }, + { + "epoch": 22.097560975609756, + "grad_norm": 5.128044128417969, + "learning_rate": 0.0001, + "loss": 68.7754, + "step": 1370 + }, + { + "epoch": 22.260162601626018, + "grad_norm": 1.7385634183883667, + "learning_rate": 0.0001, + "loss": 73.9349, + "step": 1380 + }, + { + "epoch": 22.422764227642276, + "grad_norm": 1.3946012258529663, + "learning_rate": 0.0001, + "loss": 70.9627, + "step": 1390 + }, + { + "epoch": 22.585365853658537, + "grad_norm": 1.285414457321167, + "learning_rate": 0.0001, + "loss": 72.4578, + "step": 1400 + }, + { + "epoch": 22.747967479674795, + "grad_norm": 1.4197710752487183, + "learning_rate": 0.0001, + "loss": 72.5252, + "step": 1410 + }, + { + "epoch": 22.910569105691057, + "grad_norm": 3.048942804336548, + "learning_rate": 0.0001, + "loss": 71.1107, + "step": 1420 + }, + { + "epoch": 23.065040650406505, + "grad_norm": 1.3114765882492065, + "learning_rate": 0.0001, + "loss": 68.2725, + "step": 1430 + }, + { + "epoch": 23.227642276422763, + "grad_norm": 1.2763880491256714, + "learning_rate": 0.0001, + "loss": 71.5783, + "step": 1440 + }, + { + "epoch": 23.390243902439025, + "grad_norm": 1.3913089036941528, + "learning_rate": 0.0001, + "loss": 72.5986, + "step": 1450 + }, + { + "epoch": 23.552845528455286, + "grad_norm": 1.628751516342163, + "learning_rate": 0.0001, + "loss": 71.8383, + "step": 1460 + }, + { + "epoch": 23.715447154471544, + "grad_norm": 1.4331274032592773, + "learning_rate": 0.0001, + "loss": 70.8051, + "step": 1470 + }, + { + "epoch": 23.878048780487806, + "grad_norm": 1.4154727458953857, + "learning_rate": 0.0001, + "loss": 71.3283, + "step": 1480 + }, + { + "epoch": 24.03252032520325, + "grad_norm": 1.7208784818649292, + "learning_rate": 0.0001, + "loss": 68.4719, + "step": 1490 + }, + { + "epoch": 24.195121951219512, + "grad_norm": 1.466123342514038, + "learning_rate": 0.0001, + "loss": 71.9597, + "step": 1500 + }, + { + "epoch": 24.357723577235774, + "grad_norm": 1.4823886156082153, + "learning_rate": 0.0001, + "loss": 72.4533, + "step": 1510 + }, + { + "epoch": 24.520325203252032, + "grad_norm": 1.8407355546951294, + "learning_rate": 0.0001, + "loss": 71.1468, + "step": 1520 + }, + { + "epoch": 24.682926829268293, + "grad_norm": 1.630565881729126, + "learning_rate": 0.0001, + "loss": 71.5192, + "step": 1530 + }, + { + "epoch": 24.84552845528455, + "grad_norm": 1.5248785018920898, + "learning_rate": 0.0001, + "loss": 68.4804, + "step": 1540 + }, + { + "epoch": 25.0, + "grad_norm": 1.4690219163894653, + "learning_rate": 0.0001, + "loss": 67.5716, + "step": 1550 + }, + { + "epoch": 25.16260162601626, + "grad_norm": 2.0246646404266357, + "learning_rate": 0.0001, + "loss": 71.2554, + "step": 1560 + }, + { + "epoch": 25.32520325203252, + "grad_norm": 1.3611458539962769, + "learning_rate": 0.0001, + "loss": 69.8558, + "step": 1570 + }, + { + "epoch": 25.48780487804878, + "grad_norm": 2.0006039142608643, + "learning_rate": 0.0001, + "loss": 72.5344, + "step": 1580 + }, + { + "epoch": 25.65040650406504, + "grad_norm": 1.7373167276382446, + "learning_rate": 0.0001, + "loss": 70.6253, + "step": 1590 + }, + { + "epoch": 25.8130081300813, + "grad_norm": 1.2878378629684448, + "learning_rate": 0.0001, + "loss": 72.165, + "step": 1600 + }, + { + "epoch": 25.975609756097562, + "grad_norm": 1.5337013006210327, + "learning_rate": 0.0001, + "loss": 70.6989, + "step": 1610 + }, + { + "epoch": 26.130081300813007, + "grad_norm": 1.364615559577942, + "learning_rate": 0.0001, + "loss": 67.3898, + "step": 1620 + }, + { + "epoch": 26.29268292682927, + "grad_norm": 1.3249316215515137, + "learning_rate": 0.0001, + "loss": 72.7102, + "step": 1630 + }, + { + "epoch": 26.45528455284553, + "grad_norm": 4.209898471832275, + "learning_rate": 0.0001, + "loss": 70.3185, + "step": 1640 + }, + { + "epoch": 26.617886178861788, + "grad_norm": 1.5488810539245605, + "learning_rate": 0.0001, + "loss": 72.3678, + "step": 1650 + }, + { + "epoch": 26.78048780487805, + "grad_norm": 1.7743597030639648, + "learning_rate": 0.0001, + "loss": 71.0917, + "step": 1660 + }, + { + "epoch": 26.943089430894307, + "grad_norm": 1.5433533191680908, + "learning_rate": 0.0001, + "loss": 71.4001, + "step": 1670 + }, + { + "epoch": 27.097560975609756, + "grad_norm": 1.6320385932922363, + "learning_rate": 0.0001, + "loss": 66.7441, + "step": 1680 + }, + { + "epoch": 27.260162601626018, + "grad_norm": 1.3137969970703125, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 1690 + }, + { + "epoch": 27.422764227642276, + "grad_norm": 1.9150484800338745, + "learning_rate": 0.0001, + "loss": 69.8956, + "step": 1700 + }, + { + "epoch": 27.585365853658537, + "grad_norm": 1.4442802667617798, + "learning_rate": 0.0001, + "loss": 70.7185, + "step": 1710 + }, + { + "epoch": 27.747967479674795, + "grad_norm": 2.572497606277466, + "learning_rate": 0.0001, + "loss": 70.8934, + "step": 1720 + }, + { + "epoch": 27.910569105691057, + "grad_norm": 1.6669617891311646, + "learning_rate": 0.0001, + "loss": 71.8221, + "step": 1730 + }, + { + "epoch": 28.065040650406505, + "grad_norm": 1.8449876308441162, + "learning_rate": 0.0001, + "loss": 66.3923, + "step": 1740 + }, + { + "epoch": 28.227642276422763, + "grad_norm": 1.6600004434585571, + "learning_rate": 0.0001, + "loss": 69.5301, + "step": 1750 + }, + { + "epoch": 28.390243902439025, + "grad_norm": 1.4655731916427612, + "learning_rate": 0.0001, + "loss": 72.1924, + "step": 1760 + }, + { + "epoch": 28.552845528455286, + "grad_norm": 1.134505033493042, + "learning_rate": 0.0001, + "loss": 69.5855, + "step": 1770 + }, + { + "epoch": 28.715447154471544, + "grad_norm": 1.301984190940857, + "learning_rate": 0.0001, + "loss": 70.7999, + "step": 1780 + }, + { + "epoch": 28.878048780487806, + "grad_norm": 1.5641013383865356, + "learning_rate": 0.0001, + "loss": 69.9091, + "step": 1790 + }, + { + "epoch": 29.03252032520325, + "grad_norm": 1.4850797653198242, + "learning_rate": 0.0001, + "loss": 66.6266, + "step": 1800 + }, + { + "epoch": 29.195121951219512, + "grad_norm": 1.2069118022918701, + "learning_rate": 0.0001, + "loss": 69.3802, + "step": 1810 + }, + { + "epoch": 29.357723577235774, + "grad_norm": 1.2890352010726929, + "learning_rate": 0.0001, + "loss": 68.5044, + "step": 1820 + }, + { + "epoch": 29.520325203252032, + "grad_norm": 1.4150917530059814, + "learning_rate": 0.0001, + "loss": 69.807, + "step": 1830 + }, + { + "epoch": 29.682926829268293, + "grad_norm": 1.514807939529419, + "learning_rate": 0.0001, + "loss": 71.3858, + "step": 1840 + }, + { + "epoch": 29.84552845528455, + "grad_norm": 1.4246330261230469, + "learning_rate": 0.0001, + "loss": 70.494, + "step": 1850 + }, + { + "epoch": 30.0, + "grad_norm": 1.1789597272872925, + "learning_rate": 0.0001, + "loss": 67.1056, + "step": 1860 + }, + { + "epoch": 30.16260162601626, + "grad_norm": 1.23678719997406, + "learning_rate": 0.0001, + "loss": 70.336, + "step": 1870 + }, + { + "epoch": 30.32520325203252, + "grad_norm": 1.1756274700164795, + "learning_rate": 0.0001, + "loss": 67.9909, + "step": 1880 + }, + { + "epoch": 30.48780487804878, + "grad_norm": 1.4175690412521362, + "learning_rate": 0.0001, + "loss": 69.2617, + "step": 1890 + }, + { + "epoch": 30.65040650406504, + "grad_norm": 1.3084465265274048, + "learning_rate": 0.0001, + "loss": 70.2963, + "step": 1900 + }, + { + "epoch": 30.8130081300813, + "grad_norm": 1.486464500427246, + "learning_rate": 0.0001, + "loss": 68.6317, + "step": 1910 + }, + { + "epoch": 30.975609756097562, + "grad_norm": 10.415762901306152, + "learning_rate": 0.0001, + "loss": 69.7767, + "step": 1920 + }, + { + "epoch": 31.130081300813007, + "grad_norm": 1.4484038352966309, + "learning_rate": 0.0001, + "loss": 65.6345, + "step": 1930 + }, + { + "epoch": 31.29268292682927, + "grad_norm": 2.297306537628174, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 1940 + }, + { + "epoch": 31.45528455284553, + "grad_norm": 1.223490595817566, + "learning_rate": 0.0001, + "loss": 69.2139, + "step": 1950 + }, + { + "epoch": 31.617886178861788, + "grad_norm": 1.2144125699996948, + "learning_rate": 0.0001, + "loss": 70.3061, + "step": 1960 + }, + { + "epoch": 31.78048780487805, + "grad_norm": 2.633528470993042, + "learning_rate": 0.0001, + "loss": 68.7096, + "step": 1970 + }, + { + "epoch": 31.943089430894307, + "grad_norm": 1.5048350095748901, + "learning_rate": 0.0001, + "loss": 68.6711, + "step": 1980 + }, + { + "epoch": 32.09756097560975, + "grad_norm": 1.5154467821121216, + "learning_rate": 0.0001, + "loss": 66.7989, + "step": 1990 + }, + { + "epoch": 32.260162601626014, + "grad_norm": 2.721277952194214, + "learning_rate": 0.0001, + "loss": 70.4987, + "step": 2000 + }, + { + "epoch": 32.422764227642276, + "grad_norm": 1.44464910030365, + "learning_rate": 0.0001, + "loss": 69.9712, + "step": 2010 + }, + { + "epoch": 32.58536585365854, + "grad_norm": 1.3462835550308228, + "learning_rate": 0.0001, + "loss": 68.9381, + "step": 2020 + }, + { + "epoch": 32.7479674796748, + "grad_norm": 1.3283029794692993, + "learning_rate": 0.0001, + "loss": 68.5301, + "step": 2030 + }, + { + "epoch": 32.91056910569106, + "grad_norm": 1.5172960758209229, + "learning_rate": 0.0001, + "loss": 70.1894, + "step": 2040 + }, + { + "epoch": 33.0650406504065, + "grad_norm": 1.1872576475143433, + "learning_rate": 0.0001, + "loss": 65.0158, + "step": 2050 + }, + { + "epoch": 33.22764227642276, + "grad_norm": 1.5414419174194336, + "learning_rate": 0.0001, + "loss": 69.1537, + "step": 2060 + }, + { + "epoch": 33.390243902439025, + "grad_norm": 1.3143430948257446, + "learning_rate": 0.0001, + "loss": 68.6792, + "step": 2070 + }, + { + "epoch": 33.552845528455286, + "grad_norm": 1.3863723278045654, + "learning_rate": 0.0001, + "loss": 70.3867, + "step": 2080 + }, + { + "epoch": 33.71544715447155, + "grad_norm": 1.4060479402542114, + "learning_rate": 0.0001, + "loss": 68.4981, + "step": 2090 + }, + { + "epoch": 33.8780487804878, + "grad_norm": 1.34132981300354, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 2100 + }, + { + "epoch": 34.03252032520325, + "grad_norm": 1.8303673267364502, + "learning_rate": 0.0001, + "loss": 65.819, + "step": 2110 + }, + { + "epoch": 34.19512195121951, + "grad_norm": 1.387339472770691, + "learning_rate": 0.0001, + "loss": 69.9825, + "step": 2120 + }, + { + "epoch": 34.357723577235774, + "grad_norm": 1.4125195741653442, + "learning_rate": 0.0001, + "loss": 67.5149, + "step": 2130 + }, + { + "epoch": 34.520325203252035, + "grad_norm": 1.2909412384033203, + "learning_rate": 0.0001, + "loss": 69.7848, + "step": 2140 + }, + { + "epoch": 34.68292682926829, + "grad_norm": 1.8860795497894287, + "learning_rate": 0.0001, + "loss": 68.6145, + "step": 2150 + }, + { + "epoch": 34.84552845528455, + "grad_norm": 1.5111079216003418, + "learning_rate": 0.0001, + "loss": 68.2612, + "step": 2160 + }, + { + "epoch": 35.0, + "grad_norm": 1.3275264501571655, + "learning_rate": 0.0001, + "loss": 64.0298, + "step": 2170 + }, + { + "epoch": 35.16260162601626, + "grad_norm": 1.174864411354065, + "learning_rate": 0.0001, + "loss": 66.6153, + "step": 2180 + }, + { + "epoch": 35.32520325203252, + "grad_norm": 1.6873915195465088, + "learning_rate": 0.0001, + "loss": 69.6278, + "step": 2190 + }, + { + "epoch": 35.48780487804878, + "grad_norm": 2.4457507133483887, + "learning_rate": 0.0001, + "loss": 68.3611, + "step": 2200 + }, + { + "epoch": 35.65040650406504, + "grad_norm": 1.4779958724975586, + "learning_rate": 0.0001, + "loss": 69.2609, + "step": 2210 + }, + { + "epoch": 35.8130081300813, + "grad_norm": 2.273709774017334, + "learning_rate": 0.0001, + "loss": 69.1312, + "step": 2220 + }, + { + "epoch": 35.97560975609756, + "grad_norm": 1.5741862058639526, + "learning_rate": 0.0001, + "loss": 69.1191, + "step": 2230 + }, + { + "epoch": 36.13008130081301, + "grad_norm": 1.2730036973953247, + "learning_rate": 0.0001, + "loss": 65.692, + "step": 2240 + }, + { + "epoch": 36.292682926829265, + "grad_norm": 1.2936429977416992, + "learning_rate": 0.0001, + "loss": 68.2962, + "step": 2250 + }, + { + "epoch": 36.45528455284553, + "grad_norm": 1.354279637336731, + "learning_rate": 0.0001, + "loss": 70.5507, + "step": 2260 + }, + { + "epoch": 36.61788617886179, + "grad_norm": 1.4165853261947632, + "learning_rate": 0.0001, + "loss": 68.0048, + "step": 2270 + }, + { + "epoch": 36.78048780487805, + "grad_norm": 1.2992445230484009, + "learning_rate": 0.0001, + "loss": 67.7072, + "step": 2280 + }, + { + "epoch": 36.94308943089431, + "grad_norm": 1.5053461790084839, + "learning_rate": 0.0001, + "loss": 66.7587, + "step": 2290 + }, + { + "epoch": 37.09756097560975, + "grad_norm": 1.337390661239624, + "learning_rate": 0.0001, + "loss": 63.786, + "step": 2300 + }, + { + "epoch": 37.260162601626014, + "grad_norm": 1.0407774448394775, + "learning_rate": 0.0001, + "loss": 66.6253, + "step": 2310 + }, + { + "epoch": 37.422764227642276, + "grad_norm": 1.564152479171753, + "learning_rate": 0.0001, + "loss": 68.3606, + "step": 2320 + }, + { + "epoch": 37.58536585365854, + "grad_norm": 1.2028454542160034, + "learning_rate": 0.0001, + "loss": 66.2841, + "step": 2330 + }, + { + "epoch": 37.7479674796748, + "grad_norm": 1.4528603553771973, + "learning_rate": 0.0001, + "loss": 67.4277, + "step": 2340 + }, + { + "epoch": 37.91056910569106, + "grad_norm": 1.4954335689544678, + "learning_rate": 0.0001, + "loss": 68.9954, + "step": 2350 + }, + { + "epoch": 38.0650406504065, + "grad_norm": 1.2820416688919067, + "learning_rate": 0.0001, + "loss": 65.2449, + "step": 2360 + }, + { + "epoch": 38.22764227642276, + "grad_norm": 1.2392579317092896, + "learning_rate": 0.0001, + "loss": 67.5006, + "step": 2370 + }, + { + "epoch": 38.390243902439025, + "grad_norm": 1.2135238647460938, + "learning_rate": 0.0001, + "loss": 69.2214, + "step": 2380 + }, + { + "epoch": 38.552845528455286, + "grad_norm": 1.5964744091033936, + "learning_rate": 0.0001, + "loss": 67.934, + "step": 2390 + }, + { + "epoch": 38.71544715447155, + "grad_norm": 1.2841702699661255, + "learning_rate": 0.0001, + "loss": 67.0848, + "step": 2400 + }, + { + "epoch": 38.8780487804878, + "grad_norm": 1.8026081323623657, + "learning_rate": 0.0001, + "loss": 67.5358, + "step": 2410 + }, + { + "epoch": 39.03252032520325, + "grad_norm": 1.4178807735443115, + "learning_rate": 0.0001, + "loss": 64.771, + "step": 2420 + }, + { + "epoch": 39.19512195121951, + "grad_norm": 1.3671854734420776, + "learning_rate": 0.0001, + "loss": 68.0531, + "step": 2430 + }, + { + "epoch": 39.357723577235774, + "grad_norm": 1.43596613407135, + "learning_rate": 0.0001, + "loss": 67.7735, + "step": 2440 + }, + { + "epoch": 39.520325203252035, + "grad_norm": 1.464250922203064, + "learning_rate": 0.0001, + "loss": 67.8331, + "step": 2450 + }, + { + "epoch": 39.68292682926829, + "grad_norm": 3.101461410522461, + "learning_rate": 0.0001, + "loss": 65.8956, + "step": 2460 + }, + { + "epoch": 39.84552845528455, + "grad_norm": 1.8332289457321167, + "learning_rate": 0.0001, + "loss": 66.8218, + "step": 2470 + }, + { + "epoch": 40.0, + "grad_norm": 1.2660984992980957, + "learning_rate": 0.0001, + "loss": 63.9969, + "step": 2480 + }, + { + "epoch": 40.16260162601626, + "grad_norm": 2.0309898853302, + "learning_rate": 0.0001, + "loss": 67.3538, + "step": 2490 + }, + { + "epoch": 40.32520325203252, + "grad_norm": 1.207000494003296, + "learning_rate": 0.0001, + "loss": 68.0898, + "step": 2500 + }, + { + "epoch": 40.48780487804878, + "grad_norm": 1.3609018325805664, + "learning_rate": 0.0001, + "loss": 67.0381, + "step": 2510 + }, + { + "epoch": 40.65040650406504, + "grad_norm": 1.310826301574707, + "learning_rate": 0.0001, + "loss": 69.0267, + "step": 2520 + }, + { + "epoch": 40.8130081300813, + "grad_norm": 1.4150960445404053, + "learning_rate": 0.0001, + "loss": 69.1934, + "step": 2530 + }, + { + "epoch": 40.97560975609756, + "grad_norm": 1.5414319038391113, + "learning_rate": 0.0001, + "loss": 68.0604, + "step": 2540 + }, + { + "epoch": 41.13008130081301, + "grad_norm": 2.3246140480041504, + "learning_rate": 0.0001, + "loss": 61.5698, + "step": 2550 + }, + { + "epoch": 41.292682926829265, + "grad_norm": 1.206296443939209, + "learning_rate": 0.0001, + "loss": 64.9898, + "step": 2560 + }, + { + "epoch": 41.45528455284553, + "grad_norm": 1.6080626249313354, + "learning_rate": 0.0001, + "loss": 68.2685, + "step": 2570 + }, + { + "epoch": 41.61788617886179, + "grad_norm": 1.2404112815856934, + "learning_rate": 0.0001, + "loss": 67.8457, + "step": 2580 + }, + { + "epoch": 41.78048780487805, + "grad_norm": 1.2304514646530151, + "learning_rate": 0.0001, + "loss": 67.9052, + "step": 2590 + }, + { + "epoch": 41.94308943089431, + "grad_norm": 1.5806187391281128, + "learning_rate": 0.0001, + "loss": 66.0663, + "step": 2600 + }, + { + "epoch": 42.09756097560975, + "grad_norm": 1.2564151287078857, + "learning_rate": 0.0001, + "loss": 65.7763, + "step": 2610 + }, + { + "epoch": 42.260162601626014, + "grad_norm": 1.4972114562988281, + "learning_rate": 0.0001, + "loss": 66.4269, + "step": 2620 + }, + { + "epoch": 42.422764227642276, + "grad_norm": 1.2913843393325806, + "learning_rate": 0.0001, + "loss": 67.7554, + "step": 2630 + }, + { + "epoch": 42.58536585365854, + "grad_norm": 1.3743863105773926, + "learning_rate": 0.0001, + "loss": 67.2627, + "step": 2640 + }, + { + "epoch": 42.7479674796748, + "grad_norm": 1.1267883777618408, + "learning_rate": 0.0001, + "loss": 67.4597, + "step": 2650 + }, + { + "epoch": 42.91056910569106, + "grad_norm": 2.0194661617279053, + "learning_rate": 0.0001, + "loss": 66.9295, + "step": 2660 + }, + { + "epoch": 43.0650406504065, + "grad_norm": 1.273779273033142, + "learning_rate": 0.0001, + "loss": 60.7906, + "step": 2670 + }, + { + "epoch": 43.22764227642276, + "grad_norm": 1.1498609781265259, + "learning_rate": 0.0001, + "loss": 68.0405, + "step": 2680 + }, + { + "epoch": 43.390243902439025, + "grad_norm": 1.4689130783081055, + "learning_rate": 0.0001, + "loss": 66.2358, + "step": 2690 + }, + { + "epoch": 43.552845528455286, + "grad_norm": 1.4495564699172974, + "learning_rate": 0.0001, + "loss": 67.4267, + "step": 2700 + }, + { + "epoch": 43.71544715447155, + "grad_norm": 2.5769612789154053, + "learning_rate": 0.0001, + "loss": 66.953, + "step": 2710 + }, + { + "epoch": 43.8780487804878, + "grad_norm": 2.1684205532073975, + "learning_rate": 0.0001, + "loss": 65.6393, + "step": 2720 + }, + { + "epoch": 44.03252032520325, + "grad_norm": 1.4968472719192505, + "learning_rate": 0.0001, + "loss": 60.8256, + "step": 2730 + }, + { + "epoch": 44.19512195121951, + "grad_norm": 1.1874353885650635, + "learning_rate": 0.0001, + "loss": 66.818, + "step": 2740 + }, + { + "epoch": 44.357723577235774, + "grad_norm": 2.271235227584839, + "learning_rate": 0.0001, + "loss": 66.5048, + "step": 2750 + }, + { + "epoch": 44.520325203252035, + "grad_norm": 1.20489501953125, + "learning_rate": 0.0001, + "loss": 67.0487, + "step": 2760 + }, + { + "epoch": 44.68292682926829, + "grad_norm": 1.6273691654205322, + "learning_rate": 0.0001, + "loss": 66.2228, + "step": 2770 + }, + { + "epoch": 44.84552845528455, + "grad_norm": 1.4935433864593506, + "learning_rate": 0.0001, + "loss": 67.2039, + "step": 2780 + }, + { + "epoch": 45.0, + "grad_norm": 1.2668659687042236, + "learning_rate": 0.0001, + "loss": 62.7855, + "step": 2790 + }, + { + "epoch": 45.16260162601626, + "grad_norm": 1.6439573764801025, + "learning_rate": 0.0001, + "loss": 67.919, + "step": 2800 + }, + { + "epoch": 45.32520325203252, + "grad_norm": 1.311854600906372, + "learning_rate": 0.0001, + "loss": 65.1797, + "step": 2810 + }, + { + "epoch": 45.48780487804878, + "grad_norm": 1.3596776723861694, + "learning_rate": 0.0001, + "loss": 67.469, + "step": 2820 + }, + { + "epoch": 45.65040650406504, + "grad_norm": 1.5618443489074707, + "learning_rate": 0.0001, + "loss": 66.1808, + "step": 2830 + }, + { + "epoch": 45.8130081300813, + "grad_norm": 1.7551205158233643, + "learning_rate": 0.0001, + "loss": 65.3803, + "step": 2840 + }, + { + "epoch": 45.97560975609756, + "grad_norm": 1.433794379234314, + "learning_rate": 0.0001, + "loss": 67.8314, + "step": 2850 + }, + { + "epoch": 46.13008130081301, + "grad_norm": 1.4634851217269897, + "learning_rate": 0.0001, + "loss": 63.4219, + "step": 2860 + }, + { + "epoch": 46.292682926829265, + "grad_norm": 1.083052396774292, + "learning_rate": 0.0001, + "loss": 68.0719, + "step": 2870 + }, + { + "epoch": 46.45528455284553, + "grad_norm": 1.1784703731536865, + "learning_rate": 0.0001, + "loss": 66.1391, + "step": 2880 + }, + { + "epoch": 46.61788617886179, + "grad_norm": 1.2552465200424194, + "learning_rate": 0.0001, + "loss": 65.6006, + "step": 2890 + }, + { + "epoch": 46.78048780487805, + "grad_norm": 1.4684522151947021, + "learning_rate": 0.0001, + "loss": 66.8314, + "step": 2900 + }, + { + "epoch": 46.94308943089431, + "grad_norm": 1.741514801979065, + "learning_rate": 0.0001, + "loss": 67.5056, + "step": 2910 + }, + { + "epoch": 47.09756097560975, + "grad_norm": 5.696534156799316, + "learning_rate": 0.0001, + "loss": 62.8016, + "step": 2920 + }, + { + "epoch": 47.260162601626014, + "grad_norm": 1.5000969171524048, + "learning_rate": 0.0001, + "loss": 68.0272, + "step": 2930 + }, + { + "epoch": 47.422764227642276, + "grad_norm": 1.1044933795928955, + "learning_rate": 0.0001, + "loss": 67.0905, + "step": 2940 + }, + { + "epoch": 47.58536585365854, + "grad_norm": 0.9933589100837708, + "learning_rate": 0.0001, + "loss": 67.6077, + "step": 2950 + }, + { + "epoch": 47.7479674796748, + "grad_norm": 1.2908599376678467, + "learning_rate": 0.0001, + "loss": 65.869, + "step": 2960 + }, + { + "epoch": 47.91056910569106, + "grad_norm": 1.0470529794692993, + "learning_rate": 0.0001, + "loss": 66.5353, + "step": 2970 + }, + { + "epoch": 48.0650406504065, + "grad_norm": 1.0927976369857788, + "learning_rate": 0.0001, + "loss": 62.7541, + "step": 2980 + }, + { + "epoch": 48.22764227642276, + "grad_norm": 1.0870513916015625, + "learning_rate": 0.0001, + "loss": 64.172, + "step": 2990 + }, + { + "epoch": 48.390243902439025, + "grad_norm": 1.3492090702056885, + "learning_rate": 0.0001, + "loss": 67.6642, + "step": 3000 + }, + { + "epoch": 48.552845528455286, + "grad_norm": 1.4024657011032104, + "learning_rate": 0.0001, + "loss": 65.2722, + "step": 3010 + }, + { + "epoch": 48.71544715447155, + "grad_norm": 1.1165252923965454, + "learning_rate": 0.0001, + "loss": 66.7233, + "step": 3020 + }, + { + "epoch": 48.8780487804878, + "grad_norm": 1.2147928476333618, + "learning_rate": 0.0001, + "loss": 65.7277, + "step": 3030 + }, + { + "epoch": 49.03252032520325, + "grad_norm": 0.925310492515564, + "learning_rate": 0.0001, + "loss": 62.9256, + "step": 3040 + }, + { + "epoch": 49.19512195121951, + "grad_norm": 1.2353442907333374, + "learning_rate": 0.0001, + "loss": 66.1218, + "step": 3050 + }, + { + "epoch": 49.357723577235774, + "grad_norm": 1.2177367210388184, + "learning_rate": 0.0001, + "loss": 66.7978, + "step": 3060 + }, + { + "epoch": 49.520325203252035, + "grad_norm": 1.248044729232788, + "learning_rate": 0.0001, + "loss": 66.2132, + "step": 3070 + }, + { + "epoch": 49.68292682926829, + "grad_norm": 1.366050124168396, + "learning_rate": 0.0001, + "loss": 67.0118, + "step": 3080 + }, + { + "epoch": 49.84552845528455, + "grad_norm": 1.3701151609420776, + "learning_rate": 0.0001, + "loss": 67.6099, + "step": 3090 + }, + { + "epoch": 50.0, + "grad_norm": 1.0420141220092773, + "learning_rate": 0.0001, + "loss": 61.5289, + "step": 3100 + }, + { + "epoch": 50.16260162601626, + "grad_norm": 1.0957746505737305, + "learning_rate": 0.0001, + "loss": 66.7348, + "step": 3110 + }, + { + "epoch": 50.32520325203252, + "grad_norm": 1.108628749847412, + "learning_rate": 0.0001, + "loss": 66.6199, + "step": 3120 + }, + { + "epoch": 50.48780487804878, + "grad_norm": 1.3680156469345093, + "learning_rate": 0.0001, + "loss": 67.0825, + "step": 3130 + }, + { + "epoch": 50.65040650406504, + "grad_norm": 1.169851303100586, + "learning_rate": 0.0001, + "loss": 65.4901, + "step": 3140 + }, + { + "epoch": 50.8130081300813, + "grad_norm": 2.1822028160095215, + "learning_rate": 0.0001, + "loss": 65.6841, + "step": 3150 + }, + { + "epoch": 50.97560975609756, + "grad_norm": 1.053402304649353, + "learning_rate": 0.0001, + "loss": 66.3119, + "step": 3160 + }, + { + "epoch": 51.13008130081301, + "grad_norm": 1.0627789497375488, + "learning_rate": 0.0001, + "loss": 61.3642, + "step": 3170 + }, + { + "epoch": 51.292682926829265, + "grad_norm": 1.926291823387146, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 3180 + }, + { + "epoch": 51.45528455284553, + "grad_norm": 1.5791983604431152, + "learning_rate": 0.0001, + "loss": 63.913, + "step": 3190 + }, + { + "epoch": 51.61788617886179, + "grad_norm": 1.210265040397644, + "learning_rate": 0.0001, + "loss": 64.5261, + "step": 3200 + }, + { + "epoch": 51.78048780487805, + "grad_norm": 1.0720808506011963, + "learning_rate": 0.0001, + "loss": 67.0194, + "step": 3210 + }, + { + "epoch": 51.94308943089431, + "grad_norm": 1.94563889503479, + "learning_rate": 0.0001, + "loss": 68.4662, + "step": 3220 + }, + { + "epoch": 52.09756097560975, + "grad_norm": 1.598980188369751, + "learning_rate": 0.0001, + "loss": 62.4946, + "step": 3230 + }, + { + "epoch": 52.260162601626014, + "grad_norm": 1.0564576387405396, + "learning_rate": 0.0001, + "loss": 65.7912, + "step": 3240 + }, + { + "epoch": 52.422764227642276, + "grad_norm": 1.2189645767211914, + "learning_rate": 0.0001, + "loss": 65.5772, + "step": 3250 + }, + { + "epoch": 52.58536585365854, + "grad_norm": 1.1357049942016602, + "learning_rate": 0.0001, + "loss": 65.1559, + "step": 3260 + }, + { + "epoch": 52.7479674796748, + "grad_norm": 1.316867709159851, + "learning_rate": 0.0001, + "loss": 64.7897, + "step": 3270 + }, + { + "epoch": 52.91056910569106, + "grad_norm": 1.3870197534561157, + "learning_rate": 0.0001, + "loss": 65.5195, + "step": 3280 + }, + { + "epoch": 53.0650406504065, + "grad_norm": 1.2467927932739258, + "learning_rate": 0.0001, + "loss": 61.3392, + "step": 3290 + }, + { + "epoch": 53.22764227642276, + "grad_norm": 1.1661109924316406, + "learning_rate": 0.0001, + "loss": 66.0117, + "step": 3300 + }, + { + "epoch": 53.390243902439025, + "grad_norm": 1.3509628772735596, + "learning_rate": 0.0001, + "loss": 64.7228, + "step": 3310 + }, + { + "epoch": 53.552845528455286, + "grad_norm": 1.1093642711639404, + "learning_rate": 0.0001, + "loss": 65.2534, + "step": 3320 + }, + { + "epoch": 53.71544715447155, + "grad_norm": 1.1801207065582275, + "learning_rate": 0.0001, + "loss": 65.7097, + "step": 3330 + }, + { + "epoch": 53.8780487804878, + "grad_norm": 1.175884485244751, + "learning_rate": 0.0001, + "loss": 64.4368, + "step": 3340 + }, + { + "epoch": 54.03252032520325, + "grad_norm": 1.1727889776229858, + "learning_rate": 0.0001, + "loss": 62.9373, + "step": 3350 + }, + { + "epoch": 54.19512195121951, + "grad_norm": 1.5330945253372192, + "learning_rate": 0.0001, + "loss": 66.7763, + "step": 3360 + }, + { + "epoch": 54.357723577235774, + "grad_norm": 1.2992504835128784, + "learning_rate": 0.0001, + "loss": 65.7854, + "step": 3370 + }, + { + "epoch": 54.520325203252035, + "grad_norm": 1.0185420513153076, + "learning_rate": 0.0001, + "loss": 65.7046, + "step": 3380 + }, + { + "epoch": 54.68292682926829, + "grad_norm": 1.426229476928711, + "learning_rate": 0.0001, + "loss": 64.0286, + "step": 3390 + }, + { + "epoch": 54.84552845528455, + "grad_norm": 1.2600868940353394, + "learning_rate": 0.0001, + "loss": 66.1738, + "step": 3400 + }, + { + "epoch": 55.0, + "grad_norm": 1.917455792427063, + "learning_rate": 0.0001, + "loss": 61.9381, + "step": 3410 + }, + { + "epoch": 55.16260162601626, + "grad_norm": 1.2000917196273804, + "learning_rate": 0.0001, + "loss": 65.5626, + "step": 3420 + }, + { + "epoch": 55.32520325203252, + "grad_norm": 1.362176775932312, + "learning_rate": 0.0001, + "loss": 66.8012, + "step": 3430 + }, + { + "epoch": 55.48780487804878, + "grad_norm": 1.1611402034759521, + "learning_rate": 0.0001, + "loss": 64.5227, + "step": 3440 + }, + { + "epoch": 55.65040650406504, + "grad_norm": 1.2066621780395508, + "learning_rate": 0.0001, + "loss": 63.4351, + "step": 3450 + }, + { + "epoch": 55.8130081300813, + "grad_norm": 1.005340814590454, + "learning_rate": 0.0001, + "loss": 65.9261, + "step": 3460 + }, + { + "epoch": 55.97560975609756, + "grad_norm": 1.0906637907028198, + "learning_rate": 0.0001, + "loss": 64.6635, + "step": 3470 + }, + { + "epoch": 56.13008130081301, + "grad_norm": 1.0608748197555542, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 3480 + }, + { + "epoch": 56.292682926829265, + "grad_norm": 1.025319218635559, + "learning_rate": 0.0001, + "loss": 64.7783, + "step": 3490 + }, + { + "epoch": 56.45528455284553, + "grad_norm": 1.1596784591674805, + "learning_rate": 0.0001, + "loss": 66.4629, + "step": 3500 + }, + { + "epoch": 56.61788617886179, + "grad_norm": 0.9318640232086182, + "learning_rate": 0.0001, + "loss": 65.5533, + "step": 3510 + }, + { + "epoch": 56.78048780487805, + "grad_norm": 1.0977418422698975, + "learning_rate": 0.0001, + "loss": 63.647, + "step": 3520 + }, + { + "epoch": 56.94308943089431, + "grad_norm": 1.3912436962127686, + "learning_rate": 0.0001, + "loss": 62.8273, + "step": 3530 + }, + { + "epoch": 57.09756097560975, + "grad_norm": 1.0921591520309448, + "learning_rate": 0.0001, + "loss": 61.5968, + "step": 3540 + }, + { + "epoch": 57.260162601626014, + "grad_norm": 1.17819344997406, + "learning_rate": 0.0001, + "loss": 64.9381, + "step": 3550 + }, + { + "epoch": 57.422764227642276, + "grad_norm": 1.188829779624939, + "learning_rate": 0.0001, + "loss": 64.6498, + "step": 3560 + }, + { + "epoch": 57.58536585365854, + "grad_norm": 3.664839506149292, + "learning_rate": 0.0001, + "loss": 65.3834, + "step": 3570 + }, + { + "epoch": 57.7479674796748, + "grad_norm": 1.57958984375, + "learning_rate": 0.0001, + "loss": 63.8808, + "step": 3580 + }, + { + "epoch": 57.91056910569106, + "grad_norm": 1.1837552785873413, + "learning_rate": 0.0001, + "loss": 64.1545, + "step": 3590 + }, + { + "epoch": 58.0650406504065, + "grad_norm": 1.5986013412475586, + "learning_rate": 0.0001, + "loss": 60.5742, + "step": 3600 + }, + { + "epoch": 58.22764227642276, + "grad_norm": 1.1551810503005981, + "learning_rate": 0.0001, + "loss": 66.2378, + "step": 3610 + }, + { + "epoch": 58.390243902439025, + "grad_norm": 1.0869863033294678, + "learning_rate": 0.0001, + "loss": 65.3034, + "step": 3620 + }, + { + "epoch": 58.552845528455286, + "grad_norm": 1.2263212203979492, + "learning_rate": 0.0001, + "loss": 63.976, + "step": 3630 + }, + { + "epoch": 58.71544715447155, + "grad_norm": 1.4410817623138428, + "learning_rate": 0.0001, + "loss": 63.4778, + "step": 3640 + }, + { + "epoch": 58.8780487804878, + "grad_norm": 1.9633759260177612, + "learning_rate": 0.0001, + "loss": 65.5946, + "step": 3650 + }, + { + "epoch": 59.03252032520325, + "grad_norm": 1.1729766130447388, + "learning_rate": 0.0001, + "loss": 62.2171, + "step": 3660 + }, + { + "epoch": 59.19512195121951, + "grad_norm": 1.0049556493759155, + "learning_rate": 0.0001, + "loss": 65.1854, + "step": 3670 + }, + { + "epoch": 59.357723577235774, + "grad_norm": 1.1727055311203003, + "learning_rate": 0.0001, + "loss": 65.4839, + "step": 3680 + }, + { + "epoch": 59.520325203252035, + "grad_norm": 1.5394458770751953, + "learning_rate": 0.0001, + "loss": 64.9341, + "step": 3690 + }, + { + "epoch": 59.68292682926829, + "grad_norm": 1.1583774089813232, + "learning_rate": 0.0001, + "loss": 63.9197, + "step": 3700 + }, + { + "epoch": 59.84552845528455, + "grad_norm": 1.0835607051849365, + "learning_rate": 0.0001, + "loss": 64.0494, + "step": 3710 + }, + { + "epoch": 60.0, + "grad_norm": 1.7504246234893799, + "learning_rate": 0.0001, + "loss": 59.4134, + "step": 3720 + }, + { + "epoch": 60.16260162601626, + "grad_norm": 1.1870343685150146, + "learning_rate": 0.0001, + "loss": 63.2158, + "step": 3730 + }, + { + "epoch": 60.32520325203252, + "grad_norm": 1.1983181238174438, + "learning_rate": 0.0001, + "loss": 64.7144, + "step": 3740 + }, + { + "epoch": 60.48780487804878, + "grad_norm": 1.3014581203460693, + "learning_rate": 0.0001, + "loss": 64.6219, + "step": 3750 + }, + { + "epoch": 60.65040650406504, + "grad_norm": 1.4954087734222412, + "learning_rate": 0.0001, + "loss": 63.3785, + "step": 3760 + }, + { + "epoch": 60.8130081300813, + "grad_norm": 1.2236649990081787, + "learning_rate": 0.0001, + "loss": 64.6933, + "step": 3770 + }, + { + "epoch": 60.97560975609756, + "grad_norm": 1.0796610116958618, + "learning_rate": 0.0001, + "loss": 63.2704, + "step": 3780 + }, + { + "epoch": 61.13008130081301, + "grad_norm": 0.9969968795776367, + "learning_rate": 0.0001, + "loss": 62.1838, + "step": 3790 + }, + { + "epoch": 61.292682926829265, + "grad_norm": 1.1624361276626587, + "learning_rate": 0.0001, + "loss": 63.6249, + "step": 3800 + }, + { + "epoch": 61.45528455284553, + "grad_norm": 1.3849223852157593, + "learning_rate": 0.0001, + "loss": 64.5321, + "step": 3810 + }, + { + "epoch": 61.61788617886179, + "grad_norm": 1.6607458591461182, + "learning_rate": 0.0001, + "loss": 64.651, + "step": 3820 + }, + { + "epoch": 61.78048780487805, + "grad_norm": 1.09684419631958, + "learning_rate": 0.0001, + "loss": 64.1129, + "step": 3830 + }, + { + "epoch": 61.94308943089431, + "grad_norm": 1.406246304512024, + "learning_rate": 0.0001, + "loss": 64.3642, + "step": 3840 + }, + { + "epoch": 62.09756097560975, + "grad_norm": 0.9850211143493652, + "learning_rate": 0.0001, + "loss": 60.8703, + "step": 3850 + }, + { + "epoch": 62.260162601626014, + "grad_norm": 1.001344084739685, + "learning_rate": 0.0001, + "loss": 67.5523, + "step": 3860 + }, + { + "epoch": 62.422764227642276, + "grad_norm": 2.435969591140747, + "learning_rate": 0.0001, + "loss": 62.7363, + "step": 3870 + }, + { + "epoch": 62.58536585365854, + "grad_norm": 1.0675824880599976, + "learning_rate": 0.0001, + "loss": 64.8513, + "step": 3880 + }, + { + "epoch": 62.7479674796748, + "grad_norm": 0.9793754816055298, + "learning_rate": 0.0001, + "loss": 64.9904, + "step": 3890 + }, + { + "epoch": 62.91056910569106, + "grad_norm": 1.2187775373458862, + "learning_rate": 0.0001, + "loss": 63.2821, + "step": 3900 + }, + { + "epoch": 63.0650406504065, + "grad_norm": 1.130772590637207, + "learning_rate": 0.0001, + "loss": 61.5957, + "step": 3910 + }, + { + "epoch": 63.22764227642276, + "grad_norm": 1.0573055744171143, + "learning_rate": 0.0001, + "loss": 63.6337, + "step": 3920 + }, + { + "epoch": 63.390243902439025, + "grad_norm": 2.055490255355835, + "learning_rate": 0.0001, + "loss": 63.9598, + "step": 3930 + }, + { + "epoch": 63.552845528455286, + "grad_norm": 1.0780516862869263, + "learning_rate": 0.0001, + "loss": 62.9303, + "step": 3940 + }, + { + "epoch": 63.71544715447155, + "grad_norm": 1.8136590719223022, + "learning_rate": 0.0001, + "loss": 64.4588, + "step": 3950 + }, + { + "epoch": 63.8780487804878, + "grad_norm": 1.0525016784667969, + "learning_rate": 0.0001, + "loss": 64.192, + "step": 3960 + }, + { + "epoch": 64.03252032520325, + "grad_norm": 1.12627375125885, + "learning_rate": 0.0001, + "loss": 59.1823, + "step": 3970 + }, + { + "epoch": 64.1951219512195, + "grad_norm": 1.2558730840682983, + "learning_rate": 0.0001, + "loss": 65.126, + "step": 3980 + }, + { + "epoch": 64.35772357723577, + "grad_norm": 1.2304726839065552, + "learning_rate": 0.0001, + "loss": 62.0868, + "step": 3990 + }, + { + "epoch": 64.52032520325203, + "grad_norm": 1.4179201126098633, + "learning_rate": 0.0001, + "loss": 62.3114, + "step": 4000 + }, + { + "epoch": 64.6829268292683, + "grad_norm": 1.125758409500122, + "learning_rate": 0.0001, + "loss": 63.4103, + "step": 4010 + }, + { + "epoch": 64.84552845528455, + "grad_norm": 2.321124315261841, + "learning_rate": 0.0001, + "loss": 62.9032, + "step": 4020 + }, + { + "epoch": 65.0, + "grad_norm": 1.4433993101119995, + "learning_rate": 0.0001, + "loss": 61.8766, + "step": 4030 + }, + { + "epoch": 65.16260162601625, + "grad_norm": 1.157878041267395, + "learning_rate": 0.0001, + "loss": 62.4833, + "step": 4040 + }, + { + "epoch": 65.32520325203252, + "grad_norm": 1.4127166271209717, + "learning_rate": 0.0001, + "loss": 64.768, + "step": 4050 + }, + { + "epoch": 65.48780487804878, + "grad_norm": 1.2575284242630005, + "learning_rate": 0.0001, + "loss": 62.9401, + "step": 4060 + }, + { + "epoch": 65.65040650406505, + "grad_norm": 1.0957911014556885, + "learning_rate": 0.0001, + "loss": 64.6061, + "step": 4070 + }, + { + "epoch": 65.8130081300813, + "grad_norm": 1.1773414611816406, + "learning_rate": 0.0001, + "loss": 63.9469, + "step": 4080 + }, + { + "epoch": 65.97560975609755, + "grad_norm": 2.1189942359924316, + "learning_rate": 0.0001, + "loss": 65.1422, + "step": 4090 + }, + { + "epoch": 66.130081300813, + "grad_norm": 1.1064234972000122, + "learning_rate": 0.0001, + "loss": 59.6108, + "step": 4100 + }, + { + "epoch": 66.29268292682927, + "grad_norm": 1.1135871410369873, + "learning_rate": 0.0001, + "loss": 63.4036, + "step": 4110 + }, + { + "epoch": 66.45528455284553, + "grad_norm": 4.167781829833984, + "learning_rate": 0.0001, + "loss": 63.1174, + "step": 4120 + }, + { + "epoch": 66.6178861788618, + "grad_norm": 1.1429376602172852, + "learning_rate": 0.0001, + "loss": 64.4085, + "step": 4130 + }, + { + "epoch": 66.78048780487805, + "grad_norm": 1.3188339471817017, + "learning_rate": 0.0001, + "loss": 62.5997, + "step": 4140 + }, + { + "epoch": 66.9430894308943, + "grad_norm": 1.160335659980774, + "learning_rate": 0.0001, + "loss": 64.2911, + "step": 4150 + }, + { + "epoch": 67.09756097560975, + "grad_norm": 0.9261873960494995, + "learning_rate": 0.0001, + "loss": 61.9036, + "step": 4160 + }, + { + "epoch": 67.26016260162602, + "grad_norm": 1.2254258394241333, + "learning_rate": 0.0001, + "loss": 64.1092, + "step": 4170 + }, + { + "epoch": 67.42276422764228, + "grad_norm": 0.9519097208976746, + "learning_rate": 0.0001, + "loss": 61.6428, + "step": 4180 + }, + { + "epoch": 67.58536585365853, + "grad_norm": 1.2290887832641602, + "learning_rate": 0.0001, + "loss": 64.9842, + "step": 4190 + }, + { + "epoch": 67.7479674796748, + "grad_norm": 1.0995723009109497, + "learning_rate": 0.0001, + "loss": 64.0677, + "step": 4200 + }, + { + "epoch": 67.91056910569105, + "grad_norm": 2.727957010269165, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 4210 + }, + { + "epoch": 68.0650406504065, + "grad_norm": 1.1222411394119263, + "learning_rate": 0.0001, + "loss": 61.0693, + "step": 4220 + }, + { + "epoch": 68.22764227642277, + "grad_norm": 1.159303069114685, + "learning_rate": 0.0001, + "loss": 64.2469, + "step": 4230 + }, + { + "epoch": 68.39024390243902, + "grad_norm": 1.2205990552902222, + "learning_rate": 0.0001, + "loss": 63.8831, + "step": 4240 + }, + { + "epoch": 68.55284552845528, + "grad_norm": 1.2351205348968506, + "learning_rate": 0.0001, + "loss": 63.4294, + "step": 4250 + }, + { + "epoch": 68.71544715447155, + "grad_norm": 1.4483757019042969, + "learning_rate": 0.0001, + "loss": 63.5345, + "step": 4260 + }, + { + "epoch": 68.8780487804878, + "grad_norm": 1.0808624029159546, + "learning_rate": 0.0001, + "loss": 63.6885, + "step": 4270 + }, + { + "epoch": 69.03252032520325, + "grad_norm": 1.7775999307632446, + "learning_rate": 0.0001, + "loss": 60.7266, + "step": 4280 + }, + { + "epoch": 69.1951219512195, + "grad_norm": 1.3214768171310425, + "learning_rate": 0.0001, + "loss": 63.3496, + "step": 4290 + }, + { + "epoch": 69.35772357723577, + "grad_norm": 1.1698002815246582, + "learning_rate": 0.0001, + "loss": 63.5284, + "step": 4300 + }, + { + "epoch": 69.52032520325203, + "grad_norm": 1.2050056457519531, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 4310 + }, + { + "epoch": 69.6829268292683, + "grad_norm": 1.0873243808746338, + "learning_rate": 0.0001, + "loss": 65.1709, + "step": 4320 + }, + { + "epoch": 69.84552845528455, + "grad_norm": 1.14133620262146, + "learning_rate": 0.0001, + "loss": 63.1872, + "step": 4330 + }, + { + "epoch": 70.0, + "grad_norm": 0.9684088230133057, + "learning_rate": 0.0001, + "loss": 60.1486, + "step": 4340 + }, + { + "epoch": 70.16260162601625, + "grad_norm": 1.1206653118133545, + "learning_rate": 0.0001, + "loss": 61.117, + "step": 4350 + }, + { + "epoch": 70.32520325203252, + "grad_norm": 1.2754334211349487, + "learning_rate": 0.0001, + "loss": 62.5409, + "step": 4360 + }, + { + "epoch": 70.48780487804878, + "grad_norm": 1.0154718160629272, + "learning_rate": 0.0001, + "loss": 62.9428, + "step": 4370 + }, + { + "epoch": 70.65040650406505, + "grad_norm": 1.0597270727157593, + "learning_rate": 0.0001, + "loss": 63.079, + "step": 4380 + }, + { + "epoch": 70.8130081300813, + "grad_norm": 1.254758596420288, + "learning_rate": 0.0001, + "loss": 63.3035, + "step": 4390 + }, + { + "epoch": 70.97560975609755, + "grad_norm": 1.0185120105743408, + "learning_rate": 0.0001, + "loss": 63.8922, + "step": 4400 + }, + { + "epoch": 71.130081300813, + "grad_norm": 1.007300853729248, + "learning_rate": 0.0001, + "loss": 58.7735, + "step": 4410 + }, + { + "epoch": 71.29268292682927, + "grad_norm": 1.2012463808059692, + "learning_rate": 0.0001, + "loss": 63.1782, + "step": 4420 + }, + { + "epoch": 71.45528455284553, + "grad_norm": 1.0258845090866089, + "learning_rate": 0.0001, + "loss": 63.2483, + "step": 4430 + }, + { + "epoch": 71.6178861788618, + "grad_norm": 1.0611220598220825, + "learning_rate": 0.0001, + "loss": 63.2247, + "step": 4440 + }, + { + "epoch": 71.78048780487805, + "grad_norm": 1.060114860534668, + "learning_rate": 0.0001, + "loss": 61.8352, + "step": 4450 + }, + { + "epoch": 71.9430894308943, + "grad_norm": 1.0540589094161987, + "learning_rate": 0.0001, + "loss": 62.1156, + "step": 4460 + }, + { + "epoch": 72.09756097560975, + "grad_norm": 1.4923726320266724, + "learning_rate": 0.0001, + "loss": 59.3863, + "step": 4470 + }, + { + "epoch": 72.26016260162602, + "grad_norm": 1.13791024684906, + "learning_rate": 0.0001, + "loss": 62.203, + "step": 4480 + }, + { + "epoch": 72.42276422764228, + "grad_norm": 0.9737065434455872, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 4490 + }, + { + "epoch": 72.58536585365853, + "grad_norm": 1.3980190753936768, + "learning_rate": 0.0001, + "loss": 63.5627, + "step": 4500 + }, + { + "epoch": 72.7479674796748, + "grad_norm": 1.0274118185043335, + "learning_rate": 0.0001, + "loss": 63.6129, + "step": 4510 + }, + { + "epoch": 72.91056910569105, + "grad_norm": 1.3148925304412842, + "learning_rate": 0.0001, + "loss": 63.6506, + "step": 4520 + }, + { + "epoch": 73.0650406504065, + "grad_norm": 1.4182451963424683, + "learning_rate": 0.0001, + "loss": 60.3616, + "step": 4530 + }, + { + "epoch": 73.22764227642277, + "grad_norm": 0.9235166907310486, + "learning_rate": 0.0001, + "loss": 63.6465, + "step": 4540 + }, + { + "epoch": 73.39024390243902, + "grad_norm": 0.9293469786643982, + "learning_rate": 0.0001, + "loss": 61.0984, + "step": 4550 + }, + { + "epoch": 73.55284552845528, + "grad_norm": 1.1075892448425293, + "learning_rate": 0.0001, + "loss": 62.9356, + "step": 4560 + }, + { + "epoch": 73.71544715447155, + "grad_norm": 1.4955459833145142, + "learning_rate": 0.0001, + "loss": 61.3811, + "step": 4570 + }, + { + "epoch": 73.8780487804878, + "grad_norm": 1.220595359802246, + "learning_rate": 0.0001, + "loss": 62.4364, + "step": 4580 + }, + { + "epoch": 74.03252032520325, + "grad_norm": 0.9975704550743103, + "learning_rate": 0.0001, + "loss": 58.5194, + "step": 4590 + }, + { + "epoch": 74.1951219512195, + "grad_norm": 1.0160857439041138, + "learning_rate": 0.0001, + "loss": 64.6989, + "step": 4600 + }, + { + "epoch": 74.35772357723577, + "grad_norm": 1.0052287578582764, + "learning_rate": 0.0001, + "loss": 62.526, + "step": 4610 + }, + { + "epoch": 74.52032520325203, + "grad_norm": 1.015514612197876, + "learning_rate": 0.0001, + "loss": 63.8627, + "step": 4620 + }, + { + "epoch": 74.6829268292683, + "grad_norm": 1.139051914215088, + "learning_rate": 0.0001, + "loss": 62.4525, + "step": 4630 + }, + { + "epoch": 74.84552845528455, + "grad_norm": 1.1727714538574219, + "learning_rate": 0.0001, + "loss": 64.0046, + "step": 4640 + }, + { + "epoch": 75.0, + "grad_norm": 1.1611062288284302, + "learning_rate": 0.0001, + "loss": 57.6443, + "step": 4650 + }, + { + "epoch": 75.16260162601625, + "grad_norm": 0.9893108606338501, + "learning_rate": 0.0001, + "loss": 64.1109, + "step": 4660 + }, + { + "epoch": 75.32520325203252, + "grad_norm": 1.190146803855896, + "learning_rate": 0.0001, + "loss": 62.1988, + "step": 4670 + }, + { + "epoch": 75.48780487804878, + "grad_norm": 1.1648579835891724, + "learning_rate": 0.0001, + "loss": 63.1393, + "step": 4680 + }, + { + "epoch": 75.65040650406505, + "grad_norm": 1.3303166627883911, + "learning_rate": 0.0001, + "loss": 63.7, + "step": 4690 + }, + { + "epoch": 75.8130081300813, + "grad_norm": 1.407772421836853, + "learning_rate": 0.0001, + "loss": 60.6798, + "step": 4700 + }, + { + "epoch": 75.97560975609755, + "grad_norm": 1.039741039276123, + "learning_rate": 0.0001, + "loss": 61.6126, + "step": 4710 + }, + { + "epoch": 76.130081300813, + "grad_norm": 1.013515830039978, + "learning_rate": 0.0001, + "loss": 57.8452, + "step": 4720 + }, + { + "epoch": 76.29268292682927, + "grad_norm": 1.1279789209365845, + "learning_rate": 0.0001, + "loss": 63.0592, + "step": 4730 + }, + { + "epoch": 76.45528455284553, + "grad_norm": 1.158605694770813, + "learning_rate": 0.0001, + "loss": 62.0836, + "step": 4740 + }, + { + "epoch": 76.6178861788618, + "grad_norm": 1.0200021266937256, + "learning_rate": 0.0001, + "loss": 62.1775, + "step": 4750 + }, + { + "epoch": 76.78048780487805, + "grad_norm": 1.216412901878357, + "learning_rate": 0.0001, + "loss": 62.6316, + "step": 4760 + }, + { + "epoch": 76.9430894308943, + "grad_norm": 1.0513523817062378, + "learning_rate": 0.0001, + "loss": 60.1037, + "step": 4770 + }, + { + "epoch": 77.09756097560975, + "grad_norm": 1.10075843334198, + "learning_rate": 0.0001, + "loss": 58.7599, + "step": 4780 + }, + { + "epoch": 77.26016260162602, + "grad_norm": 1.1728724241256714, + "learning_rate": 0.0001, + "loss": 63.3476, + "step": 4790 + }, + { + "epoch": 77.42276422764228, + "grad_norm": 1.3511605262756348, + "learning_rate": 0.0001, + "loss": 60.9885, + "step": 4800 + }, + { + "epoch": 77.58536585365853, + "grad_norm": 0.9733042120933533, + "learning_rate": 0.0001, + "loss": 62.4245, + "step": 4810 + }, + { + "epoch": 77.7479674796748, + "grad_norm": 1.0380535125732422, + "learning_rate": 0.0001, + "loss": 64.0755, + "step": 4820 + }, + { + "epoch": 77.91056910569105, + "grad_norm": 1.1594985723495483, + "learning_rate": 0.0001, + "loss": 62.1099, + "step": 4830 + }, + { + "epoch": 78.0650406504065, + "grad_norm": 1.0392569303512573, + "learning_rate": 0.0001, + "loss": 58.3778, + "step": 4840 + }, + { + "epoch": 78.22764227642277, + "grad_norm": 1.0032405853271484, + "learning_rate": 0.0001, + "loss": 62.1384, + "step": 4850 + }, + { + "epoch": 78.39024390243902, + "grad_norm": 1.0576013326644897, + "learning_rate": 0.0001, + "loss": 63.3308, + "step": 4860 + }, + { + "epoch": 78.55284552845528, + "grad_norm": 1.34328031539917, + "learning_rate": 0.0001, + "loss": 62.5351, + "step": 4870 + }, + { + "epoch": 78.71544715447155, + "grad_norm": 1.1040822267532349, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 4880 + }, + { + "epoch": 78.8780487804878, + "grad_norm": 1.1554768085479736, + "learning_rate": 0.0001, + "loss": 62.7308, + "step": 4890 + }, + { + "epoch": 79.03252032520325, + "grad_norm": 1.1864408254623413, + "learning_rate": 0.0001, + "loss": 60.0741, + "step": 4900 + }, + { + "epoch": 79.1951219512195, + "grad_norm": 0.9181821346282959, + "learning_rate": 0.0001, + "loss": 63.1682, + "step": 4910 + }, + { + "epoch": 79.35772357723577, + "grad_norm": 1.1400946378707886, + "learning_rate": 0.0001, + "loss": 62.0736, + "step": 4920 + }, + { + "epoch": 79.52032520325203, + "grad_norm": 1.245202660560608, + "learning_rate": 0.0001, + "loss": 60.51, + "step": 4930 + }, + { + "epoch": 79.6829268292683, + "grad_norm": 1.033347725868225, + "learning_rate": 0.0001, + "loss": 61.7845, + "step": 4940 + }, + { + "epoch": 79.84552845528455, + "grad_norm": 1.6206163167953491, + "learning_rate": 0.0001, + "loss": 61.5232, + "step": 4950 + }, + { + "epoch": 80.0, + "grad_norm": 1.0924937725067139, + "learning_rate": 0.0001, + "loss": 57.24, + "step": 4960 + }, + { + "epoch": 80.16260162601625, + "grad_norm": 1.075985074043274, + "learning_rate": 0.0001, + "loss": 61.2615, + "step": 4970 + }, + { + "epoch": 80.32520325203252, + "grad_norm": 0.9964771866798401, + "learning_rate": 0.0001, + "loss": 62.287, + "step": 4980 + }, + { + "epoch": 80.48780487804878, + "grad_norm": 1.3707735538482666, + "learning_rate": 0.0001, + "loss": 60.9383, + "step": 4990 + }, + { + "epoch": 80.65040650406505, + "grad_norm": 0.9395170211791992, + "learning_rate": 0.0001, + "loss": 61.2697, + "step": 5000 + }, + { + "epoch": 80.8130081300813, + "grad_norm": 2.0208873748779297, + "learning_rate": 0.0001, + "loss": 61.7219, + "step": 5010 + }, + { + "epoch": 80.97560975609755, + "grad_norm": 1.468702793121338, + "learning_rate": 0.0001, + "loss": 62.6967, + "step": 5020 + }, + { + "epoch": 81.130081300813, + "grad_norm": 1.0525966882705688, + "learning_rate": 0.0001, + "loss": 59.9793, + "step": 5030 + }, + { + "epoch": 81.29268292682927, + "grad_norm": 1.0951132774353027, + "learning_rate": 0.0001, + "loss": 60.9807, + "step": 5040 + }, + { + "epoch": 81.45528455284553, + "grad_norm": 1.3457857370376587, + "learning_rate": 0.0001, + "loss": 61.4137, + "step": 5050 + }, + { + "epoch": 81.6178861788618, + "grad_norm": 1.0895649194717407, + "learning_rate": 0.0001, + "loss": 61.2747, + "step": 5060 + }, + { + "epoch": 81.78048780487805, + "grad_norm": 1.0552371740341187, + "learning_rate": 0.0001, + "loss": 62.0947, + "step": 5070 + }, + { + "epoch": 81.9430894308943, + "grad_norm": 1.039038896560669, + "learning_rate": 0.0001, + "loss": 61.2571, + "step": 5080 + }, + { + "epoch": 82.09756097560975, + "grad_norm": 0.970783531665802, + "learning_rate": 0.0001, + "loss": 57.4157, + "step": 5090 + }, + { + "epoch": 82.26016260162602, + "grad_norm": 1.0276020765304565, + "learning_rate": 0.0001, + "loss": 62.1324, + "step": 5100 + }, + { + "epoch": 82.42276422764228, + "grad_norm": 1.2092448472976685, + "learning_rate": 0.0001, + "loss": 60.6448, + "step": 5110 + }, + { + "epoch": 82.58536585365853, + "grad_norm": 0.9352007508277893, + "learning_rate": 0.0001, + "loss": 61.9299, + "step": 5120 + }, + { + "epoch": 82.7479674796748, + "grad_norm": 1.2678358554840088, + "learning_rate": 0.0001, + "loss": 61.6622, + "step": 5130 + }, + { + "epoch": 82.91056910569105, + "grad_norm": 1.200494647026062, + "learning_rate": 0.0001, + "loss": 62.3169, + "step": 5140 + }, + { + "epoch": 83.0650406504065, + "grad_norm": 1.1284401416778564, + "learning_rate": 0.0001, + "loss": 58.7768, + "step": 5150 + }, + { + "epoch": 83.22764227642277, + "grad_norm": 1.1578079462051392, + "learning_rate": 0.0001, + "loss": 62.8423, + "step": 5160 + }, + { + "epoch": 83.39024390243902, + "grad_norm": 1.3426504135131836, + "learning_rate": 0.0001, + "loss": 61.5332, + "step": 5170 + }, + { + "epoch": 83.55284552845528, + "grad_norm": 0.9397301077842712, + "learning_rate": 0.0001, + "loss": 60.1043, + "step": 5180 + }, + { + "epoch": 83.71544715447155, + "grad_norm": 1.0728143453598022, + "learning_rate": 0.0001, + "loss": 60.5927, + "step": 5190 + }, + { + "epoch": 83.8780487804878, + "grad_norm": 1.2620211839675903, + "learning_rate": 0.0001, + "loss": 60.9224, + "step": 5200 + }, + { + "epoch": 84.03252032520325, + "grad_norm": 0.8232231140136719, + "learning_rate": 0.0001, + "loss": 58.8128, + "step": 5210 + }, + { + "epoch": 84.1951219512195, + "grad_norm": 1.0222564935684204, + "learning_rate": 0.0001, + "loss": 59.6995, + "step": 5220 + }, + { + "epoch": 84.35772357723577, + "grad_norm": 0.9078415632247925, + "learning_rate": 0.0001, + "loss": 61.1521, + "step": 5230 + }, + { + "epoch": 84.52032520325203, + "grad_norm": 1.0269399881362915, + "learning_rate": 0.0001, + "loss": 61.0136, + "step": 5240 + }, + { + "epoch": 84.6829268292683, + "grad_norm": 1.2108154296875, + "learning_rate": 0.0001, + "loss": 61.3564, + "step": 5250 + }, + { + "epoch": 84.84552845528455, + "grad_norm": 1.1170622110366821, + "learning_rate": 0.0001, + "loss": 61.7047, + "step": 5260 + }, + { + "epoch": 85.0, + "grad_norm": 1.467298150062561, + "learning_rate": 0.0001, + "loss": 58.1667, + "step": 5270 + }, + { + "epoch": 85.16260162601625, + "grad_norm": 0.908743143081665, + "learning_rate": 0.0001, + "loss": 61.5694, + "step": 5280 + }, + { + "epoch": 85.32520325203252, + "grad_norm": 0.9148766398429871, + "learning_rate": 0.0001, + "loss": 61.5325, + "step": 5290 + }, + { + "epoch": 85.48780487804878, + "grad_norm": 0.940021276473999, + "learning_rate": 0.0001, + "loss": 62.8807, + "step": 5300 + }, + { + "epoch": 85.65040650406505, + "grad_norm": 1.1239454746246338, + "learning_rate": 0.0001, + "loss": 62.361, + "step": 5310 + }, + { + "epoch": 85.8130081300813, + "grad_norm": 0.840437114238739, + "learning_rate": 0.0001, + "loss": 61.9057, + "step": 5320 + }, + { + "epoch": 85.97560975609755, + "grad_norm": 0.9764263033866882, + "learning_rate": 0.0001, + "loss": 61.0075, + "step": 5330 + }, + { + "epoch": 86.130081300813, + "grad_norm": 0.998540997505188, + "learning_rate": 0.0001, + "loss": 57.7146, + "step": 5340 + }, + { + "epoch": 86.29268292682927, + "grad_norm": 1.121551513671875, + "learning_rate": 0.0001, + "loss": 62.2278, + "step": 5350 + }, + { + "epoch": 86.45528455284553, + "grad_norm": 1.101842999458313, + "learning_rate": 0.0001, + "loss": 60.8837, + "step": 5360 + }, + { + "epoch": 86.6178861788618, + "grad_norm": 1.1724908351898193, + "learning_rate": 0.0001, + "loss": 61.6788, + "step": 5370 + }, + { + "epoch": 86.78048780487805, + "grad_norm": 1.3108958005905151, + "learning_rate": 0.0001, + "loss": 60.1461, + "step": 5380 + }, + { + "epoch": 86.9430894308943, + "grad_norm": 1.482298731803894, + "learning_rate": 0.0001, + "loss": 61.0457, + "step": 5390 + }, + { + "epoch": 87.09756097560975, + "grad_norm": 1.1471034288406372, + "learning_rate": 0.0001, + "loss": 57.6835, + "step": 5400 + }, + { + "epoch": 87.26016260162602, + "grad_norm": 0.9349374175071716, + "learning_rate": 0.0001, + "loss": 62.7018, + "step": 5410 + }, + { + "epoch": 87.42276422764228, + "grad_norm": 0.9917856454849243, + "learning_rate": 0.0001, + "loss": 60.2201, + "step": 5420 + }, + { + "epoch": 87.58536585365853, + "grad_norm": 1.0112310647964478, + "learning_rate": 0.0001, + "loss": 62.1185, + "step": 5430 + }, + { + "epoch": 87.7479674796748, + "grad_norm": 1.288894534111023, + "learning_rate": 0.0001, + "loss": 61.4348, + "step": 5440 + }, + { + "epoch": 87.91056910569105, + "grad_norm": 0.8577322363853455, + "learning_rate": 0.0001, + "loss": 60.7867, + "step": 5450 + }, + { + "epoch": 88.0650406504065, + "grad_norm": 0.8162860870361328, + "learning_rate": 0.0001, + "loss": 59.5224, + "step": 5460 + }, + { + "epoch": 88.22764227642277, + "grad_norm": 0.9491421580314636, + "learning_rate": 0.0001, + "loss": 60.4936, + "step": 5470 + }, + { + "epoch": 88.39024390243902, + "grad_norm": 4.164788246154785, + "learning_rate": 0.0001, + "loss": 61.8397, + "step": 5480 + }, + { + "epoch": 88.55284552845528, + "grad_norm": 0.9954016804695129, + "learning_rate": 0.0001, + "loss": 61.286, + "step": 5490 + }, + { + "epoch": 88.71544715447155, + "grad_norm": 1.4991120100021362, + "learning_rate": 0.0001, + "loss": 60.6157, + "step": 5500 + }, + { + "epoch": 88.8780487804878, + "grad_norm": 1.3678480386734009, + "learning_rate": 0.0001, + "loss": 62.5187, + "step": 5510 + }, + { + "epoch": 89.03252032520325, + "grad_norm": 1.057661771774292, + "learning_rate": 0.0001, + "loss": 57.5134, + "step": 5520 + }, + { + "epoch": 89.1951219512195, + "grad_norm": 0.8331280946731567, + "learning_rate": 0.0001, + "loss": 62.007, + "step": 5530 + }, + { + "epoch": 89.35772357723577, + "grad_norm": 0.9842398762702942, + "learning_rate": 0.0001, + "loss": 60.6178, + "step": 5540 + }, + { + "epoch": 89.52032520325203, + "grad_norm": 1.0510483980178833, + "learning_rate": 0.0001, + "loss": 59.6382, + "step": 5550 + }, + { + "epoch": 89.6829268292683, + "grad_norm": 1.013115406036377, + "learning_rate": 0.0001, + "loss": 61.015, + "step": 5560 + }, + { + "epoch": 89.84552845528455, + "grad_norm": 1.1595944166183472, + "learning_rate": 0.0001, + "loss": 62.5374, + "step": 5570 + }, + { + "epoch": 90.0, + "grad_norm": 1.1506270170211792, + "learning_rate": 0.0001, + "loss": 61.0063, + "step": 5580 + }, + { + "epoch": 90.16260162601625, + "grad_norm": 0.9940153360366821, + "learning_rate": 0.0001, + "loss": 62.0944, + "step": 5590 + }, + { + "epoch": 90.32520325203252, + "grad_norm": 1.1302493810653687, + "learning_rate": 0.0001, + "loss": 61.616, + "step": 5600 + }, + { + "epoch": 90.48780487804878, + "grad_norm": 1.2370656728744507, + "learning_rate": 0.0001, + "loss": 61.291, + "step": 5610 + }, + { + "epoch": 90.65040650406505, + "grad_norm": 1.228763461112976, + "learning_rate": 0.0001, + "loss": 60.0629, + "step": 5620 + }, + { + "epoch": 90.8130081300813, + "grad_norm": 1.2230069637298584, + "learning_rate": 0.0001, + "loss": 61.5391, + "step": 5630 + }, + { + "epoch": 90.97560975609755, + "grad_norm": 0.9849008917808533, + "learning_rate": 0.0001, + "loss": 63.1633, + "step": 5640 + }, + { + "epoch": 91.130081300813, + "grad_norm": 1.408632755279541, + "learning_rate": 0.0001, + "loss": 58.4195, + "step": 5650 + }, + { + "epoch": 91.29268292682927, + "grad_norm": 0.929027795791626, + "learning_rate": 0.0001, + "loss": 60.9862, + "step": 5660 + }, + { + "epoch": 91.45528455284553, + "grad_norm": 0.9228377342224121, + "learning_rate": 0.0001, + "loss": 60.8337, + "step": 5670 + }, + { + "epoch": 91.6178861788618, + "grad_norm": 1.0354220867156982, + "learning_rate": 0.0001, + "loss": 59.4667, + "step": 5680 + }, + { + "epoch": 91.78048780487805, + "grad_norm": 1.0338927507400513, + "learning_rate": 0.0001, + "loss": 62.0064, + "step": 5690 + }, + { + "epoch": 91.9430894308943, + "grad_norm": 1.101967215538025, + "learning_rate": 0.0001, + "loss": 61.6317, + "step": 5700 + }, + { + "epoch": 92.09756097560975, + "grad_norm": 1.370689034461975, + "learning_rate": 0.0001, + "loss": 57.2157, + "step": 5710 + }, + { + "epoch": 92.26016260162602, + "grad_norm": 0.9904115796089172, + "learning_rate": 0.0001, + "loss": 61.2786, + "step": 5720 + }, + { + "epoch": 92.42276422764228, + "grad_norm": 1.0545448064804077, + "learning_rate": 0.0001, + "loss": 63.2384, + "step": 5730 + }, + { + "epoch": 92.58536585365853, + "grad_norm": 1.0657731294631958, + "learning_rate": 0.0001, + "loss": 60.582, + "step": 5740 + }, + { + "epoch": 92.7479674796748, + "grad_norm": 0.9333183765411377, + "learning_rate": 0.0001, + "loss": 62.4627, + "step": 5750 + }, + { + "epoch": 92.91056910569105, + "grad_norm": 0.9810492396354675, + "learning_rate": 0.0001, + "loss": 60.0055, + "step": 5760 + }, + { + "epoch": 93.0650406504065, + "grad_norm": 1.6973884105682373, + "learning_rate": 0.0001, + "loss": 57.5381, + "step": 5770 + }, + { + "epoch": 93.22764227642277, + "grad_norm": 1.0075498819351196, + "learning_rate": 0.0001, + "loss": 61.2272, + "step": 5780 + }, + { + "epoch": 93.39024390243902, + "grad_norm": 0.980633556842804, + "learning_rate": 0.0001, + "loss": 60.4794, + "step": 5790 + }, + { + "epoch": 93.55284552845528, + "grad_norm": 1.128214955329895, + "learning_rate": 0.0001, + "loss": 60.4825, + "step": 5800 + }, + { + "epoch": 93.71544715447155, + "grad_norm": 1.302827000617981, + "learning_rate": 0.0001, + "loss": 62.4397, + "step": 5810 + }, + { + "epoch": 93.8780487804878, + "grad_norm": 0.826600193977356, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 5820 + }, + { + "epoch": 94.03252032520325, + "grad_norm": 11.418835639953613, + "learning_rate": 0.0001, + "loss": 58.5383, + "step": 5830 + }, + { + "epoch": 94.1951219512195, + "grad_norm": 1.0686477422714233, + "learning_rate": 0.0001, + "loss": 60.5696, + "step": 5840 + }, + { + "epoch": 94.35772357723577, + "grad_norm": 1.1232597827911377, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 5850 + }, + { + "epoch": 94.52032520325203, + "grad_norm": 1.1051944494247437, + "learning_rate": 0.0001, + "loss": 60.5387, + "step": 5860 + }, + { + "epoch": 94.6829268292683, + "grad_norm": 0.9954739809036255, + "learning_rate": 0.0001, + "loss": 61.3899, + "step": 5870 + }, + { + "epoch": 94.84552845528455, + "grad_norm": 1.9687999486923218, + "learning_rate": 0.0001, + "loss": 60.7711, + "step": 5880 + }, + { + "epoch": 95.0, + "grad_norm": 1.3172541856765747, + "learning_rate": 0.0001, + "loss": 58.5394, + "step": 5890 + }, + { + "epoch": 95.16260162601625, + "grad_norm": 1.2251427173614502, + "learning_rate": 0.0001, + "loss": 61.8319, + "step": 5900 + }, + { + "epoch": 95.32520325203252, + "grad_norm": 1.1701544523239136, + "learning_rate": 0.0001, + "loss": 60.1928, + "step": 5910 + }, + { + "epoch": 95.48780487804878, + "grad_norm": 1.209375262260437, + "learning_rate": 0.0001, + "loss": 61.1347, + "step": 5920 + }, + { + "epoch": 95.65040650406505, + "grad_norm": 0.9307044148445129, + "learning_rate": 0.0001, + "loss": 62.6676, + "step": 5930 + }, + { + "epoch": 95.8130081300813, + "grad_norm": 0.9679144024848938, + "learning_rate": 0.0001, + "loss": 60.7623, + "step": 5940 + }, + { + "epoch": 95.97560975609755, + "grad_norm": 1.398711085319519, + "learning_rate": 0.0001, + "loss": 59.0407, + "step": 5950 + }, + { + "epoch": 96.130081300813, + "grad_norm": 1.9118247032165527, + "learning_rate": 0.0001, + "loss": 59.9417, + "step": 5960 + }, + { + "epoch": 96.29268292682927, + "grad_norm": 1.036737084388733, + "learning_rate": 0.0001, + "loss": 60.6891, + "step": 5970 + }, + { + "epoch": 96.45528455284553, + "grad_norm": 1.5916132926940918, + "learning_rate": 0.0001, + "loss": 59.44, + "step": 5980 + }, + { + "epoch": 96.6178861788618, + "grad_norm": 1.526591181755066, + "learning_rate": 0.0001, + "loss": 58.9081, + "step": 5990 + }, + { + "epoch": 96.78048780487805, + "grad_norm": 0.9556435942649841, + "learning_rate": 0.0001, + "loss": 61.5743, + "step": 6000 + }, + { + "epoch": 96.9430894308943, + "grad_norm": 1.0041887760162354, + "learning_rate": 0.0001, + "loss": 60.0724, + "step": 6010 + }, + { + "epoch": 97.09756097560975, + "grad_norm": 0.9792081117630005, + "learning_rate": 0.0001, + "loss": 60.0613, + "step": 6020 + }, + { + "epoch": 97.26016260162602, + "grad_norm": 1.029634714126587, + "learning_rate": 0.0001, + "loss": 60.3526, + "step": 6030 + }, + { + "epoch": 97.42276422764228, + "grad_norm": 0.9812865257263184, + "learning_rate": 0.0001, + "loss": 60.9818, + "step": 6040 + }, + { + "epoch": 97.58536585365853, + "grad_norm": 0.8826327323913574, + "learning_rate": 0.0001, + "loss": 61.6375, + "step": 6050 + }, + { + "epoch": 97.7479674796748, + "grad_norm": 1.2197599411010742, + "learning_rate": 0.0001, + "loss": 60.2698, + "step": 6060 + }, + { + "epoch": 97.91056910569105, + "grad_norm": 0.9068922400474548, + "learning_rate": 0.0001, + "loss": 58.8698, + "step": 6070 + }, + { + "epoch": 98.0650406504065, + "grad_norm": 0.9008221626281738, + "learning_rate": 0.0001, + "loss": 56.4687, + "step": 6080 + }, + { + "epoch": 98.22764227642277, + "grad_norm": 1.8165878057479858, + "learning_rate": 0.0001, + "loss": 61.4213, + "step": 6090 + }, + { + "epoch": 98.39024390243902, + "grad_norm": 0.9541304111480713, + "learning_rate": 0.0001, + "loss": 62.8381, + "step": 6100 + }, + { + "epoch": 98.55284552845528, + "grad_norm": 1.293872594833374, + "learning_rate": 0.0001, + "loss": 61.3292, + "step": 6110 + }, + { + "epoch": 98.71544715447155, + "grad_norm": 0.9569422006607056, + "learning_rate": 0.0001, + "loss": 61.436, + "step": 6120 + }, + { + "epoch": 98.8780487804878, + "grad_norm": 1.160468578338623, + "learning_rate": 0.0001, + "loss": 60.8419, + "step": 6130 + }, + { + "epoch": 99.03252032520325, + "grad_norm": 1.3102303743362427, + "learning_rate": 0.0001, + "loss": 59.0064, + "step": 6140 + }, + { + "epoch": 99.1951219512195, + "grad_norm": 0.9553463459014893, + "learning_rate": 0.0001, + "loss": 59.925, + "step": 6150 + }, + { + "epoch": 99.35772357723577, + "grad_norm": 1.1993688344955444, + "learning_rate": 0.0001, + "loss": 62.3683, + "step": 6160 + }, + { + "epoch": 99.52032520325203, + "grad_norm": 1.0867350101470947, + "learning_rate": 0.0001, + "loss": 61.8848, + "step": 6170 + }, + { + "epoch": 99.6829268292683, + "grad_norm": 0.9924171566963196, + "learning_rate": 0.0001, + "loss": 60.9243, + "step": 6180 + }, + { + "epoch": 99.84552845528455, + "grad_norm": 0.9548614025115967, + "learning_rate": 0.0001, + "loss": 61.0234, + "step": 6190 + }, + { + "epoch": 100.0, + "grad_norm": 1.0070925951004028, + "learning_rate": 0.0001, + "loss": 56.6554, + "step": 6200 + }, + { + "epoch": 100.16260162601625, + "grad_norm": 1.202168583869934, + "learning_rate": 0.0001, + "loss": 61.3235, + "step": 6210 + }, + { + "epoch": 100.32520325203252, + "grad_norm": 0.9931091070175171, + "learning_rate": 0.0001, + "loss": 59.651, + "step": 6220 + }, + { + "epoch": 100.48780487804878, + "grad_norm": 0.9679157137870789, + "learning_rate": 0.0001, + "loss": 61.5785, + "step": 6230 + }, + { + "epoch": 100.65040650406505, + "grad_norm": 1.049842119216919, + "learning_rate": 0.0001, + "loss": 58.9646, + "step": 6240 + }, + { + "epoch": 100.8130081300813, + "grad_norm": 1.0772044658660889, + "learning_rate": 0.0001, + "loss": 62.3309, + "step": 6250 + }, + { + "epoch": 100.97560975609755, + "grad_norm": 0.9494168162345886, + "learning_rate": 0.0001, + "loss": 61.8663, + "step": 6260 + }, + { + "epoch": 101.130081300813, + "grad_norm": 1.0491962432861328, + "learning_rate": 0.0001, + "loss": 56.5872, + "step": 6270 + }, + { + "epoch": 101.29268292682927, + "grad_norm": 1.118611216545105, + "learning_rate": 0.0001, + "loss": 58.1338, + "step": 6280 + }, + { + "epoch": 101.45528455284553, + "grad_norm": 1.0419825315475464, + "learning_rate": 0.0001, + "loss": 59.6661, + "step": 6290 + }, + { + "epoch": 101.6178861788618, + "grad_norm": 1.2351056337356567, + "learning_rate": 0.0001, + "loss": 61.1697, + "step": 6300 + }, + { + "epoch": 101.78048780487805, + "grad_norm": 1.0556049346923828, + "learning_rate": 0.0001, + "loss": 60.463, + "step": 6310 + }, + { + "epoch": 101.9430894308943, + "grad_norm": 1.090761661529541, + "learning_rate": 0.0001, + "loss": 61.983, + "step": 6320 + }, + { + "epoch": 102.09756097560975, + "grad_norm": 1.4305297136306763, + "learning_rate": 0.0001, + "loss": 55.559, + "step": 6330 + }, + { + "epoch": 102.26016260162602, + "grad_norm": 1.1773794889450073, + "learning_rate": 0.0001, + "loss": 59.5931, + "step": 6340 + }, + { + "epoch": 102.42276422764228, + "grad_norm": 1.0028730630874634, + "learning_rate": 0.0001, + "loss": 60.2714, + "step": 6350 + }, + { + "epoch": 102.58536585365853, + "grad_norm": 0.9494019150733948, + "learning_rate": 0.0001, + "loss": 59.7757, + "step": 6360 + }, + { + "epoch": 102.7479674796748, + "grad_norm": 1.0415751934051514, + "learning_rate": 0.0001, + "loss": 60.7123, + "step": 6370 + }, + { + "epoch": 102.91056910569105, + "grad_norm": 1.2036449909210205, + "learning_rate": 0.0001, + "loss": 60.1201, + "step": 6380 + }, + { + "epoch": 103.0650406504065, + "grad_norm": 0.8801053166389465, + "learning_rate": 0.0001, + "loss": 56.4943, + "step": 6390 + }, + { + "epoch": 103.22764227642277, + "grad_norm": 0.9018735289573669, + "learning_rate": 0.0001, + "loss": 62.4522, + "step": 6400 + }, + { + "epoch": 103.39024390243902, + "grad_norm": 0.971685528755188, + "learning_rate": 0.0001, + "loss": 62.2367, + "step": 6410 + }, + { + "epoch": 103.55284552845528, + "grad_norm": 0.8531002402305603, + "learning_rate": 0.0001, + "loss": 59.5436, + "step": 6420 + }, + { + "epoch": 103.71544715447155, + "grad_norm": 2.123033285140991, + "learning_rate": 0.0001, + "loss": 59.0421, + "step": 6430 + }, + { + "epoch": 103.8780487804878, + "grad_norm": 1.209804654121399, + "learning_rate": 0.0001, + "loss": 60.7791, + "step": 6440 + }, + { + "epoch": 104.03252032520325, + "grad_norm": 0.8219161033630371, + "learning_rate": 0.0001, + "loss": 55.7522, + "step": 6450 + }, + { + "epoch": 104.1951219512195, + "grad_norm": 1.8217495679855347, + "learning_rate": 0.0001, + "loss": 60.6916, + "step": 6460 + }, + { + "epoch": 104.35772357723577, + "grad_norm": 1.060073971748352, + "learning_rate": 0.0001, + "loss": 59.971, + "step": 6470 + }, + { + "epoch": 104.52032520325203, + "grad_norm": 0.9892823696136475, + "learning_rate": 0.0001, + "loss": 61.467, + "step": 6480 + }, + { + "epoch": 104.6829268292683, + "grad_norm": 1.128934621810913, + "learning_rate": 0.0001, + "loss": 62.0451, + "step": 6490 + }, + { + "epoch": 104.84552845528455, + "grad_norm": 0.9627387523651123, + "learning_rate": 0.0001, + "loss": 61.532, + "step": 6500 + }, + { + "epoch": 105.0, + "grad_norm": 1.5344375371932983, + "learning_rate": 0.0001, + "loss": 56.4323, + "step": 6510 + }, + { + "epoch": 105.16260162601625, + "grad_norm": 0.9524087309837341, + "learning_rate": 0.0001, + "loss": 59.6628, + "step": 6520 + }, + { + "epoch": 105.32520325203252, + "grad_norm": 1.0469943284988403, + "learning_rate": 0.0001, + "loss": 59.0605, + "step": 6530 + }, + { + "epoch": 105.48780487804878, + "grad_norm": 2.7462990283966064, + "learning_rate": 0.0001, + "loss": 61.1429, + "step": 6540 + }, + { + "epoch": 105.65040650406505, + "grad_norm": 1.3140382766723633, + "learning_rate": 0.0001, + "loss": 58.2711, + "step": 6550 + }, + { + "epoch": 105.8130081300813, + "grad_norm": 1.1528750658035278, + "learning_rate": 0.0001, + "loss": 59.9937, + "step": 6560 + }, + { + "epoch": 105.97560975609755, + "grad_norm": 1.4449951648712158, + "learning_rate": 0.0001, + "loss": 59.1526, + "step": 6570 + }, + { + "epoch": 106.130081300813, + "grad_norm": 1.1350126266479492, + "learning_rate": 0.0001, + "loss": 55.1142, + "step": 6580 + }, + { + "epoch": 106.29268292682927, + "grad_norm": 0.9332845211029053, + "learning_rate": 0.0001, + "loss": 59.8049, + "step": 6590 + }, + { + "epoch": 106.45528455284553, + "grad_norm": 0.9678266644477844, + "learning_rate": 0.0001, + "loss": 60.9627, + "step": 6600 + }, + { + "epoch": 106.6178861788618, + "grad_norm": 1.1794427633285522, + "learning_rate": 0.0001, + "loss": 59.5372, + "step": 6610 + }, + { + "epoch": 106.78048780487805, + "grad_norm": 1.0404181480407715, + "learning_rate": 0.0001, + "loss": 59.2715, + "step": 6620 + }, + { + "epoch": 106.9430894308943, + "grad_norm": 0.8667073845863342, + "learning_rate": 0.0001, + "loss": 60.2625, + "step": 6630 + }, + { + "epoch": 107.09756097560975, + "grad_norm": 0.9940314888954163, + "learning_rate": 0.0001, + "loss": 55.6906, + "step": 6640 + }, + { + "epoch": 107.26016260162602, + "grad_norm": 1.3880919218063354, + "learning_rate": 0.0001, + "loss": 59.8103, + "step": 6650 + }, + { + "epoch": 107.42276422764228, + "grad_norm": 0.900931179523468, + "learning_rate": 0.0001, + "loss": 59.1929, + "step": 6660 + }, + { + "epoch": 107.58536585365853, + "grad_norm": 0.9374089241027832, + "learning_rate": 0.0001, + "loss": 60.1047, + "step": 6670 + }, + { + "epoch": 107.7479674796748, + "grad_norm": 1.0847140550613403, + "learning_rate": 0.0001, + "loss": 58.6943, + "step": 6680 + }, + { + "epoch": 107.91056910569105, + "grad_norm": 1.086506962776184, + "learning_rate": 0.0001, + "loss": 60.0341, + "step": 6690 + }, + { + "epoch": 108.0650406504065, + "grad_norm": 0.8984777331352234, + "learning_rate": 0.0001, + "loss": 57.0422, + "step": 6700 + }, + { + "epoch": 108.22764227642277, + "grad_norm": 1.187198519706726, + "learning_rate": 0.0001, + "loss": 60.7935, + "step": 6710 + }, + { + "epoch": 108.39024390243902, + "grad_norm": 0.8393696546554565, + "learning_rate": 0.0001, + "loss": 59.7896, + "step": 6720 + }, + { + "epoch": 108.55284552845528, + "grad_norm": 1.0898207426071167, + "learning_rate": 0.0001, + "loss": 60.2579, + "step": 6730 + }, + { + "epoch": 108.71544715447155, + "grad_norm": 1.016926646232605, + "learning_rate": 0.0001, + "loss": 60.9018, + "step": 6740 + }, + { + "epoch": 108.8780487804878, + "grad_norm": 1.3991293907165527, + "learning_rate": 0.0001, + "loss": 60.889, + "step": 6750 + }, + { + "epoch": 109.03252032520325, + "grad_norm": 0.9950843453407288, + "learning_rate": 0.0001, + "loss": 55.1098, + "step": 6760 + }, + { + "epoch": 109.1951219512195, + "grad_norm": 1.0843164920806885, + "learning_rate": 0.0001, + "loss": 60.3208, + "step": 6770 + }, + { + "epoch": 109.35772357723577, + "grad_norm": 0.9067014455795288, + "learning_rate": 0.0001, + "loss": 59.1254, + "step": 6780 + }, + { + "epoch": 109.52032520325203, + "grad_norm": 1.2166807651519775, + "learning_rate": 0.0001, + "loss": 59.9978, + "step": 6790 + }, + { + "epoch": 109.6829268292683, + "grad_norm": 1.256101131439209, + "learning_rate": 0.0001, + "loss": 60.2544, + "step": 6800 + }, + { + "epoch": 109.84552845528455, + "grad_norm": 1.312761664390564, + "learning_rate": 0.0001, + "loss": 59.1564, + "step": 6810 + }, + { + "epoch": 110.0, + "grad_norm": 0.9668724536895752, + "learning_rate": 0.0001, + "loss": 58.067, + "step": 6820 + }, + { + "epoch": 110.16260162601625, + "grad_norm": 1.1048885583877563, + "learning_rate": 0.0001, + "loss": 59.1506, + "step": 6830 + }, + { + "epoch": 110.32520325203252, + "grad_norm": 1.498469352722168, + "learning_rate": 0.0001, + "loss": 59.1366, + "step": 6840 + }, + { + "epoch": 110.48780487804878, + "grad_norm": 1.1747376918792725, + "learning_rate": 0.0001, + "loss": 59.8449, + "step": 6850 + }, + { + "epoch": 110.65040650406505, + "grad_norm": 1.1041772365570068, + "learning_rate": 0.0001, + "loss": 61.0879, + "step": 6860 + }, + { + "epoch": 110.8130081300813, + "grad_norm": 0.9452431797981262, + "learning_rate": 0.0001, + "loss": 60.8911, + "step": 6870 + }, + { + "epoch": 110.97560975609755, + "grad_norm": 1.0108182430267334, + "learning_rate": 0.0001, + "loss": 58.7924, + "step": 6880 + }, + { + "epoch": 111.130081300813, + "grad_norm": 0.9914487600326538, + "learning_rate": 0.0001, + "loss": 57.4749, + "step": 6890 + }, + { + "epoch": 111.29268292682927, + "grad_norm": 1.0404850244522095, + "learning_rate": 0.0001, + "loss": 59.5329, + "step": 6900 + }, + { + "epoch": 111.45528455284553, + "grad_norm": 1.0363966226577759, + "learning_rate": 0.0001, + "loss": 58.8972, + "step": 6910 + }, + { + "epoch": 111.6178861788618, + "grad_norm": 1.0189651250839233, + "learning_rate": 0.0001, + "loss": 59.3613, + "step": 6920 + }, + { + "epoch": 111.78048780487805, + "grad_norm": 0.9696914553642273, + "learning_rate": 0.0001, + "loss": 59.8001, + "step": 6930 + }, + { + "epoch": 111.9430894308943, + "grad_norm": 1.406821370124817, + "learning_rate": 0.0001, + "loss": 58.6405, + "step": 6940 + }, + { + "epoch": 112.09756097560975, + "grad_norm": 1.0969383716583252, + "learning_rate": 0.0001, + "loss": 57.4458, + "step": 6950 + }, + { + "epoch": 112.26016260162602, + "grad_norm": 1.117593765258789, + "learning_rate": 0.0001, + "loss": 62.599, + "step": 6960 + }, + { + "epoch": 112.42276422764228, + "grad_norm": 0.9327741861343384, + "learning_rate": 0.0001, + "loss": 60.632, + "step": 6970 + }, + { + "epoch": 112.58536585365853, + "grad_norm": 1.1198201179504395, + "learning_rate": 0.0001, + "loss": 59.6119, + "step": 6980 + }, + { + "epoch": 112.7479674796748, + "grad_norm": 0.8709354400634766, + "learning_rate": 0.0001, + "loss": 60.831, + "step": 6990 + }, + { + "epoch": 112.91056910569105, + "grad_norm": 0.9589444994926453, + "learning_rate": 0.0001, + "loss": 58.0002, + "step": 7000 + }, + { + "epoch": 113.0650406504065, + "grad_norm": 1.0971899032592773, + "learning_rate": 0.0001, + "loss": 56.8572, + "step": 7010 + }, + { + "epoch": 113.22764227642277, + "grad_norm": 0.9764513969421387, + "learning_rate": 0.0001, + "loss": 59.3038, + "step": 7020 + }, + { + "epoch": 113.39024390243902, + "grad_norm": 0.9050093293190002, + "learning_rate": 0.0001, + "loss": 59.9146, + "step": 7030 + }, + { + "epoch": 113.55284552845528, + "grad_norm": 0.9506774544715881, + "learning_rate": 0.0001, + "loss": 59.7523, + "step": 7040 + }, + { + "epoch": 113.71544715447155, + "grad_norm": 0.9678346514701843, + "learning_rate": 0.0001, + "loss": 61.3103, + "step": 7050 + }, + { + "epoch": 113.8780487804878, + "grad_norm": 0.9859515428543091, + "learning_rate": 0.0001, + "loss": 56.3866, + "step": 7060 + }, + { + "epoch": 114.03252032520325, + "grad_norm": 1.2696164846420288, + "learning_rate": 0.0001, + "loss": 58.16, + "step": 7070 + }, + { + "epoch": 114.1951219512195, + "grad_norm": 0.9323938488960266, + "learning_rate": 0.0001, + "loss": 58.9483, + "step": 7080 + }, + { + "epoch": 114.35772357723577, + "grad_norm": 1.4123785495758057, + "learning_rate": 0.0001, + "loss": 60.4666, + "step": 7090 + }, + { + "epoch": 114.52032520325203, + "grad_norm": 1.009020447731018, + "learning_rate": 0.0001, + "loss": 58.7152, + "step": 7100 + }, + { + "epoch": 114.6829268292683, + "grad_norm": 1.219383955001831, + "learning_rate": 0.0001, + "loss": 59.1494, + "step": 7110 + }, + { + "epoch": 114.84552845528455, + "grad_norm": 0.9579033255577087, + "learning_rate": 0.0001, + "loss": 58.9879, + "step": 7120 + }, + { + "epoch": 115.0, + "grad_norm": 0.899074912071228, + "learning_rate": 0.0001, + "loss": 55.2031, + "step": 7130 + }, + { + "epoch": 115.16260162601625, + "grad_norm": 0.9924830794334412, + "learning_rate": 0.0001, + "loss": 58.583, + "step": 7140 + }, + { + "epoch": 115.32520325203252, + "grad_norm": 1.0280295610427856, + "learning_rate": 0.0001, + "loss": 60.0948, + "step": 7150 + }, + { + "epoch": 115.48780487804878, + "grad_norm": 0.8108797669410706, + "learning_rate": 0.0001, + "loss": 60.9433, + "step": 7160 + }, + { + "epoch": 115.65040650406505, + "grad_norm": 1.2597944736480713, + "learning_rate": 0.0001, + "loss": 59.1905, + "step": 7170 + }, + { + "epoch": 115.8130081300813, + "grad_norm": 1.2565767765045166, + "learning_rate": 0.0001, + "loss": 59.3545, + "step": 7180 + }, + { + "epoch": 115.97560975609755, + "grad_norm": 1.2012240886688232, + "learning_rate": 0.0001, + "loss": 58.348, + "step": 7190 + }, + { + "epoch": 116.130081300813, + "grad_norm": 1.2883665561676025, + "learning_rate": 0.0001, + "loss": 57.462, + "step": 7200 + }, + { + "epoch": 116.29268292682927, + "grad_norm": 0.904152512550354, + "learning_rate": 0.0001, + "loss": 58.958, + "step": 7210 + }, + { + "epoch": 116.45528455284553, + "grad_norm": 1.1928468942642212, + "learning_rate": 0.0001, + "loss": 58.6106, + "step": 7220 + }, + { + "epoch": 116.6178861788618, + "grad_norm": 1.0655683279037476, + "learning_rate": 0.0001, + "loss": 58.4948, + "step": 7230 + }, + { + "epoch": 116.78048780487805, + "grad_norm": 1.0527434349060059, + "learning_rate": 0.0001, + "loss": 60.1771, + "step": 7240 + }, + { + "epoch": 116.9430894308943, + "grad_norm": 1.1439871788024902, + "learning_rate": 0.0001, + "loss": 58.6737, + "step": 7250 + }, + { + "epoch": 117.09756097560975, + "grad_norm": 1.506571650505066, + "learning_rate": 0.0001, + "loss": 58.0901, + "step": 7260 + }, + { + "epoch": 117.26016260162602, + "grad_norm": 1.1419955492019653, + "learning_rate": 0.0001, + "loss": 59.0705, + "step": 7270 + }, + { + "epoch": 117.42276422764228, + "grad_norm": 1.1211196184158325, + "learning_rate": 0.0001, + "loss": 57.801, + "step": 7280 + }, + { + "epoch": 117.58536585365853, + "grad_norm": 1.1424155235290527, + "learning_rate": 0.0001, + "loss": 59.7449, + "step": 7290 + }, + { + "epoch": 117.7479674796748, + "grad_norm": 1.0593281984329224, + "learning_rate": 0.0001, + "loss": 60.1789, + "step": 7300 + }, + { + "epoch": 117.91056910569105, + "grad_norm": 1.3680177927017212, + "learning_rate": 0.0001, + "loss": 58.4225, + "step": 7310 + }, + { + "epoch": 118.0650406504065, + "grad_norm": 0.8391700983047485, + "learning_rate": 0.0001, + "loss": 56.2102, + "step": 7320 + }, + { + "epoch": 118.22764227642277, + "grad_norm": 1.3166592121124268, + "learning_rate": 0.0001, + "loss": 61.0399, + "step": 7330 + }, + { + "epoch": 118.39024390243902, + "grad_norm": 1.929949402809143, + "learning_rate": 0.0001, + "loss": 59.6514, + "step": 7340 + }, + { + "epoch": 118.55284552845528, + "grad_norm": 0.9464254975318909, + "learning_rate": 0.0001, + "loss": 58.6488, + "step": 7350 + }, + { + "epoch": 118.71544715447155, + "grad_norm": 1.1066193580627441, + "learning_rate": 0.0001, + "loss": 61.5959, + "step": 7360 + }, + { + "epoch": 118.8780487804878, + "grad_norm": 1.216125726699829, + "learning_rate": 0.0001, + "loss": 58.8231, + "step": 7370 + }, + { + "epoch": 119.03252032520325, + "grad_norm": 1.1182583570480347, + "learning_rate": 0.0001, + "loss": 56.9496, + "step": 7380 + }, + { + "epoch": 119.1951219512195, + "grad_norm": 1.105729341506958, + "learning_rate": 0.0001, + "loss": 58.4291, + "step": 7390 + }, + { + "epoch": 119.35772357723577, + "grad_norm": 1.017879843711853, + "learning_rate": 0.0001, + "loss": 59.4963, + "step": 7400 + }, + { + "epoch": 119.52032520325203, + "grad_norm": 0.7882587909698486, + "learning_rate": 0.0001, + "loss": 57.7243, + "step": 7410 + }, + { + "epoch": 119.6829268292683, + "grad_norm": 0.8356695175170898, + "learning_rate": 0.0001, + "loss": 60.4962, + "step": 7420 + }, + { + "epoch": 119.84552845528455, + "grad_norm": 1.1358702182769775, + "learning_rate": 0.0001, + "loss": 60.7173, + "step": 7430 + }, + { + "epoch": 120.0, + "grad_norm": 1.116322636604309, + "learning_rate": 0.0001, + "loss": 53.8387, + "step": 7440 + }, + { + "epoch": 120.16260162601625, + "grad_norm": 0.8522642254829407, + "learning_rate": 0.0001, + "loss": 57.9095, + "step": 7450 + }, + { + "epoch": 120.32520325203252, + "grad_norm": 0.890731692314148, + "learning_rate": 0.0001, + "loss": 57.2296, + "step": 7460 + }, + { + "epoch": 120.48780487804878, + "grad_norm": 1.6366336345672607, + "learning_rate": 0.0001, + "loss": 58.9417, + "step": 7470 + }, + { + "epoch": 120.65040650406505, + "grad_norm": 0.9774733781814575, + "learning_rate": 0.0001, + "loss": 58.8446, + "step": 7480 + }, + { + "epoch": 120.8130081300813, + "grad_norm": 1.6188803911209106, + "learning_rate": 0.0001, + "loss": 59.9794, + "step": 7490 + }, + { + "epoch": 120.97560975609755, + "grad_norm": 1.146599292755127, + "learning_rate": 0.0001, + "loss": 59.8794, + "step": 7500 + }, + { + "epoch": 121.130081300813, + "grad_norm": 0.9472863078117371, + "learning_rate": 0.0001, + "loss": 55.2454, + "step": 7510 + }, + { + "epoch": 121.29268292682927, + "grad_norm": 0.8604205250740051, + "learning_rate": 0.0001, + "loss": 57.6314, + "step": 7520 + }, + { + "epoch": 121.45528455284553, + "grad_norm": 1.0755794048309326, + "learning_rate": 0.0001, + "loss": 60.909, + "step": 7530 + }, + { + "epoch": 121.6178861788618, + "grad_norm": 0.7967071533203125, + "learning_rate": 0.0001, + "loss": 60.1755, + "step": 7540 + }, + { + "epoch": 121.78048780487805, + "grad_norm": 1.037655234336853, + "learning_rate": 0.0001, + "loss": 59.283, + "step": 7550 + }, + { + "epoch": 121.9430894308943, + "grad_norm": 1.070142388343811, + "learning_rate": 0.0001, + "loss": 57.0941, + "step": 7560 + }, + { + "epoch": 122.09756097560975, + "grad_norm": 1.4201022386550903, + "learning_rate": 0.0001, + "loss": 56.2891, + "step": 7570 + }, + { + "epoch": 122.26016260162602, + "grad_norm": 1.0909277200698853, + "learning_rate": 0.0001, + "loss": 57.5213, + "step": 7580 + }, + { + "epoch": 122.42276422764228, + "grad_norm": 0.7573383450508118, + "learning_rate": 0.0001, + "loss": 59.0847, + "step": 7590 + }, + { + "epoch": 122.58536585365853, + "grad_norm": 0.8931725025177002, + "learning_rate": 0.0001, + "loss": 58.9706, + "step": 7600 + }, + { + "epoch": 122.7479674796748, + "grad_norm": 0.9459935426712036, + "learning_rate": 0.0001, + "loss": 58.6267, + "step": 7610 + }, + { + "epoch": 122.91056910569105, + "grad_norm": 0.9463319778442383, + "learning_rate": 0.0001, + "loss": 60.8536, + "step": 7620 + }, + { + "epoch": 123.0650406504065, + "grad_norm": 0.9286382794380188, + "learning_rate": 0.0001, + "loss": 55.2711, + "step": 7630 + }, + { + "epoch": 123.22764227642277, + "grad_norm": 0.9317494630813599, + "learning_rate": 0.0001, + "loss": 60.1578, + "step": 7640 + }, + { + "epoch": 123.39024390243902, + "grad_norm": 0.9834237098693848, + "learning_rate": 0.0001, + "loss": 58.8121, + "step": 7650 + }, + { + "epoch": 123.55284552845528, + "grad_norm": 0.9992889761924744, + "learning_rate": 0.0001, + "loss": 59.8061, + "step": 7660 + }, + { + "epoch": 123.71544715447155, + "grad_norm": 1.0354453325271606, + "learning_rate": 0.0001, + "loss": 59.5853, + "step": 7670 + }, + { + "epoch": 123.8780487804878, + "grad_norm": 1.0609121322631836, + "learning_rate": 0.0001, + "loss": 59.6556, + "step": 7680 + }, + { + "epoch": 124.03252032520325, + "grad_norm": 0.9742912650108337, + "learning_rate": 0.0001, + "loss": 57.2294, + "step": 7690 + }, + { + "epoch": 124.1951219512195, + "grad_norm": 0.9758710265159607, + "learning_rate": 0.0001, + "loss": 59.8581, + "step": 7700 + }, + { + "epoch": 124.35772357723577, + "grad_norm": 0.9492778182029724, + "learning_rate": 0.0001, + "loss": 60.0624, + "step": 7710 + }, + { + "epoch": 124.52032520325203, + "grad_norm": 0.8311835527420044, + "learning_rate": 0.0001, + "loss": 60.0849, + "step": 7720 + }, + { + "epoch": 124.6829268292683, + "grad_norm": 0.8652944564819336, + "learning_rate": 0.0001, + "loss": 58.2321, + "step": 7730 + }, + { + "epoch": 124.84552845528455, + "grad_norm": 1.2980835437774658, + "learning_rate": 0.0001, + "loss": 57.5315, + "step": 7740 + }, + { + "epoch": 125.0, + "grad_norm": 1.0550237894058228, + "learning_rate": 0.0001, + "loss": 56.9615, + "step": 7750 + }, + { + "epoch": 125.16260162601625, + "grad_norm": 1.1248353719711304, + "learning_rate": 0.0001, + "loss": 58.4853, + "step": 7760 + }, + { + "epoch": 125.32520325203252, + "grad_norm": 1.338152289390564, + "learning_rate": 0.0001, + "loss": 57.8604, + "step": 7770 + }, + { + "epoch": 125.48780487804878, + "grad_norm": 0.8162221908569336, + "learning_rate": 0.0001, + "loss": 56.6396, + "step": 7780 + }, + { + "epoch": 125.65040650406505, + "grad_norm": 1.0431848764419556, + "learning_rate": 0.0001, + "loss": 58.9896, + "step": 7790 + }, + { + "epoch": 125.8130081300813, + "grad_norm": 0.848052978515625, + "learning_rate": 0.0001, + "loss": 57.7459, + "step": 7800 + }, + { + "epoch": 125.97560975609755, + "grad_norm": 0.8441061973571777, + "learning_rate": 0.0001, + "loss": 57.9488, + "step": 7810 + }, + { + "epoch": 126.130081300813, + "grad_norm": 0.774566650390625, + "learning_rate": 0.0001, + "loss": 55.9949, + "step": 7820 + }, + { + "epoch": 126.29268292682927, + "grad_norm": 1.178572416305542, + "learning_rate": 0.0001, + "loss": 58.3554, + "step": 7830 + }, + { + "epoch": 126.45528455284553, + "grad_norm": 0.9363889098167419, + "learning_rate": 0.0001, + "loss": 60.6901, + "step": 7840 + }, + { + "epoch": 126.6178861788618, + "grad_norm": 1.0010961294174194, + "learning_rate": 0.0001, + "loss": 57.0078, + "step": 7850 + }, + { + "epoch": 126.78048780487805, + "grad_norm": 0.9573602080345154, + "learning_rate": 0.0001, + "loss": 58.0567, + "step": 7860 + }, + { + "epoch": 126.9430894308943, + "grad_norm": 1.1903672218322754, + "learning_rate": 0.0001, + "loss": 58.3699, + "step": 7870 + }, + { + "epoch": 127.09756097560975, + "grad_norm": 1.2158106565475464, + "learning_rate": 0.0001, + "loss": 57.1154, + "step": 7880 + }, + { + "epoch": 127.26016260162602, + "grad_norm": 1.198789119720459, + "learning_rate": 0.0001, + "loss": 59.1004, + "step": 7890 + }, + { + "epoch": 127.42276422764228, + "grad_norm": 0.8893532752990723, + "learning_rate": 0.0001, + "loss": 59.6481, + "step": 7900 + }, + { + "epoch": 127.58536585365853, + "grad_norm": 0.9834772348403931, + "learning_rate": 0.0001, + "loss": 57.4394, + "step": 7910 + }, + { + "epoch": 127.7479674796748, + "grad_norm": 1.1957768201828003, + "learning_rate": 0.0001, + "loss": 59.1622, + "step": 7920 + }, + { + "epoch": 127.91056910569105, + "grad_norm": 0.822515070438385, + "learning_rate": 0.0001, + "loss": 58.4967, + "step": 7930 + }, + { + "epoch": 128.0650406504065, + "grad_norm": 1.0374864339828491, + "learning_rate": 0.0001, + "loss": 53.9391, + "step": 7940 + }, + { + "epoch": 128.22764227642276, + "grad_norm": 1.1767654418945312, + "learning_rate": 0.0001, + "loss": 59.4305, + "step": 7950 + }, + { + "epoch": 128.390243902439, + "grad_norm": 1.0048503875732422, + "learning_rate": 0.0001, + "loss": 59.7954, + "step": 7960 + }, + { + "epoch": 128.5528455284553, + "grad_norm": 0.9039503931999207, + "learning_rate": 0.0001, + "loss": 58.6945, + "step": 7970 + }, + { + "epoch": 128.71544715447155, + "grad_norm": 1.047418236732483, + "learning_rate": 0.0001, + "loss": 58.6324, + "step": 7980 + }, + { + "epoch": 128.8780487804878, + "grad_norm": 1.7829573154449463, + "learning_rate": 0.0001, + "loss": 57.3699, + "step": 7990 + }, + { + "epoch": 129.03252032520325, + "grad_norm": 0.8874698281288147, + "learning_rate": 0.0001, + "loss": 57.8226, + "step": 8000 + }, + { + "epoch": 129.1951219512195, + "grad_norm": 0.8585357069969177, + "learning_rate": 0.0001, + "loss": 59.1828, + "step": 8010 + }, + { + "epoch": 129.35772357723576, + "grad_norm": 0.7905164957046509, + "learning_rate": 0.0001, + "loss": 58.4257, + "step": 8020 + }, + { + "epoch": 129.52032520325204, + "grad_norm": 1.15513014793396, + "learning_rate": 0.0001, + "loss": 57.712, + "step": 8030 + }, + { + "epoch": 129.6829268292683, + "grad_norm": 1.4509782791137695, + "learning_rate": 0.0001, + "loss": 59.965, + "step": 8040 + }, + { + "epoch": 129.84552845528455, + "grad_norm": 0.8998057246208191, + "learning_rate": 0.0001, + "loss": 58.6614, + "step": 8050 + }, + { + "epoch": 130.0, + "grad_norm": 1.3381586074829102, + "learning_rate": 0.0001, + "loss": 55.2756, + "step": 8060 + }, + { + "epoch": 130.16260162601625, + "grad_norm": 0.9927449226379395, + "learning_rate": 0.0001, + "loss": 59.6809, + "step": 8070 + }, + { + "epoch": 130.3252032520325, + "grad_norm": 0.9900999665260315, + "learning_rate": 0.0001, + "loss": 60.0017, + "step": 8080 + }, + { + "epoch": 130.4878048780488, + "grad_norm": 0.8317362070083618, + "learning_rate": 0.0001, + "loss": 58.2334, + "step": 8090 + }, + { + "epoch": 130.65040650406505, + "grad_norm": 1.0413036346435547, + "learning_rate": 0.0001, + "loss": 58.987, + "step": 8100 + }, + { + "epoch": 130.8130081300813, + "grad_norm": 1.0915170907974243, + "learning_rate": 0.0001, + "loss": 58.9502, + "step": 8110 + }, + { + "epoch": 130.97560975609755, + "grad_norm": 0.9717564582824707, + "learning_rate": 0.0001, + "loss": 58.6578, + "step": 8120 + }, + { + "epoch": 131.130081300813, + "grad_norm": 0.9847933650016785, + "learning_rate": 0.0001, + "loss": 55.0067, + "step": 8130 + }, + { + "epoch": 131.29268292682926, + "grad_norm": 1.2512580156326294, + "learning_rate": 0.0001, + "loss": 56.0223, + "step": 8140 + }, + { + "epoch": 131.45528455284554, + "grad_norm": 0.8859831094741821, + "learning_rate": 0.0001, + "loss": 57.8123, + "step": 8150 + }, + { + "epoch": 131.6178861788618, + "grad_norm": 1.1545639038085938, + "learning_rate": 0.0001, + "loss": 60.2219, + "step": 8160 + }, + { + "epoch": 131.78048780487805, + "grad_norm": 1.3861064910888672, + "learning_rate": 0.0001, + "loss": 60.2156, + "step": 8170 + }, + { + "epoch": 131.9430894308943, + "grad_norm": 1.2621673345565796, + "learning_rate": 0.0001, + "loss": 58.6422, + "step": 8180 + }, + { + "epoch": 132.09756097560975, + "grad_norm": 1.2075496912002563, + "learning_rate": 0.0001, + "loss": 55.4202, + "step": 8190 + }, + { + "epoch": 132.260162601626, + "grad_norm": 1.1960432529449463, + "learning_rate": 0.0001, + "loss": 59.6405, + "step": 8200 + }, + { + "epoch": 132.4227642276423, + "grad_norm": 1.683369755744934, + "learning_rate": 0.0001, + "loss": 60.1361, + "step": 8210 + }, + { + "epoch": 132.58536585365854, + "grad_norm": 0.9033818244934082, + "learning_rate": 0.0001, + "loss": 58.2383, + "step": 8220 + }, + { + "epoch": 132.7479674796748, + "grad_norm": 1.0529452562332153, + "learning_rate": 0.0001, + "loss": 58.9644, + "step": 8230 + }, + { + "epoch": 132.91056910569105, + "grad_norm": 1.0898828506469727, + "learning_rate": 0.0001, + "loss": 57.0898, + "step": 8240 + }, + { + "epoch": 133.0650406504065, + "grad_norm": 1.0203288793563843, + "learning_rate": 0.0001, + "loss": 56.8206, + "step": 8250 + }, + { + "epoch": 133.22764227642276, + "grad_norm": 0.9973819255828857, + "learning_rate": 0.0001, + "loss": 59.2097, + "step": 8260 + }, + { + "epoch": 133.390243902439, + "grad_norm": 0.9728502035140991, + "learning_rate": 0.0001, + "loss": 57.9959, + "step": 8270 + }, + { + "epoch": 133.5528455284553, + "grad_norm": 0.8645833134651184, + "learning_rate": 0.0001, + "loss": 58.9452, + "step": 8280 + }, + { + "epoch": 133.71544715447155, + "grad_norm": 1.0179742574691772, + "learning_rate": 0.0001, + "loss": 57.8577, + "step": 8290 + }, + { + "epoch": 133.8780487804878, + "grad_norm": 0.8542449474334717, + "learning_rate": 0.0001, + "loss": 57.5798, + "step": 8300 + }, + { + "epoch": 134.03252032520325, + "grad_norm": 0.9958596229553223, + "learning_rate": 0.0001, + "loss": 56.6277, + "step": 8310 + }, + { + "epoch": 134.1951219512195, + "grad_norm": 1.1476788520812988, + "learning_rate": 0.0001, + "loss": 60.8397, + "step": 8320 + }, + { + "epoch": 134.35772357723576, + "grad_norm": 0.9098069071769714, + "learning_rate": 0.0001, + "loss": 59.6647, + "step": 8330 + }, + { + "epoch": 134.52032520325204, + "grad_norm": 0.8126556277275085, + "learning_rate": 0.0001, + "loss": 57.147, + "step": 8340 + }, + { + "epoch": 134.6829268292683, + "grad_norm": 1.4846469163894653, + "learning_rate": 0.0001, + "loss": 58.1245, + "step": 8350 + }, + { + "epoch": 134.84552845528455, + "grad_norm": 1.0117210149765015, + "learning_rate": 0.0001, + "loss": 58.7183, + "step": 8360 + }, + { + "epoch": 135.0, + "grad_norm": 0.9517931938171387, + "learning_rate": 0.0001, + "loss": 57.0423, + "step": 8370 + }, + { + "epoch": 135.16260162601625, + "grad_norm": 1.0269144773483276, + "learning_rate": 0.0001, + "loss": 59.6931, + "step": 8380 + }, + { + "epoch": 135.3252032520325, + "grad_norm": 1.1544915437698364, + "learning_rate": 0.0001, + "loss": 57.4079, + "step": 8390 + }, + { + "epoch": 135.4878048780488, + "grad_norm": 1.1180702447891235, + "learning_rate": 0.0001, + "loss": 57.855, + "step": 8400 + }, + { + "epoch": 135.65040650406505, + "grad_norm": 1.2864829301834106, + "learning_rate": 0.0001, + "loss": 57.677, + "step": 8410 + }, + { + "epoch": 135.8130081300813, + "grad_norm": 1.1786772012710571, + "learning_rate": 0.0001, + "loss": 59.5238, + "step": 8420 + }, + { + "epoch": 135.97560975609755, + "grad_norm": 1.8596229553222656, + "learning_rate": 0.0001, + "loss": 57.1596, + "step": 8430 + }, + { + "epoch": 136.130081300813, + "grad_norm": 0.8087555170059204, + "learning_rate": 0.0001, + "loss": 55.8342, + "step": 8440 + }, + { + "epoch": 136.29268292682926, + "grad_norm": 0.8107356429100037, + "learning_rate": 0.0001, + "loss": 58.5965, + "step": 8450 + }, + { + "epoch": 136.45528455284554, + "grad_norm": 1.2416011095046997, + "learning_rate": 0.0001, + "loss": 57.5909, + "step": 8460 + }, + { + "epoch": 136.6178861788618, + "grad_norm": 0.9763500690460205, + "learning_rate": 0.0001, + "loss": 58.1822, + "step": 8470 + }, + { + "epoch": 136.78048780487805, + "grad_norm": 1.1418349742889404, + "learning_rate": 0.0001, + "loss": 59.3942, + "step": 8480 + }, + { + "epoch": 136.9430894308943, + "grad_norm": 1.206878900527954, + "learning_rate": 0.0001, + "loss": 58.4128, + "step": 8490 + }, + { + "epoch": 137.09756097560975, + "grad_norm": 0.861142098903656, + "learning_rate": 0.0001, + "loss": 55.8831, + "step": 8500 + }, + { + "epoch": 137.260162601626, + "grad_norm": 1.0507313013076782, + "learning_rate": 0.0001, + "loss": 59.755, + "step": 8510 + }, + { + "epoch": 137.4227642276423, + "grad_norm": 0.7070153951644897, + "learning_rate": 0.0001, + "loss": 55.6448, + "step": 8520 + }, + { + "epoch": 137.58536585365854, + "grad_norm": 0.9338926672935486, + "learning_rate": 0.0001, + "loss": 58.4242, + "step": 8530 + }, + { + "epoch": 137.7479674796748, + "grad_norm": 1.0600488185882568, + "learning_rate": 0.0001, + "loss": 59.4947, + "step": 8540 + }, + { + "epoch": 137.91056910569105, + "grad_norm": 1.069604754447937, + "learning_rate": 0.0001, + "loss": 57.5164, + "step": 8550 + }, + { + "epoch": 138.0650406504065, + "grad_norm": 0.872332751750946, + "learning_rate": 0.0001, + "loss": 56.453, + "step": 8560 + }, + { + "epoch": 138.22764227642276, + "grad_norm": 0.9670246243476868, + "learning_rate": 0.0001, + "loss": 59.6467, + "step": 8570 + }, + { + "epoch": 138.390243902439, + "grad_norm": 1.1045281887054443, + "learning_rate": 0.0001, + "loss": 59.3334, + "step": 8580 + }, + { + "epoch": 138.5528455284553, + "grad_norm": 1.0553908348083496, + "learning_rate": 0.0001, + "loss": 59.9081, + "step": 8590 + }, + { + "epoch": 138.71544715447155, + "grad_norm": 0.7342197895050049, + "learning_rate": 0.0001, + "loss": 60.2319, + "step": 8600 + }, + { + "epoch": 138.8780487804878, + "grad_norm": 0.9574354290962219, + "learning_rate": 0.0001, + "loss": 56.7984, + "step": 8610 + }, + { + "epoch": 139.03252032520325, + "grad_norm": 0.9238156676292419, + "learning_rate": 0.0001, + "loss": 56.0007, + "step": 8620 + }, + { + "epoch": 139.1951219512195, + "grad_norm": 1.0704458951950073, + "learning_rate": 0.0001, + "loss": 59.4573, + "step": 8630 + }, + { + "epoch": 139.35772357723576, + "grad_norm": 0.8392401337623596, + "learning_rate": 0.0001, + "loss": 57.7589, + "step": 8640 + }, + { + "epoch": 139.52032520325204, + "grad_norm": 0.9042978286743164, + "learning_rate": 0.0001, + "loss": 59.0548, + "step": 8650 + }, + { + "epoch": 139.6829268292683, + "grad_norm": 2.024853467941284, + "learning_rate": 0.0001, + "loss": 58.0019, + "step": 8660 + }, + { + "epoch": 139.84552845528455, + "grad_norm": 1.9823323488235474, + "learning_rate": 0.0001, + "loss": 57.4398, + "step": 8670 + }, + { + "epoch": 140.0, + "grad_norm": 1.626451849937439, + "learning_rate": 0.0001, + "loss": 54.969, + "step": 8680 + }, + { + "epoch": 140.16260162601625, + "grad_norm": 2.435973644256592, + "learning_rate": 0.0001, + "loss": 57.8945, + "step": 8690 + }, + { + "epoch": 140.3252032520325, + "grad_norm": 1.0417332649230957, + "learning_rate": 0.0001, + "loss": 59.9172, + "step": 8700 + }, + { + "epoch": 140.4878048780488, + "grad_norm": 1.0246574878692627, + "learning_rate": 0.0001, + "loss": 57.7337, + "step": 8710 + }, + { + "epoch": 140.65040650406505, + "grad_norm": 0.9492414593696594, + "learning_rate": 0.0001, + "loss": 57.6049, + "step": 8720 + }, + { + "epoch": 140.8130081300813, + "grad_norm": 1.054689645767212, + "learning_rate": 0.0001, + "loss": 56.4394, + "step": 8730 + }, + { + "epoch": 140.97560975609755, + "grad_norm": 0.7142108082771301, + "learning_rate": 0.0001, + "loss": 58.0006, + "step": 8740 + }, + { + "epoch": 141.130081300813, + "grad_norm": 1.1387385129928589, + "learning_rate": 0.0001, + "loss": 55.2639, + "step": 8750 + }, + { + "epoch": 141.29268292682926, + "grad_norm": 2.105823040008545, + "learning_rate": 0.0001, + "loss": 59.3938, + "step": 8760 + }, + { + "epoch": 141.45528455284554, + "grad_norm": 0.9188179969787598, + "learning_rate": 0.0001, + "loss": 58.731, + "step": 8770 + }, + { + "epoch": 141.6178861788618, + "grad_norm": 1.043778896331787, + "learning_rate": 0.0001, + "loss": 56.2486, + "step": 8780 + }, + { + "epoch": 141.78048780487805, + "grad_norm": 1.438591718673706, + "learning_rate": 0.0001, + "loss": 58.4228, + "step": 8790 + }, + { + "epoch": 141.9430894308943, + "grad_norm": 1.116957187652588, + "learning_rate": 0.0001, + "loss": 59.2317, + "step": 8800 + }, + { + "epoch": 142.09756097560975, + "grad_norm": 0.8644933104515076, + "learning_rate": 0.0001, + "loss": 55.3905, + "step": 8810 + }, + { + "epoch": 142.260162601626, + "grad_norm": 1.0249485969543457, + "learning_rate": 0.0001, + "loss": 59.0372, + "step": 8820 + }, + { + "epoch": 142.4227642276423, + "grad_norm": 0.9939082264900208, + "learning_rate": 0.0001, + "loss": 57.3947, + "step": 8830 + }, + { + "epoch": 142.58536585365854, + "grad_norm": 0.9836716055870056, + "learning_rate": 0.0001, + "loss": 57.9229, + "step": 8840 + }, + { + "epoch": 142.7479674796748, + "grad_norm": 0.9596876502037048, + "learning_rate": 0.0001, + "loss": 57.0285, + "step": 8850 + }, + { + "epoch": 142.91056910569105, + "grad_norm": 1.052899956703186, + "learning_rate": 0.0001, + "loss": 60.1384, + "step": 8860 + }, + { + "epoch": 143.0650406504065, + "grad_norm": 1.1220297813415527, + "learning_rate": 0.0001, + "loss": 55.3928, + "step": 8870 + }, + { + "epoch": 143.22764227642276, + "grad_norm": 0.8395596742630005, + "learning_rate": 0.0001, + "loss": 57.8185, + "step": 8880 + }, + { + "epoch": 143.390243902439, + "grad_norm": 1.1940062046051025, + "learning_rate": 0.0001, + "loss": 58.0322, + "step": 8890 + }, + { + "epoch": 143.5528455284553, + "grad_norm": 1.1662484407424927, + "learning_rate": 0.0001, + "loss": 58.2746, + "step": 8900 + }, + { + "epoch": 143.71544715447155, + "grad_norm": 0.9028235077857971, + "learning_rate": 0.0001, + "loss": 56.5817, + "step": 8910 + }, + { + "epoch": 143.8780487804878, + "grad_norm": 1.0597296953201294, + "learning_rate": 0.0001, + "loss": 56.8426, + "step": 8920 + }, + { + "epoch": 144.03252032520325, + "grad_norm": 0.9968701004981995, + "learning_rate": 0.0001, + "loss": 54.9727, + "step": 8930 + }, + { + "epoch": 144.1951219512195, + "grad_norm": 0.9306491613388062, + "learning_rate": 0.0001, + "loss": 56.5321, + "step": 8940 + }, + { + "epoch": 144.35772357723576, + "grad_norm": 1.1079609394073486, + "learning_rate": 0.0001, + "loss": 56.9714, + "step": 8950 + }, + { + "epoch": 144.52032520325204, + "grad_norm": 0.9644935131072998, + "learning_rate": 0.0001, + "loss": 58.613, + "step": 8960 + }, + { + "epoch": 144.6829268292683, + "grad_norm": 1.2037341594696045, + "learning_rate": 0.0001, + "loss": 56.7768, + "step": 8970 + }, + { + "epoch": 144.84552845528455, + "grad_norm": 0.7347294092178345, + "learning_rate": 0.0001, + "loss": 58.5653, + "step": 8980 + }, + { + "epoch": 145.0, + "grad_norm": 0.8877835273742676, + "learning_rate": 0.0001, + "loss": 53.7768, + "step": 8990 + }, + { + "epoch": 145.16260162601625, + "grad_norm": 0.9649038910865784, + "learning_rate": 0.0001, + "loss": 57.7391, + "step": 9000 + }, + { + "epoch": 145.3252032520325, + "grad_norm": 1.1804879903793335, + "learning_rate": 0.0001, + "loss": 60.3727, + "step": 9010 + }, + { + "epoch": 145.4878048780488, + "grad_norm": 0.8406301140785217, + "learning_rate": 0.0001, + "loss": 57.9417, + "step": 9020 + }, + { + "epoch": 145.65040650406505, + "grad_norm": 0.9555912613868713, + "learning_rate": 0.0001, + "loss": 58.0936, + "step": 9030 + }, + { + "epoch": 145.8130081300813, + "grad_norm": 1.20796537399292, + "learning_rate": 0.0001, + "loss": 57.0784, + "step": 9040 + }, + { + "epoch": 145.97560975609755, + "grad_norm": 1.0527915954589844, + "learning_rate": 0.0001, + "loss": 56.752, + "step": 9050 + }, + { + "epoch": 146.130081300813, + "grad_norm": 1.0104299783706665, + "learning_rate": 0.0001, + "loss": 56.2158, + "step": 9060 + }, + { + "epoch": 146.29268292682926, + "grad_norm": 1.4884357452392578, + "learning_rate": 0.0001, + "loss": 57.8631, + "step": 9070 + }, + { + "epoch": 146.45528455284554, + "grad_norm": 0.9095257520675659, + "learning_rate": 0.0001, + "loss": 57.6492, + "step": 9080 + }, + { + "epoch": 146.6178861788618, + "grad_norm": 0.8465835452079773, + "learning_rate": 0.0001, + "loss": 57.78, + "step": 9090 + }, + { + "epoch": 146.78048780487805, + "grad_norm": 1.0254603624343872, + "learning_rate": 0.0001, + "loss": 57.7678, + "step": 9100 + }, + { + "epoch": 146.9430894308943, + "grad_norm": 1.0137311220169067, + "learning_rate": 0.0001, + "loss": 58.2813, + "step": 9110 + }, + { + "epoch": 147.09756097560975, + "grad_norm": 0.9816886186599731, + "learning_rate": 0.0001, + "loss": 54.4789, + "step": 9120 + }, + { + "epoch": 147.260162601626, + "grad_norm": 0.9038514494895935, + "learning_rate": 0.0001, + "loss": 58.5366, + "step": 9130 + }, + { + "epoch": 147.4227642276423, + "grad_norm": 0.7649261355400085, + "learning_rate": 0.0001, + "loss": 57.2898, + "step": 9140 + }, + { + "epoch": 147.58536585365854, + "grad_norm": 0.8841894865036011, + "learning_rate": 0.0001, + "loss": 57.9109, + "step": 9150 + }, + { + "epoch": 147.7479674796748, + "grad_norm": 1.0480542182922363, + "learning_rate": 0.0001, + "loss": 56.9625, + "step": 9160 + }, + { + "epoch": 147.91056910569105, + "grad_norm": 0.8710455894470215, + "learning_rate": 0.0001, + "loss": 57.1721, + "step": 9170 + }, + { + "epoch": 148.0650406504065, + "grad_norm": 1.374118447303772, + "learning_rate": 0.0001, + "loss": 55.3243, + "step": 9180 + }, + { + "epoch": 148.22764227642276, + "grad_norm": 1.0879178047180176, + "learning_rate": 0.0001, + "loss": 57.1631, + "step": 9190 + }, + { + "epoch": 148.390243902439, + "grad_norm": 0.8242596387863159, + "learning_rate": 0.0001, + "loss": 56.9993, + "step": 9200 + }, + { + "epoch": 148.5528455284553, + "grad_norm": 1.02505362033844, + "learning_rate": 0.0001, + "loss": 57.9201, + "step": 9210 + }, + { + "epoch": 148.71544715447155, + "grad_norm": 0.7904821038246155, + "learning_rate": 0.0001, + "loss": 59.2553, + "step": 9220 + }, + { + "epoch": 148.8780487804878, + "grad_norm": 1.4904934167861938, + "learning_rate": 0.0001, + "loss": 59.498, + "step": 9230 + }, + { + "epoch": 149.03252032520325, + "grad_norm": 1.4203144311904907, + "learning_rate": 0.0001, + "loss": 54.6357, + "step": 9240 + }, + { + "epoch": 149.1951219512195, + "grad_norm": 0.8256629705429077, + "learning_rate": 0.0001, + "loss": 57.5996, + "step": 9250 + }, + { + "epoch": 149.35772357723576, + "grad_norm": 0.8736788630485535, + "learning_rate": 0.0001, + "loss": 56.8112, + "step": 9260 + }, + { + "epoch": 149.52032520325204, + "grad_norm": 3.20312762260437, + "learning_rate": 0.0001, + "loss": 55.8763, + "step": 9270 + }, + { + "epoch": 149.6829268292683, + "grad_norm": 0.7431173920631409, + "learning_rate": 0.0001, + "loss": 56.7385, + "step": 9280 + }, + { + "epoch": 149.84552845528455, + "grad_norm": 0.8165793418884277, + "learning_rate": 0.0001, + "loss": 57.8441, + "step": 9290 + }, + { + "epoch": 150.0, + "grad_norm": 1.2387259006500244, + "learning_rate": 0.0001, + "loss": 53.47, + "step": 9300 + }, + { + "epoch": 150.16260162601625, + "grad_norm": 0.8952508568763733, + "learning_rate": 0.0001, + "loss": 57.7642, + "step": 9310 + }, + { + "epoch": 150.3252032520325, + "grad_norm": 0.9837163686752319, + "learning_rate": 0.0001, + "loss": 57.2288, + "step": 9320 + }, + { + "epoch": 150.4878048780488, + "grad_norm": 0.9019481539726257, + "learning_rate": 0.0001, + "loss": 57.4239, + "step": 9330 + }, + { + "epoch": 150.65040650406505, + "grad_norm": 1.0404818058013916, + "learning_rate": 0.0001, + "loss": 57.7678, + "step": 9340 + }, + { + "epoch": 150.8130081300813, + "grad_norm": 1.0074576139450073, + "learning_rate": 0.0001, + "loss": 58.1508, + "step": 9350 + }, + { + "epoch": 150.97560975609755, + "grad_norm": 0.9951609969139099, + "learning_rate": 0.0001, + "loss": 56.1586, + "step": 9360 + }, + { + "epoch": 151.130081300813, + "grad_norm": 1.0799294710159302, + "learning_rate": 0.0001, + "loss": 52.2597, + "step": 9370 + }, + { + "epoch": 151.29268292682926, + "grad_norm": 0.9001291394233704, + "learning_rate": 0.0001, + "loss": 58.1814, + "step": 9380 + }, + { + "epoch": 151.45528455284554, + "grad_norm": 1.0388239622116089, + "learning_rate": 0.0001, + "loss": 57.1812, + "step": 9390 + }, + { + "epoch": 151.6178861788618, + "grad_norm": 0.7662557363510132, + "learning_rate": 0.0001, + "loss": 56.087, + "step": 9400 + }, + { + "epoch": 151.78048780487805, + "grad_norm": 1.0790860652923584, + "learning_rate": 0.0001, + "loss": 55.9247, + "step": 9410 + }, + { + "epoch": 151.9430894308943, + "grad_norm": 0.8173624277114868, + "learning_rate": 0.0001, + "loss": 57.5506, + "step": 9420 + }, + { + "epoch": 152.09756097560975, + "grad_norm": 1.0954891443252563, + "learning_rate": 0.0001, + "loss": 54.6799, + "step": 9430 + }, + { + "epoch": 152.260162601626, + "grad_norm": 1.0103733539581299, + "learning_rate": 0.0001, + "loss": 58.5454, + "step": 9440 + }, + { + "epoch": 152.4227642276423, + "grad_norm": 0.6966420412063599, + "learning_rate": 0.0001, + "loss": 57.561, + "step": 9450 + }, + { + "epoch": 152.58536585365854, + "grad_norm": 0.9085037708282471, + "learning_rate": 0.0001, + "loss": 57.1746, + "step": 9460 + }, + { + "epoch": 152.7479674796748, + "grad_norm": 1.0009453296661377, + "learning_rate": 0.0001, + "loss": 56.8, + "step": 9470 + }, + { + "epoch": 152.91056910569105, + "grad_norm": 0.9139633178710938, + "learning_rate": 0.0001, + "loss": 56.8229, + "step": 9480 + }, + { + "epoch": 153.0650406504065, + "grad_norm": 1.4209275245666504, + "learning_rate": 0.0001, + "loss": 55.3074, + "step": 9490 + }, + { + "epoch": 153.22764227642276, + "grad_norm": 1.3912067413330078, + "learning_rate": 0.0001, + "loss": 58.1401, + "step": 9500 + }, + { + "epoch": 153.390243902439, + "grad_norm": 0.8208648562431335, + "learning_rate": 0.0001, + "loss": 55.2981, + "step": 9510 + }, + { + "epoch": 153.5528455284553, + "grad_norm": 1.1563907861709595, + "learning_rate": 0.0001, + "loss": 59.0262, + "step": 9520 + }, + { + "epoch": 153.71544715447155, + "grad_norm": 1.1142042875289917, + "learning_rate": 0.0001, + "loss": 55.5979, + "step": 9530 + }, + { + "epoch": 153.8780487804878, + "grad_norm": 0.8886147141456604, + "learning_rate": 0.0001, + "loss": 58.5592, + "step": 9540 + }, + { + "epoch": 154.03252032520325, + "grad_norm": 0.9479674696922302, + "learning_rate": 0.0001, + "loss": 53.956, + "step": 9550 + }, + { + "epoch": 154.1951219512195, + "grad_norm": 1.0572675466537476, + "learning_rate": 0.0001, + "loss": 59.5914, + "step": 9560 + }, + { + "epoch": 154.35772357723576, + "grad_norm": 0.8284531831741333, + "learning_rate": 0.0001, + "loss": 56.4313, + "step": 9570 + }, + { + "epoch": 154.52032520325204, + "grad_norm": 0.8882488012313843, + "learning_rate": 0.0001, + "loss": 57.707, + "step": 9580 + }, + { + "epoch": 154.6829268292683, + "grad_norm": 0.9115352630615234, + "learning_rate": 0.0001, + "loss": 58.0544, + "step": 9590 + }, + { + "epoch": 154.84552845528455, + "grad_norm": 1.0006799697875977, + "learning_rate": 0.0001, + "loss": 59.4528, + "step": 9600 + }, + { + "epoch": 155.0, + "grad_norm": 0.9925333261489868, + "learning_rate": 0.0001, + "loss": 52.977, + "step": 9610 + }, + { + "epoch": 155.16260162601625, + "grad_norm": 1.423391580581665, + "learning_rate": 0.0001, + "loss": 57.0083, + "step": 9620 + }, + { + "epoch": 155.3252032520325, + "grad_norm": 1.0822738409042358, + "learning_rate": 0.0001, + "loss": 58.187, + "step": 9630 + }, + { + "epoch": 155.4878048780488, + "grad_norm": 0.932689905166626, + "learning_rate": 0.0001, + "loss": 55.6032, + "step": 9640 + }, + { + "epoch": 155.65040650406505, + "grad_norm": 0.895504355430603, + "learning_rate": 0.0001, + "loss": 57.1324, + "step": 9650 + }, + { + "epoch": 155.8130081300813, + "grad_norm": 1.0883748531341553, + "learning_rate": 0.0001, + "loss": 58.8422, + "step": 9660 + }, + { + "epoch": 155.97560975609755, + "grad_norm": 1.0161819458007812, + "learning_rate": 0.0001, + "loss": 56.8333, + "step": 9670 + }, + { + "epoch": 156.130081300813, + "grad_norm": 0.9483168721199036, + "learning_rate": 0.0001, + "loss": 52.8434, + "step": 9680 + }, + { + "epoch": 156.29268292682926, + "grad_norm": 1.0677295923233032, + "learning_rate": 0.0001, + "loss": 57.749, + "step": 9690 + }, + { + "epoch": 156.45528455284554, + "grad_norm": 0.8123661875724792, + "learning_rate": 0.0001, + "loss": 56.0896, + "step": 9700 + }, + { + "epoch": 156.6178861788618, + "grad_norm": 0.7926080822944641, + "learning_rate": 0.0001, + "loss": 57.8899, + "step": 9710 + }, + { + "epoch": 156.78048780487805, + "grad_norm": 1.0433725118637085, + "learning_rate": 0.0001, + "loss": 59.6728, + "step": 9720 + }, + { + "epoch": 156.9430894308943, + "grad_norm": 0.8284909129142761, + "learning_rate": 0.0001, + "loss": 55.6093, + "step": 9730 + }, + { + "epoch": 157.09756097560975, + "grad_norm": 0.9702232480049133, + "learning_rate": 0.0001, + "loss": 55.3595, + "step": 9740 + }, + { + "epoch": 157.260162601626, + "grad_norm": 0.8909348249435425, + "learning_rate": 0.0001, + "loss": 57.6344, + "step": 9750 + }, + { + "epoch": 157.4227642276423, + "grad_norm": 0.9539676308631897, + "learning_rate": 0.0001, + "loss": 55.8826, + "step": 9760 + }, + { + "epoch": 157.58536585365854, + "grad_norm": 1.1801178455352783, + "learning_rate": 0.0001, + "loss": 55.7141, + "step": 9770 + }, + { + "epoch": 157.7479674796748, + "grad_norm": 0.8674126863479614, + "learning_rate": 0.0001, + "loss": 56.0571, + "step": 9780 + }, + { + "epoch": 157.91056910569105, + "grad_norm": 1.0690264701843262, + "learning_rate": 0.0001, + "loss": 56.1485, + "step": 9790 + }, + { + "epoch": 158.0650406504065, + "grad_norm": 0.8807869553565979, + "learning_rate": 0.0001, + "loss": 56.9874, + "step": 9800 + }, + { + "epoch": 158.22764227642276, + "grad_norm": 0.9477714896202087, + "learning_rate": 0.0001, + "loss": 58.2035, + "step": 9810 + }, + { + "epoch": 158.390243902439, + "grad_norm": 1.4875513315200806, + "learning_rate": 0.0001, + "loss": 59.0302, + "step": 9820 + }, + { + "epoch": 158.5528455284553, + "grad_norm": 0.7979354858398438, + "learning_rate": 0.0001, + "loss": 57.2801, + "step": 9830 + }, + { + "epoch": 158.71544715447155, + "grad_norm": 0.8941402435302734, + "learning_rate": 0.0001, + "loss": 56.3615, + "step": 9840 + }, + { + "epoch": 158.8780487804878, + "grad_norm": 0.8475911021232605, + "learning_rate": 0.0001, + "loss": 57.266, + "step": 9850 + }, + { + "epoch": 159.03252032520325, + "grad_norm": 0.8971277475357056, + "learning_rate": 0.0001, + "loss": 52.792, + "step": 9860 + }, + { + "epoch": 159.1951219512195, + "grad_norm": 0.930777907371521, + "learning_rate": 0.0001, + "loss": 57.4098, + "step": 9870 + }, + { + "epoch": 159.35772357723576, + "grad_norm": 0.9232887625694275, + "learning_rate": 0.0001, + "loss": 56.0267, + "step": 9880 + }, + { + "epoch": 159.52032520325204, + "grad_norm": 0.8904425501823425, + "learning_rate": 0.0001, + "loss": 57.6255, + "step": 9890 + }, + { + "epoch": 159.6829268292683, + "grad_norm": 0.9250755906105042, + "learning_rate": 0.0001, + "loss": 56.5248, + "step": 9900 + }, + { + "epoch": 159.84552845528455, + "grad_norm": 1.3746178150177002, + "learning_rate": 0.0001, + "loss": 56.7843, + "step": 9910 + }, + { + "epoch": 160.0, + "grad_norm": 0.9464873671531677, + "learning_rate": 0.0001, + "loss": 54.1325, + "step": 9920 + }, + { + "epoch": 160.16260162601625, + "grad_norm": 0.9019768238067627, + "learning_rate": 0.0001, + "loss": 58.3641, + "step": 9930 + }, + { + "epoch": 160.3252032520325, + "grad_norm": 0.9282474517822266, + "learning_rate": 0.0001, + "loss": 57.9474, + "step": 9940 + }, + { + "epoch": 160.4878048780488, + "grad_norm": 1.1150450706481934, + "learning_rate": 0.0001, + "loss": 58.2999, + "step": 9950 + }, + { + "epoch": 160.65040650406505, + "grad_norm": 0.9780807495117188, + "learning_rate": 0.0001, + "loss": 55.6409, + "step": 9960 + }, + { + "epoch": 160.8130081300813, + "grad_norm": 0.9708426594734192, + "learning_rate": 0.0001, + "loss": 58.651, + "step": 9970 + }, + { + "epoch": 160.97560975609755, + "grad_norm": 0.9891226887702942, + "learning_rate": 0.0001, + "loss": 57.3798, + "step": 9980 + }, + { + "epoch": 161.130081300813, + "grad_norm": 0.8949639797210693, + "learning_rate": 0.0001, + "loss": 53.4708, + "step": 9990 + }, + { + "epoch": 161.29268292682926, + "grad_norm": 0.831688404083252, + "learning_rate": 0.0001, + "loss": 57.1608, + "step": 10000 + }, + { + "epoch": 161.45528455284554, + "grad_norm": 1.1002815961837769, + "learning_rate": 0.0001, + "loss": 57.5943, + "step": 10010 + }, + { + "epoch": 161.6178861788618, + "grad_norm": 0.823810338973999, + "learning_rate": 0.0001, + "loss": 56.3465, + "step": 10020 + }, + { + "epoch": 161.78048780487805, + "grad_norm": 0.8624166250228882, + "learning_rate": 0.0001, + "loss": 58.2043, + "step": 10030 + }, + { + "epoch": 161.9430894308943, + "grad_norm": 0.9643419981002808, + "learning_rate": 0.0001, + "loss": 58.6286, + "step": 10040 + }, + { + "epoch": 162.09756097560975, + "grad_norm": 0.8477599024772644, + "learning_rate": 0.0001, + "loss": 54.0391, + "step": 10050 + }, + { + "epoch": 162.260162601626, + "grad_norm": 0.932232677936554, + "learning_rate": 0.0001, + "loss": 57.3955, + "step": 10060 + }, + { + "epoch": 162.4227642276423, + "grad_norm": 0.9273334741592407, + "learning_rate": 0.0001, + "loss": 57.3421, + "step": 10070 + }, + { + "epoch": 162.58536585365854, + "grad_norm": 0.746417224407196, + "learning_rate": 0.0001, + "loss": 56.1001, + "step": 10080 + }, + { + "epoch": 162.7479674796748, + "grad_norm": 0.9870802760124207, + "learning_rate": 0.0001, + "loss": 57.1234, + "step": 10090 + }, + { + "epoch": 162.91056910569105, + "grad_norm": 1.2378058433532715, + "learning_rate": 0.0001, + "loss": 58.035, + "step": 10100 + }, + { + "epoch": 163.0650406504065, + "grad_norm": 0.8810584545135498, + "learning_rate": 0.0001, + "loss": 52.1406, + "step": 10110 + }, + { + "epoch": 163.22764227642276, + "grad_norm": 1.1010875701904297, + "learning_rate": 0.0001, + "loss": 57.2534, + "step": 10120 + }, + { + "epoch": 163.390243902439, + "grad_norm": 1.1182267665863037, + "learning_rate": 0.0001, + "loss": 58.0887, + "step": 10130 + }, + { + "epoch": 163.5528455284553, + "grad_norm": 0.8677687048912048, + "learning_rate": 0.0001, + "loss": 56.3502, + "step": 10140 + }, + { + "epoch": 163.71544715447155, + "grad_norm": 1.3890044689178467, + "learning_rate": 0.0001, + "loss": 57.1057, + "step": 10150 + }, + { + "epoch": 163.8780487804878, + "grad_norm": 1.340419054031372, + "learning_rate": 0.0001, + "loss": 56.9196, + "step": 10160 + }, + { + "epoch": 164.03252032520325, + "grad_norm": 0.8187946081161499, + "learning_rate": 0.0001, + "loss": 53.388, + "step": 10170 + }, + { + "epoch": 164.1951219512195, + "grad_norm": 0.7856727838516235, + "learning_rate": 0.0001, + "loss": 57.0134, + "step": 10180 + }, + { + "epoch": 164.35772357723576, + "grad_norm": 0.8677143454551697, + "learning_rate": 0.0001, + "loss": 56.1385, + "step": 10190 + }, + { + "epoch": 164.52032520325204, + "grad_norm": 0.8334061503410339, + "learning_rate": 0.0001, + "loss": 58.4353, + "step": 10200 + }, + { + "epoch": 164.6829268292683, + "grad_norm": 0.9447886943817139, + "learning_rate": 0.0001, + "loss": 56.2007, + "step": 10210 + }, + { + "epoch": 164.84552845528455, + "grad_norm": 0.7787997722625732, + "learning_rate": 0.0001, + "loss": 56.7128, + "step": 10220 + }, + { + "epoch": 165.0, + "grad_norm": 0.864169716835022, + "learning_rate": 0.0001, + "loss": 53.6559, + "step": 10230 + }, + { + "epoch": 165.16260162601625, + "grad_norm": 0.9252000451087952, + "learning_rate": 0.0001, + "loss": 57.1013, + "step": 10240 + }, + { + "epoch": 165.3252032520325, + "grad_norm": 0.9552235007286072, + "learning_rate": 0.0001, + "loss": 58.1834, + "step": 10250 + }, + { + "epoch": 165.4878048780488, + "grad_norm": 1.0003581047058105, + "learning_rate": 0.0001, + "loss": 56.3756, + "step": 10260 + }, + { + "epoch": 165.65040650406505, + "grad_norm": 1.1489440202713013, + "learning_rate": 0.0001, + "loss": 54.8963, + "step": 10270 + }, + { + "epoch": 165.8130081300813, + "grad_norm": 0.9719933867454529, + "learning_rate": 0.0001, + "loss": 59.2853, + "step": 10280 + }, + { + "epoch": 165.97560975609755, + "grad_norm": 0.888495147228241, + "learning_rate": 0.0001, + "loss": 56.7793, + "step": 10290 + }, + { + "epoch": 166.130081300813, + "grad_norm": 0.793580174446106, + "learning_rate": 0.0001, + "loss": 53.6792, + "step": 10300 + }, + { + "epoch": 166.29268292682926, + "grad_norm": 0.928779125213623, + "learning_rate": 0.0001, + "loss": 57.5329, + "step": 10310 + }, + { + "epoch": 166.45528455284554, + "grad_norm": 1.2024686336517334, + "learning_rate": 0.0001, + "loss": 58.08, + "step": 10320 + }, + { + "epoch": 166.6178861788618, + "grad_norm": 1.421026349067688, + "learning_rate": 0.0001, + "loss": 57.5318, + "step": 10330 + }, + { + "epoch": 166.78048780487805, + "grad_norm": 0.9189603328704834, + "learning_rate": 0.0001, + "loss": 57.1355, + "step": 10340 + }, + { + "epoch": 166.9430894308943, + "grad_norm": 1.0614054203033447, + "learning_rate": 0.0001, + "loss": 56.0604, + "step": 10350 + }, + { + "epoch": 167.09756097560975, + "grad_norm": 1.0266008377075195, + "learning_rate": 0.0001, + "loss": 55.0443, + "step": 10360 + }, + { + "epoch": 167.260162601626, + "grad_norm": 0.8291250467300415, + "learning_rate": 0.0001, + "loss": 57.6836, + "step": 10370 + }, + { + "epoch": 167.4227642276423, + "grad_norm": 1.0415455102920532, + "learning_rate": 0.0001, + "loss": 56.1081, + "step": 10380 + }, + { + "epoch": 167.58536585365854, + "grad_norm": 0.8045644164085388, + "learning_rate": 0.0001, + "loss": 54.8943, + "step": 10390 + }, + { + "epoch": 167.7479674796748, + "grad_norm": 0.9025611281394958, + "learning_rate": 0.0001, + "loss": 56.5143, + "step": 10400 + }, + { + "epoch": 167.91056910569105, + "grad_norm": 0.93849116563797, + "learning_rate": 0.0001, + "loss": 57.4787, + "step": 10410 + }, + { + "epoch": 168.0650406504065, + "grad_norm": 0.8948699235916138, + "learning_rate": 0.0001, + "loss": 54.9249, + "step": 10420 + }, + { + "epoch": 168.22764227642276, + "grad_norm": 0.7900148630142212, + "learning_rate": 0.0001, + "loss": 55.9813, + "step": 10430 + }, + { + "epoch": 168.390243902439, + "grad_norm": 1.1853744983673096, + "learning_rate": 0.0001, + "loss": 54.9089, + "step": 10440 + }, + { + "epoch": 168.5528455284553, + "grad_norm": 1.225123643875122, + "learning_rate": 0.0001, + "loss": 58.238, + "step": 10450 + }, + { + "epoch": 168.71544715447155, + "grad_norm": 0.8648725748062134, + "learning_rate": 0.0001, + "loss": 55.4905, + "step": 10460 + }, + { + "epoch": 168.8780487804878, + "grad_norm": 0.9054272770881653, + "learning_rate": 0.0001, + "loss": 58.2697, + "step": 10470 + }, + { + "epoch": 169.03252032520325, + "grad_norm": 0.946158230304718, + "learning_rate": 0.0001, + "loss": 53.3644, + "step": 10480 + }, + { + "epoch": 169.1951219512195, + "grad_norm": 0.8751044273376465, + "learning_rate": 0.0001, + "loss": 54.3469, + "step": 10490 + }, + { + "epoch": 169.35772357723576, + "grad_norm": 0.9814269542694092, + "learning_rate": 0.0001, + "loss": 56.5358, + "step": 10500 + }, + { + "epoch": 169.52032520325204, + "grad_norm": 0.8131198883056641, + "learning_rate": 0.0001, + "loss": 56.2025, + "step": 10510 + }, + { + "epoch": 169.6829268292683, + "grad_norm": 0.7442125678062439, + "learning_rate": 0.0001, + "loss": 54.7679, + "step": 10520 + }, + { + "epoch": 169.84552845528455, + "grad_norm": 0.8504229187965393, + "learning_rate": 0.0001, + "loss": 59.024, + "step": 10530 + }, + { + "epoch": 170.0, + "grad_norm": 0.8483893871307373, + "learning_rate": 0.0001, + "loss": 53.8021, + "step": 10540 + }, + { + "epoch": 170.16260162601625, + "grad_norm": 0.8889706134796143, + "learning_rate": 0.0001, + "loss": 55.1988, + "step": 10550 + }, + { + "epoch": 170.3252032520325, + "grad_norm": 0.9083985686302185, + "learning_rate": 0.0001, + "loss": 57.092, + "step": 10560 + }, + { + "epoch": 170.4878048780488, + "grad_norm": 27.54094123840332, + "learning_rate": 0.0001, + "loss": 56.945, + "step": 10570 + }, + { + "epoch": 170.65040650406505, + "grad_norm": 0.6707919836044312, + "learning_rate": 0.0001, + "loss": 55.8705, + "step": 10580 + }, + { + "epoch": 170.8130081300813, + "grad_norm": 1.0104289054870605, + "learning_rate": 0.0001, + "loss": 56.6519, + "step": 10590 + }, + { + "epoch": 170.97560975609755, + "grad_norm": 1.221200704574585, + "learning_rate": 0.0001, + "loss": 57.0875, + "step": 10600 + }, + { + "epoch": 171.130081300813, + "grad_norm": 1.3986961841583252, + "learning_rate": 0.0001, + "loss": 53.9815, + "step": 10610 + }, + { + "epoch": 171.29268292682926, + "grad_norm": 1.0881519317626953, + "learning_rate": 0.0001, + "loss": 55.8683, + "step": 10620 + }, + { + "epoch": 171.45528455284554, + "grad_norm": 1.172620415687561, + "learning_rate": 0.0001, + "loss": 56.7678, + "step": 10630 + }, + { + "epoch": 171.6178861788618, + "grad_norm": 0.867314875125885, + "learning_rate": 0.0001, + "loss": 55.0647, + "step": 10640 + }, + { + "epoch": 171.78048780487805, + "grad_norm": 1.415421962738037, + "learning_rate": 0.0001, + "loss": 57.5599, + "step": 10650 + }, + { + "epoch": 171.9430894308943, + "grad_norm": 1.0275741815567017, + "learning_rate": 0.0001, + "loss": 55.4741, + "step": 10660 + }, + { + "epoch": 172.09756097560975, + "grad_norm": 0.9037189483642578, + "learning_rate": 0.0001, + "loss": 51.9268, + "step": 10670 + }, + { + "epoch": 172.260162601626, + "grad_norm": 0.9657711386680603, + "learning_rate": 0.0001, + "loss": 57.3183, + "step": 10680 + }, + { + "epoch": 172.4227642276423, + "grad_norm": 0.9031921029090881, + "learning_rate": 0.0001, + "loss": 58.6986, + "step": 10690 + }, + { + "epoch": 172.58536585365854, + "grad_norm": 0.9555232524871826, + "learning_rate": 0.0001, + "loss": 55.7602, + "step": 10700 + }, + { + "epoch": 172.7479674796748, + "grad_norm": 0.985213041305542, + "learning_rate": 0.0001, + "loss": 55.3734, + "step": 10710 + }, + { + "epoch": 172.91056910569105, + "grad_norm": 1.1574981212615967, + "learning_rate": 0.0001, + "loss": 57.8304, + "step": 10720 + }, + { + "epoch": 173.0650406504065, + "grad_norm": 1.0298277139663696, + "learning_rate": 0.0001, + "loss": 54.2185, + "step": 10730 + }, + { + "epoch": 173.22764227642276, + "grad_norm": 1.0708621740341187, + "learning_rate": 0.0001, + "loss": 57.1618, + "step": 10740 + }, + { + "epoch": 173.390243902439, + "grad_norm": 1.0337039232254028, + "learning_rate": 0.0001, + "loss": 56.8093, + "step": 10750 + }, + { + "epoch": 173.5528455284553, + "grad_norm": 1.1115336418151855, + "learning_rate": 0.0001, + "loss": 56.1233, + "step": 10760 + }, + { + "epoch": 173.71544715447155, + "grad_norm": 0.9465245008468628, + "learning_rate": 0.0001, + "loss": 57.79, + "step": 10770 + }, + { + "epoch": 173.8780487804878, + "grad_norm": 1.0282483100891113, + "learning_rate": 0.0001, + "loss": 56.7261, + "step": 10780 + }, + { + "epoch": 174.03252032520325, + "grad_norm": 0.8371811509132385, + "learning_rate": 0.0001, + "loss": 54.3208, + "step": 10790 + }, + { + "epoch": 174.1951219512195, + "grad_norm": 0.75701504945755, + "learning_rate": 0.0001, + "loss": 58.4083, + "step": 10800 + }, + { + "epoch": 174.35772357723576, + "grad_norm": 0.8716368675231934, + "learning_rate": 0.0001, + "loss": 56.3668, + "step": 10810 + }, + { + "epoch": 174.52032520325204, + "grad_norm": 0.8204325437545776, + "learning_rate": 0.0001, + "loss": 57.5088, + "step": 10820 + }, + { + "epoch": 174.6829268292683, + "grad_norm": 0.912240743637085, + "learning_rate": 0.0001, + "loss": 54.392, + "step": 10830 + }, + { + "epoch": 174.84552845528455, + "grad_norm": 1.0001140832901, + "learning_rate": 0.0001, + "loss": 56.2384, + "step": 10840 + }, + { + "epoch": 175.0, + "grad_norm": 1.1657922267913818, + "learning_rate": 0.0001, + "loss": 53.9591, + "step": 10850 + }, + { + "epoch": 175.16260162601625, + "grad_norm": 0.9120311141014099, + "learning_rate": 0.0001, + "loss": 56.7532, + "step": 10860 + }, + { + "epoch": 175.3252032520325, + "grad_norm": 0.821774423122406, + "learning_rate": 0.0001, + "loss": 56.1124, + "step": 10870 + }, + { + "epoch": 175.4878048780488, + "grad_norm": 0.8408815264701843, + "learning_rate": 0.0001, + "loss": 56.8947, + "step": 10880 + }, + { + "epoch": 175.65040650406505, + "grad_norm": 1.2977283000946045, + "learning_rate": 0.0001, + "loss": 55.476, + "step": 10890 + }, + { + "epoch": 175.8130081300813, + "grad_norm": 0.8232496380805969, + "learning_rate": 0.0001, + "loss": 56.0607, + "step": 10900 + }, + { + "epoch": 175.97560975609755, + "grad_norm": 0.8235589265823364, + "learning_rate": 0.0001, + "loss": 56.3365, + "step": 10910 + }, + { + "epoch": 176.130081300813, + "grad_norm": 1.0942591428756714, + "learning_rate": 0.0001, + "loss": 53.065, + "step": 10920 + }, + { + "epoch": 176.29268292682926, + "grad_norm": 0.9026378989219666, + "learning_rate": 0.0001, + "loss": 55.6453, + "step": 10930 + }, + { + "epoch": 176.45528455284554, + "grad_norm": 0.9491869211196899, + "learning_rate": 0.0001, + "loss": 56.4853, + "step": 10940 + }, + { + "epoch": 176.6178861788618, + "grad_norm": 1.072492241859436, + "learning_rate": 0.0001, + "loss": 55.7488, + "step": 10950 + }, + { + "epoch": 176.78048780487805, + "grad_norm": 0.9384275674819946, + "learning_rate": 0.0001, + "loss": 56.8056, + "step": 10960 + }, + { + "epoch": 176.9430894308943, + "grad_norm": 0.9458582997322083, + "learning_rate": 0.0001, + "loss": 57.187, + "step": 10970 + }, + { + "epoch": 177.09756097560975, + "grad_norm": 0.9112198352813721, + "learning_rate": 0.0001, + "loss": 52.9576, + "step": 10980 + }, + { + "epoch": 177.260162601626, + "grad_norm": 1.0807278156280518, + "learning_rate": 0.0001, + "loss": 56.0773, + "step": 10990 + }, + { + "epoch": 177.4227642276423, + "grad_norm": 0.8122154474258423, + "learning_rate": 0.0001, + "loss": 56.0297, + "step": 11000 + }, + { + "epoch": 177.58536585365854, + "grad_norm": 0.7836926579475403, + "learning_rate": 0.0001, + "loss": 56.74, + "step": 11010 + }, + { + "epoch": 177.7479674796748, + "grad_norm": 1.0483779907226562, + "learning_rate": 0.0001, + "loss": 55.1174, + "step": 11020 + }, + { + "epoch": 177.91056910569105, + "grad_norm": 0.8454721570014954, + "learning_rate": 0.0001, + "loss": 55.3519, + "step": 11030 + }, + { + "epoch": 178.0650406504065, + "grad_norm": 1.2100988626480103, + "learning_rate": 0.0001, + "loss": 54.5955, + "step": 11040 + }, + { + "epoch": 178.22764227642276, + "grad_norm": 1.0322265625, + "learning_rate": 0.0001, + "loss": 57.2623, + "step": 11050 + }, + { + "epoch": 178.390243902439, + "grad_norm": 0.8880140781402588, + "learning_rate": 0.0001, + "loss": 56.5935, + "step": 11060 + }, + { + "epoch": 178.5528455284553, + "grad_norm": 0.8864359259605408, + "learning_rate": 0.0001, + "loss": 56.802, + "step": 11070 + }, + { + "epoch": 178.71544715447155, + "grad_norm": 1.1736643314361572, + "learning_rate": 0.0001, + "loss": 56.8208, + "step": 11080 + }, + { + "epoch": 178.8780487804878, + "grad_norm": 0.975029706954956, + "learning_rate": 0.0001, + "loss": 57.5741, + "step": 11090 + }, + { + "epoch": 179.03252032520325, + "grad_norm": 0.852942168712616, + "learning_rate": 0.0001, + "loss": 54.327, + "step": 11100 + }, + { + "epoch": 179.1951219512195, + "grad_norm": 1.1381421089172363, + "learning_rate": 0.0001, + "loss": 56.0875, + "step": 11110 + }, + { + "epoch": 179.35772357723576, + "grad_norm": 1.0923271179199219, + "learning_rate": 0.0001, + "loss": 58.1302, + "step": 11120 + }, + { + "epoch": 179.52032520325204, + "grad_norm": 0.9422796964645386, + "learning_rate": 0.0001, + "loss": 56.2919, + "step": 11130 + }, + { + "epoch": 179.6829268292683, + "grad_norm": 0.9511571526527405, + "learning_rate": 0.0001, + "loss": 55.1329, + "step": 11140 + }, + { + "epoch": 179.84552845528455, + "grad_norm": 0.9221625328063965, + "learning_rate": 0.0001, + "loss": 56.6727, + "step": 11150 + }, + { + "epoch": 180.0, + "grad_norm": 0.8903642296791077, + "learning_rate": 0.0001, + "loss": 53.7472, + "step": 11160 + }, + { + "epoch": 180.16260162601625, + "grad_norm": 0.9752726554870605, + "learning_rate": 0.0001, + "loss": 56.2322, + "step": 11170 + }, + { + "epoch": 180.3252032520325, + "grad_norm": 0.8058391809463501, + "learning_rate": 0.0001, + "loss": 57.7904, + "step": 11180 + }, + { + "epoch": 180.4878048780488, + "grad_norm": 0.9203750491142273, + "learning_rate": 0.0001, + "loss": 55.3608, + "step": 11190 + }, + { + "epoch": 180.65040650406505, + "grad_norm": 1.4366613626480103, + "learning_rate": 0.0001, + "loss": 54.5204, + "step": 11200 + }, + { + "epoch": 180.8130081300813, + "grad_norm": 1.040142297744751, + "learning_rate": 0.0001, + "loss": 56.3904, + "step": 11210 + }, + { + "epoch": 180.97560975609755, + "grad_norm": 1.1629812717437744, + "learning_rate": 0.0001, + "loss": 55.4976, + "step": 11220 + }, + { + "epoch": 181.130081300813, + "grad_norm": 0.847724437713623, + "learning_rate": 0.0001, + "loss": 54.4277, + "step": 11230 + }, + { + "epoch": 181.29268292682926, + "grad_norm": 1.0196763277053833, + "learning_rate": 0.0001, + "loss": 58.1868, + "step": 11240 + }, + { + "epoch": 181.45528455284554, + "grad_norm": 1.2344937324523926, + "learning_rate": 0.0001, + "loss": 56.1104, + "step": 11250 + }, + { + "epoch": 181.6178861788618, + "grad_norm": 0.8804627060890198, + "learning_rate": 0.0001, + "loss": 55.728, + "step": 11260 + }, + { + "epoch": 181.78048780487805, + "grad_norm": 0.8948118686676025, + "learning_rate": 0.0001, + "loss": 56.9878, + "step": 11270 + }, + { + "epoch": 181.9430894308943, + "grad_norm": 0.6648201942443848, + "learning_rate": 0.0001, + "loss": 56.5425, + "step": 11280 + }, + { + "epoch": 182.09756097560975, + "grad_norm": 0.8076626658439636, + "learning_rate": 0.0001, + "loss": 53.4703, + "step": 11290 + }, + { + "epoch": 182.260162601626, + "grad_norm": 0.9063478708267212, + "learning_rate": 0.0001, + "loss": 57.6056, + "step": 11300 + }, + { + "epoch": 182.4227642276423, + "grad_norm": 0.8387530446052551, + "learning_rate": 0.0001, + "loss": 56.0165, + "step": 11310 + }, + { + "epoch": 182.58536585365854, + "grad_norm": 0.8866046667098999, + "learning_rate": 0.0001, + "loss": 57.774, + "step": 11320 + }, + { + "epoch": 182.7479674796748, + "grad_norm": 0.9022682905197144, + "learning_rate": 0.0001, + "loss": 57.5811, + "step": 11330 + }, + { + "epoch": 182.91056910569105, + "grad_norm": 0.8135976195335388, + "learning_rate": 0.0001, + "loss": 56.048, + "step": 11340 + }, + { + "epoch": 183.0650406504065, + "grad_norm": 1.0014373064041138, + "learning_rate": 0.0001, + "loss": 52.5195, + "step": 11350 + }, + { + "epoch": 183.22764227642276, + "grad_norm": 1.0275895595550537, + "learning_rate": 0.0001, + "loss": 57.1421, + "step": 11360 + }, + { + "epoch": 183.390243902439, + "grad_norm": 0.8826107382774353, + "learning_rate": 0.0001, + "loss": 57.5582, + "step": 11370 + }, + { + "epoch": 183.5528455284553, + "grad_norm": 0.9023060202598572, + "learning_rate": 0.0001, + "loss": 56.6695, + "step": 11380 + }, + { + "epoch": 183.71544715447155, + "grad_norm": 0.7782049179077148, + "learning_rate": 0.0001, + "loss": 55.4362, + "step": 11390 + }, + { + "epoch": 183.8780487804878, + "grad_norm": 0.9904818534851074, + "learning_rate": 0.0001, + "loss": 56.7164, + "step": 11400 + }, + { + "epoch": 184.03252032520325, + "grad_norm": 0.9218631982803345, + "learning_rate": 0.0001, + "loss": 53.8536, + "step": 11410 + }, + { + "epoch": 184.1951219512195, + "grad_norm": 0.7807294130325317, + "learning_rate": 0.0001, + "loss": 57.1502, + "step": 11420 + }, + { + "epoch": 184.35772357723576, + "grad_norm": 0.8353742957115173, + "learning_rate": 0.0001, + "loss": 56.8276, + "step": 11430 + }, + { + "epoch": 184.52032520325204, + "grad_norm": 0.7909665107727051, + "learning_rate": 0.0001, + "loss": 56.4164, + "step": 11440 + }, + { + "epoch": 184.6829268292683, + "grad_norm": 1.0313756465911865, + "learning_rate": 0.0001, + "loss": 56.1275, + "step": 11450 + }, + { + "epoch": 184.84552845528455, + "grad_norm": 0.9377022385597229, + "learning_rate": 0.0001, + "loss": 55.9829, + "step": 11460 + }, + { + "epoch": 185.0, + "grad_norm": 1.2855385541915894, + "learning_rate": 0.0001, + "loss": 53.6264, + "step": 11470 + }, + { + "epoch": 185.16260162601625, + "grad_norm": 1.7889217138290405, + "learning_rate": 0.0001, + "loss": 55.1279, + "step": 11480 + }, + { + "epoch": 185.3252032520325, + "grad_norm": 0.8415203094482422, + "learning_rate": 0.0001, + "loss": 57.1529, + "step": 11490 + }, + { + "epoch": 185.4878048780488, + "grad_norm": 1.1245211362838745, + "learning_rate": 0.0001, + "loss": 56.2543, + "step": 11500 + }, + { + "epoch": 185.65040650406505, + "grad_norm": 1.607924222946167, + "learning_rate": 0.0001, + "loss": 55.9007, + "step": 11510 + }, + { + "epoch": 185.8130081300813, + "grad_norm": 0.8068832755088806, + "learning_rate": 0.0001, + "loss": 56.5567, + "step": 11520 + }, + { + "epoch": 185.97560975609755, + "grad_norm": 1.109636902809143, + "learning_rate": 0.0001, + "loss": 55.8676, + "step": 11530 + }, + { + "epoch": 186.130081300813, + "grad_norm": 0.9668437242507935, + "learning_rate": 0.0001, + "loss": 52.9045, + "step": 11540 + }, + { + "epoch": 186.29268292682926, + "grad_norm": 1.1277968883514404, + "learning_rate": 0.0001, + "loss": 55.0794, + "step": 11550 + }, + { + "epoch": 186.45528455284554, + "grad_norm": 0.6413527727127075, + "learning_rate": 0.0001, + "loss": 55.4634, + "step": 11560 + }, + { + "epoch": 186.6178861788618, + "grad_norm": 1.039292812347412, + "learning_rate": 0.0001, + "loss": 56.2108, + "step": 11570 + }, + { + "epoch": 186.78048780487805, + "grad_norm": 0.9112428426742554, + "learning_rate": 0.0001, + "loss": 56.4036, + "step": 11580 + }, + { + "epoch": 186.9430894308943, + "grad_norm": 0.8918914794921875, + "learning_rate": 0.0001, + "loss": 54.9626, + "step": 11590 + }, + { + "epoch": 187.09756097560975, + "grad_norm": 0.9715940952301025, + "learning_rate": 0.0001, + "loss": 54.0208, + "step": 11600 + }, + { + "epoch": 187.260162601626, + "grad_norm": 0.8578653931617737, + "learning_rate": 0.0001, + "loss": 56.4479, + "step": 11610 + }, + { + "epoch": 187.4227642276423, + "grad_norm": 1.7716244459152222, + "learning_rate": 0.0001, + "loss": 55.3543, + "step": 11620 + }, + { + "epoch": 187.58536585365854, + "grad_norm": 0.9152880311012268, + "learning_rate": 0.0001, + "loss": 56.8464, + "step": 11630 + }, + { + "epoch": 187.7479674796748, + "grad_norm": 0.8824559450149536, + "learning_rate": 0.0001, + "loss": 55.9501, + "step": 11640 + }, + { + "epoch": 187.91056910569105, + "grad_norm": 0.780864953994751, + "learning_rate": 0.0001, + "loss": 57.3594, + "step": 11650 + }, + { + "epoch": 188.0650406504065, + "grad_norm": 0.7242524027824402, + "learning_rate": 0.0001, + "loss": 53.9958, + "step": 11660 + }, + { + "epoch": 188.22764227642276, + "grad_norm": 0.9025135636329651, + "learning_rate": 0.0001, + "loss": 54.6309, + "step": 11670 + }, + { + "epoch": 188.390243902439, + "grad_norm": 0.7995520830154419, + "learning_rate": 0.0001, + "loss": 56.7882, + "step": 11680 + }, + { + "epoch": 188.5528455284553, + "grad_norm": 0.8525217175483704, + "learning_rate": 0.0001, + "loss": 54.7559, + "step": 11690 + }, + { + "epoch": 188.71544715447155, + "grad_norm": 0.9954654574394226, + "learning_rate": 0.0001, + "loss": 56.4574, + "step": 11700 + }, + { + "epoch": 188.8780487804878, + "grad_norm": 0.851827085018158, + "learning_rate": 0.0001, + "loss": 55.996, + "step": 11710 + }, + { + "epoch": 189.03252032520325, + "grad_norm": 0.9404662847518921, + "learning_rate": 0.0001, + "loss": 53.8035, + "step": 11720 + }, + { + "epoch": 189.1951219512195, + "grad_norm": 0.8355186581611633, + "learning_rate": 0.0001, + "loss": 54.9002, + "step": 11730 + }, + { + "epoch": 189.35772357723576, + "grad_norm": 1.0156077146530151, + "learning_rate": 0.0001, + "loss": 55.8483, + "step": 11740 + }, + { + "epoch": 189.52032520325204, + "grad_norm": 0.7925308346748352, + "learning_rate": 0.0001, + "loss": 54.1772, + "step": 11750 + }, + { + "epoch": 189.6829268292683, + "grad_norm": 0.8137376308441162, + "learning_rate": 0.0001, + "loss": 53.6278, + "step": 11760 + }, + { + "epoch": 189.84552845528455, + "grad_norm": 0.7871745824813843, + "learning_rate": 0.0001, + "loss": 57.7457, + "step": 11770 + }, + { + "epoch": 190.0, + "grad_norm": 0.7344806790351868, + "learning_rate": 0.0001, + "loss": 55.6082, + "step": 11780 + }, + { + "epoch": 190.16260162601625, + "grad_norm": 0.9094588756561279, + "learning_rate": 0.0001, + "loss": 57.2329, + "step": 11790 + }, + { + "epoch": 190.3252032520325, + "grad_norm": 1.2261676788330078, + "learning_rate": 0.0001, + "loss": 56.2071, + "step": 11800 + }, + { + "epoch": 190.4878048780488, + "grad_norm": 0.9519370794296265, + "learning_rate": 0.0001, + "loss": 56.2488, + "step": 11810 + }, + { + "epoch": 190.65040650406505, + "grad_norm": 0.8949872851371765, + "learning_rate": 0.0001, + "loss": 57.9283, + "step": 11820 + }, + { + "epoch": 190.8130081300813, + "grad_norm": 0.8764474391937256, + "learning_rate": 0.0001, + "loss": 55.2962, + "step": 11830 + }, + { + "epoch": 190.97560975609755, + "grad_norm": 0.9472342729568481, + "learning_rate": 0.0001, + "loss": 56.2262, + "step": 11840 + }, + { + "epoch": 191.130081300813, + "grad_norm": 0.7477512359619141, + "learning_rate": 0.0001, + "loss": 51.2613, + "step": 11850 + }, + { + "epoch": 191.29268292682926, + "grad_norm": 1.0346648693084717, + "learning_rate": 0.0001, + "loss": 56.0406, + "step": 11860 + }, + { + "epoch": 191.45528455284554, + "grad_norm": 1.0473452806472778, + "learning_rate": 0.0001, + "loss": 56.3583, + "step": 11870 + }, + { + "epoch": 191.6178861788618, + "grad_norm": 1.0167906284332275, + "learning_rate": 0.0001, + "loss": 56.3803, + "step": 11880 + }, + { + "epoch": 191.78048780487805, + "grad_norm": 0.8768972754478455, + "learning_rate": 0.0001, + "loss": 57.2783, + "step": 11890 + }, + { + "epoch": 191.9430894308943, + "grad_norm": 0.9370661973953247, + "learning_rate": 0.0001, + "loss": 55.9976, + "step": 11900 + }, + { + "epoch": 192.09756097560975, + "grad_norm": 0.7853850722312927, + "learning_rate": 0.0001, + "loss": 51.9341, + "step": 11910 + }, + { + "epoch": 192.260162601626, + "grad_norm": 1.122916579246521, + "learning_rate": 0.0001, + "loss": 55.832, + "step": 11920 + }, + { + "epoch": 192.4227642276423, + "grad_norm": 0.982596755027771, + "learning_rate": 0.0001, + "loss": 54.2484, + "step": 11930 + }, + { + "epoch": 192.58536585365854, + "grad_norm": 1.0893633365631104, + "learning_rate": 0.0001, + "loss": 54.4806, + "step": 11940 + }, + { + "epoch": 192.7479674796748, + "grad_norm": 0.8936525583267212, + "learning_rate": 0.0001, + "loss": 55.1085, + "step": 11950 + }, + { + "epoch": 192.91056910569105, + "grad_norm": 0.9169464111328125, + "learning_rate": 0.0001, + "loss": 56.8134, + "step": 11960 + }, + { + "epoch": 193.0650406504065, + "grad_norm": 1.2202202081680298, + "learning_rate": 0.0001, + "loss": 53.3595, + "step": 11970 + }, + { + "epoch": 193.22764227642276, + "grad_norm": 0.7786313891410828, + "learning_rate": 0.0001, + "loss": 57.6186, + "step": 11980 + }, + { + "epoch": 193.390243902439, + "grad_norm": 0.9757220149040222, + "learning_rate": 0.0001, + "loss": 54.5559, + "step": 11990 + }, + { + "epoch": 193.5528455284553, + "grad_norm": 0.8599177002906799, + "learning_rate": 0.0001, + "loss": 54.9894, + "step": 12000 + }, + { + "epoch": 193.71544715447155, + "grad_norm": 1.614792823791504, + "learning_rate": 0.0001, + "loss": 56.6223, + "step": 12010 + }, + { + "epoch": 193.8780487804878, + "grad_norm": 0.9011794924736023, + "learning_rate": 0.0001, + "loss": 54.9678, + "step": 12020 + }, + { + "epoch": 194.03252032520325, + "grad_norm": 1.1813637018203735, + "learning_rate": 0.0001, + "loss": 51.7197, + "step": 12030 + }, + { + "epoch": 194.1951219512195, + "grad_norm": 1.420372724533081, + "learning_rate": 0.0001, + "loss": 57.0948, + "step": 12040 + }, + { + "epoch": 194.35772357723576, + "grad_norm": 0.8253830671310425, + "learning_rate": 0.0001, + "loss": 54.8281, + "step": 12050 + }, + { + "epoch": 194.52032520325204, + "grad_norm": 1.055162787437439, + "learning_rate": 0.0001, + "loss": 56.3061, + "step": 12060 + }, + { + "epoch": 194.6829268292683, + "grad_norm": 1.0420011281967163, + "learning_rate": 0.0001, + "loss": 54.7829, + "step": 12070 + }, + { + "epoch": 194.84552845528455, + "grad_norm": 0.7156410813331604, + "learning_rate": 0.0001, + "loss": 55.1366, + "step": 12080 + }, + { + "epoch": 195.0, + "grad_norm": 1.2018579244613647, + "learning_rate": 0.0001, + "loss": 55.2568, + "step": 12090 + }, + { + "epoch": 195.16260162601625, + "grad_norm": 0.9234516620635986, + "learning_rate": 0.0001, + "loss": 55.6524, + "step": 12100 + }, + { + "epoch": 195.3252032520325, + "grad_norm": 0.7889829874038696, + "learning_rate": 0.0001, + "loss": 54.6211, + "step": 12110 + }, + { + "epoch": 195.4878048780488, + "grad_norm": 0.6909085512161255, + "learning_rate": 0.0001, + "loss": 55.161, + "step": 12120 + }, + { + "epoch": 195.65040650406505, + "grad_norm": 1.0910391807556152, + "learning_rate": 0.0001, + "loss": 54.289, + "step": 12130 + }, + { + "epoch": 195.8130081300813, + "grad_norm": 0.8114144206047058, + "learning_rate": 0.0001, + "loss": 57.1045, + "step": 12140 + }, + { + "epoch": 195.97560975609755, + "grad_norm": 0.8750683069229126, + "learning_rate": 0.0001, + "loss": 56.6573, + "step": 12150 + }, + { + "epoch": 196.130081300813, + "grad_norm": 0.9273776412010193, + "learning_rate": 0.0001, + "loss": 52.4588, + "step": 12160 + }, + { + "epoch": 196.29268292682926, + "grad_norm": 0.8890953063964844, + "learning_rate": 0.0001, + "loss": 56.2728, + "step": 12170 + }, + { + "epoch": 196.45528455284554, + "grad_norm": 0.7885172367095947, + "learning_rate": 0.0001, + "loss": 56.9814, + "step": 12180 + }, + { + "epoch": 196.6178861788618, + "grad_norm": 1.0934451818466187, + "learning_rate": 0.0001, + "loss": 54.8799, + "step": 12190 + }, + { + "epoch": 196.78048780487805, + "grad_norm": 0.7452919483184814, + "learning_rate": 0.0001, + "loss": 54.8566, + "step": 12200 + }, + { + "epoch": 196.9430894308943, + "grad_norm": 1.2472970485687256, + "learning_rate": 0.0001, + "loss": 56.9789, + "step": 12210 + }, + { + "epoch": 197.09756097560975, + "grad_norm": 1.0619820356369019, + "learning_rate": 0.0001, + "loss": 52.0264, + "step": 12220 + }, + { + "epoch": 197.260162601626, + "grad_norm": 0.9423636198043823, + "learning_rate": 0.0001, + "loss": 54.6033, + "step": 12230 + }, + { + "epoch": 197.4227642276423, + "grad_norm": 0.6707811951637268, + "learning_rate": 0.0001, + "loss": 54.5683, + "step": 12240 + }, + { + "epoch": 197.58536585365854, + "grad_norm": 1.0682896375656128, + "learning_rate": 0.0001, + "loss": 53.6539, + "step": 12250 + }, + { + "epoch": 197.7479674796748, + "grad_norm": 0.9057245254516602, + "learning_rate": 0.0001, + "loss": 57.1636, + "step": 12260 + }, + { + "epoch": 197.91056910569105, + "grad_norm": 1.0249336957931519, + "learning_rate": 0.0001, + "loss": 54.9638, + "step": 12270 + }, + { + "epoch": 198.0650406504065, + "grad_norm": 0.8086504936218262, + "learning_rate": 0.0001, + "loss": 53.5776, + "step": 12280 + }, + { + "epoch": 198.22764227642276, + "grad_norm": 0.8970116376876831, + "learning_rate": 0.0001, + "loss": 55.9899, + "step": 12290 + }, + { + "epoch": 198.390243902439, + "grad_norm": 0.9402612447738647, + "learning_rate": 0.0001, + "loss": 56.7494, + "step": 12300 + }, + { + "epoch": 198.5528455284553, + "grad_norm": 0.865734338760376, + "learning_rate": 0.0001, + "loss": 56.3205, + "step": 12310 + }, + { + "epoch": 198.71544715447155, + "grad_norm": 0.9791702628135681, + "learning_rate": 0.0001, + "loss": 55.1574, + "step": 12320 + }, + { + "epoch": 198.8780487804878, + "grad_norm": 1.0380010604858398, + "learning_rate": 0.0001, + "loss": 56.5315, + "step": 12330 + }, + { + "epoch": 199.03252032520325, + "grad_norm": 1.14964759349823, + "learning_rate": 0.0001, + "loss": 52.6714, + "step": 12340 + }, + { + "epoch": 199.1951219512195, + "grad_norm": 0.9895046949386597, + "learning_rate": 0.0001, + "loss": 55.0279, + "step": 12350 + }, + { + "epoch": 199.35772357723576, + "grad_norm": 0.8557648062705994, + "learning_rate": 0.0001, + "loss": 54.9455, + "step": 12360 + }, + { + "epoch": 199.52032520325204, + "grad_norm": 0.9442970752716064, + "learning_rate": 0.0001, + "loss": 56.2179, + "step": 12370 + }, + { + "epoch": 199.6829268292683, + "grad_norm": 1.2164298295974731, + "learning_rate": 0.0001, + "loss": 54.7891, + "step": 12380 + }, + { + "epoch": 199.84552845528455, + "grad_norm": 1.1152207851409912, + "learning_rate": 0.0001, + "loss": 53.7524, + "step": 12390 + }, + { + "epoch": 200.0, + "grad_norm": 0.9624357223510742, + "learning_rate": 0.0001, + "loss": 52.4309, + "step": 12400 + }, + { + "epoch": 200.16260162601625, + "grad_norm": 1.1027910709381104, + "learning_rate": 0.0001, + "loss": 55.8449, + "step": 12410 + }, + { + "epoch": 200.3252032520325, + "grad_norm": 0.9644309878349304, + "learning_rate": 0.0001, + "loss": 54.6539, + "step": 12420 + }, + { + "epoch": 200.4878048780488, + "grad_norm": 0.8299253582954407, + "learning_rate": 0.0001, + "loss": 53.8338, + "step": 12430 + }, + { + "epoch": 200.65040650406505, + "grad_norm": 1.0003494024276733, + "learning_rate": 0.0001, + "loss": 56.5585, + "step": 12440 + }, + { + "epoch": 200.8130081300813, + "grad_norm": 0.9802242517471313, + "learning_rate": 0.0001, + "loss": 56.4943, + "step": 12450 + }, + { + "epoch": 200.97560975609755, + "grad_norm": 0.8739794492721558, + "learning_rate": 0.0001, + "loss": 55.1041, + "step": 12460 + }, + { + "epoch": 201.130081300813, + "grad_norm": 1.8339481353759766, + "learning_rate": 0.0001, + "loss": 52.89, + "step": 12470 + }, + { + "epoch": 201.29268292682926, + "grad_norm": 1.3706036806106567, + "learning_rate": 0.0001, + "loss": 55.5897, + "step": 12480 + }, + { + "epoch": 201.45528455284554, + "grad_norm": 1.2508513927459717, + "learning_rate": 0.0001, + "loss": 56.3578, + "step": 12490 + }, + { + "epoch": 201.6178861788618, + "grad_norm": 0.7036287188529968, + "learning_rate": 0.0001, + "loss": 54.2583, + "step": 12500 + }, + { + "epoch": 201.78048780487805, + "grad_norm": 0.9160637259483337, + "learning_rate": 0.0001, + "loss": 56.3668, + "step": 12510 + }, + { + "epoch": 201.9430894308943, + "grad_norm": 0.8451681137084961, + "learning_rate": 0.0001, + "loss": 55.0297, + "step": 12520 + }, + { + "epoch": 202.09756097560975, + "grad_norm": 1.103285789489746, + "learning_rate": 0.0001, + "loss": 52.6796, + "step": 12530 + }, + { + "epoch": 202.260162601626, + "grad_norm": 0.9927880167961121, + "learning_rate": 0.0001, + "loss": 55.9175, + "step": 12540 + }, + { + "epoch": 202.4227642276423, + "grad_norm": 0.6602478623390198, + "learning_rate": 0.0001, + "loss": 56.0773, + "step": 12550 + }, + { + "epoch": 202.58536585365854, + "grad_norm": 0.8949015140533447, + "learning_rate": 0.0001, + "loss": 55.9995, + "step": 12560 + }, + { + "epoch": 202.7479674796748, + "grad_norm": 0.7111349701881409, + "learning_rate": 0.0001, + "loss": 55.4216, + "step": 12570 + }, + { + "epoch": 202.91056910569105, + "grad_norm": 0.8465375900268555, + "learning_rate": 0.0001, + "loss": 57.2378, + "step": 12580 + }, + { + "epoch": 203.0650406504065, + "grad_norm": 0.7122918367385864, + "learning_rate": 0.0001, + "loss": 54.5047, + "step": 12590 + }, + { + "epoch": 203.22764227642276, + "grad_norm": 0.9927072525024414, + "learning_rate": 0.0001, + "loss": 53.9752, + "step": 12600 + }, + { + "epoch": 203.390243902439, + "grad_norm": 0.7084205746650696, + "learning_rate": 0.0001, + "loss": 53.7208, + "step": 12610 + }, + { + "epoch": 203.5528455284553, + "grad_norm": 0.7796747088432312, + "learning_rate": 0.0001, + "loss": 54.5118, + "step": 12620 + }, + { + "epoch": 203.71544715447155, + "grad_norm": 0.7925731539726257, + "learning_rate": 0.0001, + "loss": 55.8361, + "step": 12630 + }, + { + "epoch": 203.8780487804878, + "grad_norm": 0.8884345889091492, + "learning_rate": 0.0001, + "loss": 52.6655, + "step": 12640 + }, + { + "epoch": 204.03252032520325, + "grad_norm": 0.7525721192359924, + "learning_rate": 0.0001, + "loss": 52.0389, + "step": 12650 + }, + { + "epoch": 204.1951219512195, + "grad_norm": 1.071750521659851, + "learning_rate": 0.0001, + "loss": 55.6427, + "step": 12660 + }, + { + "epoch": 204.35772357723576, + "grad_norm": 0.7978437542915344, + "learning_rate": 0.0001, + "loss": 55.1445, + "step": 12670 + }, + { + "epoch": 204.52032520325204, + "grad_norm": 0.6743281483650208, + "learning_rate": 0.0001, + "loss": 56.5749, + "step": 12680 + }, + { + "epoch": 204.6829268292683, + "grad_norm": 0.9658812284469604, + "learning_rate": 0.0001, + "loss": 55.2514, + "step": 12690 + }, + { + "epoch": 204.84552845528455, + "grad_norm": 0.8930355310440063, + "learning_rate": 0.0001, + "loss": 56.5771, + "step": 12700 + }, + { + "epoch": 205.0, + "grad_norm": 2.1135165691375732, + "learning_rate": 0.0001, + "loss": 53.9625, + "step": 12710 + }, + { + "epoch": 205.16260162601625, + "grad_norm": 1.0068354606628418, + "learning_rate": 0.0001, + "loss": 55.2095, + "step": 12720 + }, + { + "epoch": 205.3252032520325, + "grad_norm": 0.8000341653823853, + "learning_rate": 0.0001, + "loss": 56.155, + "step": 12730 + }, + { + "epoch": 205.4878048780488, + "grad_norm": 1.201411485671997, + "learning_rate": 0.0001, + "loss": 56.8318, + "step": 12740 + }, + { + "epoch": 205.65040650406505, + "grad_norm": 1.005187749862671, + "learning_rate": 0.0001, + "loss": 55.8241, + "step": 12750 + }, + { + "epoch": 205.8130081300813, + "grad_norm": 1.0376118421554565, + "learning_rate": 0.0001, + "loss": 55.3489, + "step": 12760 + }, + { + "epoch": 205.97560975609755, + "grad_norm": 0.8955312967300415, + "learning_rate": 0.0001, + "loss": 55.583, + "step": 12770 + }, + { + "epoch": 206.130081300813, + "grad_norm": 0.9010198712348938, + "learning_rate": 0.0001, + "loss": 52.4093, + "step": 12780 + }, + { + "epoch": 206.29268292682926, + "grad_norm": 0.758160412311554, + "learning_rate": 0.0001, + "loss": 56.1256, + "step": 12790 + }, + { + "epoch": 206.45528455284554, + "grad_norm": 0.7211036086082458, + "learning_rate": 0.0001, + "loss": 55.0504, + "step": 12800 + }, + { + "epoch": 206.6178861788618, + "grad_norm": 1.065086841583252, + "learning_rate": 0.0001, + "loss": 55.7009, + "step": 12810 + }, + { + "epoch": 206.78048780487805, + "grad_norm": 0.6722949743270874, + "learning_rate": 0.0001, + "loss": 54.0928, + "step": 12820 + }, + { + "epoch": 206.9430894308943, + "grad_norm": 0.9325671792030334, + "learning_rate": 0.0001, + "loss": 55.6211, + "step": 12830 + }, + { + "epoch": 207.09756097560975, + "grad_norm": 0.9299114942550659, + "learning_rate": 0.0001, + "loss": 51.8944, + "step": 12840 + }, + { + "epoch": 207.260162601626, + "grad_norm": 0.8544942140579224, + "learning_rate": 0.0001, + "loss": 55.1983, + "step": 12850 + }, + { + "epoch": 207.4227642276423, + "grad_norm": 1.0352020263671875, + "learning_rate": 0.0001, + "loss": 53.4288, + "step": 12860 + }, + { + "epoch": 207.58536585365854, + "grad_norm": 1.1637558937072754, + "learning_rate": 0.0001, + "loss": 54.5078, + "step": 12870 + }, + { + "epoch": 207.7479674796748, + "grad_norm": 1.0634640455245972, + "learning_rate": 0.0001, + "loss": 53.8032, + "step": 12880 + }, + { + "epoch": 207.91056910569105, + "grad_norm": 1.0478582382202148, + "learning_rate": 0.0001, + "loss": 54.774, + "step": 12890 + }, + { + "epoch": 208.0650406504065, + "grad_norm": 0.772788405418396, + "learning_rate": 0.0001, + "loss": 52.4807, + "step": 12900 + }, + { + "epoch": 208.22764227642276, + "grad_norm": 1.0519691705703735, + "learning_rate": 0.0001, + "loss": 54.8709, + "step": 12910 + }, + { + "epoch": 208.390243902439, + "grad_norm": 0.9291022419929504, + "learning_rate": 0.0001, + "loss": 55.8279, + "step": 12920 + }, + { + "epoch": 208.5528455284553, + "grad_norm": 0.9870265126228333, + "learning_rate": 0.0001, + "loss": 56.3803, + "step": 12930 + }, + { + "epoch": 208.71544715447155, + "grad_norm": 0.787817656993866, + "learning_rate": 0.0001, + "loss": 55.1475, + "step": 12940 + }, + { + "epoch": 208.8780487804878, + "grad_norm": 1.1020783185958862, + "learning_rate": 0.0001, + "loss": 56.1078, + "step": 12950 + }, + { + "epoch": 209.03252032520325, + "grad_norm": 0.8097794055938721, + "learning_rate": 0.0001, + "loss": 53.1355, + "step": 12960 + }, + { + "epoch": 209.1951219512195, + "grad_norm": 1.1928120851516724, + "learning_rate": 0.0001, + "loss": 53.1578, + "step": 12970 + }, + { + "epoch": 209.35772357723576, + "grad_norm": 1.131969928741455, + "learning_rate": 0.0001, + "loss": 56.2361, + "step": 12980 + }, + { + "epoch": 209.52032520325204, + "grad_norm": 0.7544846534729004, + "learning_rate": 0.0001, + "loss": 55.9766, + "step": 12990 + }, + { + "epoch": 209.6829268292683, + "grad_norm": 1.2758643627166748, + "learning_rate": 0.0001, + "loss": 53.5174, + "step": 13000 + }, + { + "epoch": 209.84552845528455, + "grad_norm": 0.8507160544395447, + "learning_rate": 0.0001, + "loss": 53.9673, + "step": 13010 + }, + { + "epoch": 210.0, + "grad_norm": 1.2480005025863647, + "learning_rate": 0.0001, + "loss": 51.9134, + "step": 13020 + }, + { + "epoch": 210.16260162601625, + "grad_norm": 1.0838913917541504, + "learning_rate": 0.0001, + "loss": 52.4845, + "step": 13030 + }, + { + "epoch": 210.3252032520325, + "grad_norm": 0.8026198744773865, + "learning_rate": 0.0001, + "loss": 54.2063, + "step": 13040 + }, + { + "epoch": 210.4878048780488, + "grad_norm": 0.7196829319000244, + "learning_rate": 0.0001, + "loss": 54.91, + "step": 13050 + }, + { + "epoch": 210.65040650406505, + "grad_norm": 0.8283985257148743, + "learning_rate": 0.0001, + "loss": 56.3195, + "step": 13060 + }, + { + "epoch": 210.8130081300813, + "grad_norm": 0.9254607558250427, + "learning_rate": 0.0001, + "loss": 55.7841, + "step": 13070 + }, + { + "epoch": 210.97560975609755, + "grad_norm": 1.0719525814056396, + "learning_rate": 0.0001, + "loss": 55.4097, + "step": 13080 + }, + { + "epoch": 211.130081300813, + "grad_norm": 0.7990654110908508, + "learning_rate": 0.0001, + "loss": 50.4808, + "step": 13090 + }, + { + "epoch": 211.29268292682926, + "grad_norm": 0.7976981401443481, + "learning_rate": 0.0001, + "loss": 53.8439, + "step": 13100 + }, + { + "epoch": 211.45528455284554, + "grad_norm": 0.8668096661567688, + "learning_rate": 0.0001, + "loss": 55.9312, + "step": 13110 + }, + { + "epoch": 211.6178861788618, + "grad_norm": 0.7176859974861145, + "learning_rate": 0.0001, + "loss": 53.5898, + "step": 13120 + }, + { + "epoch": 211.78048780487805, + "grad_norm": 0.6893277168273926, + "learning_rate": 0.0001, + "loss": 54.099, + "step": 13130 + }, + { + "epoch": 211.9430894308943, + "grad_norm": 0.7272772192955017, + "learning_rate": 0.0001, + "loss": 55.8779, + "step": 13140 + }, + { + "epoch": 212.09756097560975, + "grad_norm": 0.9501060843467712, + "learning_rate": 0.0001, + "loss": 52.049, + "step": 13150 + }, + { + "epoch": 212.260162601626, + "grad_norm": 0.8938133716583252, + "learning_rate": 0.0001, + "loss": 54.9783, + "step": 13160 + }, + { + "epoch": 212.4227642276423, + "grad_norm": 1.3081871271133423, + "learning_rate": 0.0001, + "loss": 54.1436, + "step": 13170 + }, + { + "epoch": 212.58536585365854, + "grad_norm": 1.182673454284668, + "learning_rate": 0.0001, + "loss": 55.5235, + "step": 13180 + }, + { + "epoch": 212.7479674796748, + "grad_norm": 0.7060847878456116, + "learning_rate": 0.0001, + "loss": 54.8775, + "step": 13190 + }, + { + "epoch": 212.91056910569105, + "grad_norm": 0.6047799587249756, + "learning_rate": 0.0001, + "loss": 56.6604, + "step": 13200 + }, + { + "epoch": 213.0650406504065, + "grad_norm": 0.8285370469093323, + "learning_rate": 0.0001, + "loss": 52.9922, + "step": 13210 + }, + { + "epoch": 213.22764227642276, + "grad_norm": 0.8437235951423645, + "learning_rate": 0.0001, + "loss": 54.9914, + "step": 13220 + }, + { + "epoch": 213.390243902439, + "grad_norm": 0.7426273822784424, + "learning_rate": 0.0001, + "loss": 54.7659, + "step": 13230 + }, + { + "epoch": 213.5528455284553, + "grad_norm": 0.8756979703903198, + "learning_rate": 0.0001, + "loss": 53.2097, + "step": 13240 + }, + { + "epoch": 213.71544715447155, + "grad_norm": 0.9625734090805054, + "learning_rate": 0.0001, + "loss": 55.2452, + "step": 13250 + }, + { + "epoch": 213.8780487804878, + "grad_norm": 1.5846962928771973, + "learning_rate": 0.0001, + "loss": 54.1381, + "step": 13260 + }, + { + "epoch": 214.03252032520325, + "grad_norm": 0.6845040321350098, + "learning_rate": 0.0001, + "loss": 50.8792, + "step": 13270 + }, + { + "epoch": 214.1951219512195, + "grad_norm": 1.0069342851638794, + "learning_rate": 0.0001, + "loss": 56.4215, + "step": 13280 + }, + { + "epoch": 214.35772357723576, + "grad_norm": 0.8575270771980286, + "learning_rate": 0.0001, + "loss": 54.7543, + "step": 13290 + }, + { + "epoch": 214.52032520325204, + "grad_norm": 1.1226451396942139, + "learning_rate": 0.0001, + "loss": 53.5045, + "step": 13300 + }, + { + "epoch": 214.6829268292683, + "grad_norm": 0.8359237313270569, + "learning_rate": 0.0001, + "loss": 54.7099, + "step": 13310 + }, + { + "epoch": 214.84552845528455, + "grad_norm": 0.823088526725769, + "learning_rate": 0.0001, + "loss": 55.2504, + "step": 13320 + }, + { + "epoch": 215.0, + "grad_norm": 0.9530013799667358, + "learning_rate": 0.0001, + "loss": 51.2342, + "step": 13330 + }, + { + "epoch": 215.16260162601625, + "grad_norm": 1.024066686630249, + "learning_rate": 0.0001, + "loss": 54.8409, + "step": 13340 + }, + { + "epoch": 215.3252032520325, + "grad_norm": 0.7555123567581177, + "learning_rate": 0.0001, + "loss": 54.6994, + "step": 13350 + }, + { + "epoch": 215.4878048780488, + "grad_norm": 0.8845812678337097, + "learning_rate": 0.0001, + "loss": 55.1865, + "step": 13360 + }, + { + "epoch": 215.65040650406505, + "grad_norm": 0.8159278035163879, + "learning_rate": 0.0001, + "loss": 54.4797, + "step": 13370 + }, + { + "epoch": 215.8130081300813, + "grad_norm": 0.8099329471588135, + "learning_rate": 0.0001, + "loss": 53.4086, + "step": 13380 + }, + { + "epoch": 215.97560975609755, + "grad_norm": 0.7204589247703552, + "learning_rate": 0.0001, + "loss": 54.1444, + "step": 13390 + }, + { + "epoch": 216.130081300813, + "grad_norm": 1.0179848670959473, + "learning_rate": 0.0001, + "loss": 52.6701, + "step": 13400 + }, + { + "epoch": 216.29268292682926, + "grad_norm": 0.7029995322227478, + "learning_rate": 0.0001, + "loss": 55.2125, + "step": 13410 + }, + { + "epoch": 216.45528455284554, + "grad_norm": 0.7679967880249023, + "learning_rate": 0.0001, + "loss": 55.5851, + "step": 13420 + }, + { + "epoch": 216.6178861788618, + "grad_norm": 0.925176739692688, + "learning_rate": 0.0001, + "loss": 54.6698, + "step": 13430 + }, + { + "epoch": 216.78048780487805, + "grad_norm": 1.0775889158248901, + "learning_rate": 0.0001, + "loss": 53.8535, + "step": 13440 + }, + { + "epoch": 216.9430894308943, + "grad_norm": 1.9590994119644165, + "learning_rate": 0.0001, + "loss": 53.3539, + "step": 13450 + }, + { + "epoch": 217.09756097560975, + "grad_norm": 1.1416126489639282, + "learning_rate": 0.0001, + "loss": 53.07, + "step": 13460 + }, + { + "epoch": 217.260162601626, + "grad_norm": 0.6063092947006226, + "learning_rate": 0.0001, + "loss": 55.2222, + "step": 13470 + }, + { + "epoch": 217.4227642276423, + "grad_norm": 1.082871913909912, + "learning_rate": 0.0001, + "loss": 54.9424, + "step": 13480 + }, + { + "epoch": 217.58536585365854, + "grad_norm": 1.0098278522491455, + "learning_rate": 0.0001, + "loss": 55.5704, + "step": 13490 + }, + { + "epoch": 217.7479674796748, + "grad_norm": 0.7821612358093262, + "learning_rate": 0.0001, + "loss": 55.6112, + "step": 13500 + }, + { + "epoch": 217.91056910569105, + "grad_norm": 0.7899888753890991, + "learning_rate": 0.0001, + "loss": 54.2449, + "step": 13510 + }, + { + "epoch": 218.0650406504065, + "grad_norm": 0.8772909045219421, + "learning_rate": 0.0001, + "loss": 51.3149, + "step": 13520 + }, + { + "epoch": 218.22764227642276, + "grad_norm": 0.835057258605957, + "learning_rate": 0.0001, + "loss": 54.9729, + "step": 13530 + }, + { + "epoch": 218.390243902439, + "grad_norm": 0.9645805954933167, + "learning_rate": 0.0001, + "loss": 53.1176, + "step": 13540 + }, + { + "epoch": 218.5528455284553, + "grad_norm": 1.1908318996429443, + "learning_rate": 0.0001, + "loss": 54.3461, + "step": 13550 + }, + { + "epoch": 218.71544715447155, + "grad_norm": 0.8490066528320312, + "learning_rate": 0.0001, + "loss": 56.1804, + "step": 13560 + }, + { + "epoch": 218.8780487804878, + "grad_norm": 0.855278491973877, + "learning_rate": 0.0001, + "loss": 54.6583, + "step": 13570 + }, + { + "epoch": 219.03252032520325, + "grad_norm": 0.8656384944915771, + "learning_rate": 0.0001, + "loss": 51.4255, + "step": 13580 + }, + { + "epoch": 219.1951219512195, + "grad_norm": 0.8291656970977783, + "learning_rate": 0.0001, + "loss": 55.1752, + "step": 13590 + }, + { + "epoch": 219.35772357723576, + "grad_norm": 0.9894065260887146, + "learning_rate": 0.0001, + "loss": 52.9551, + "step": 13600 + }, + { + "epoch": 219.52032520325204, + "grad_norm": 0.7791743278503418, + "learning_rate": 0.0001, + "loss": 53.2005, + "step": 13610 + }, + { + "epoch": 219.6829268292683, + "grad_norm": 0.8251843452453613, + "learning_rate": 0.0001, + "loss": 54.3272, + "step": 13620 + }, + { + "epoch": 219.84552845528455, + "grad_norm": 0.6909898519515991, + "learning_rate": 0.0001, + "loss": 53.4259, + "step": 13630 + }, + { + "epoch": 220.0, + "grad_norm": 0.7680906057357788, + "learning_rate": 0.0001, + "loss": 53.6018, + "step": 13640 + }, + { + "epoch": 220.16260162601625, + "grad_norm": 1.1677294969558716, + "learning_rate": 0.0001, + "loss": 53.7904, + "step": 13650 + }, + { + "epoch": 220.3252032520325, + "grad_norm": 0.8123244047164917, + "learning_rate": 0.0001, + "loss": 55.255, + "step": 13660 + }, + { + "epoch": 220.4878048780488, + "grad_norm": 0.8476759791374207, + "learning_rate": 0.0001, + "loss": 56.5699, + "step": 13670 + }, + { + "epoch": 220.65040650406505, + "grad_norm": 0.9571487903594971, + "learning_rate": 0.0001, + "loss": 54.228, + "step": 13680 + }, + { + "epoch": 220.8130081300813, + "grad_norm": 0.7796895503997803, + "learning_rate": 0.0001, + "loss": 54.8423, + "step": 13690 + }, + { + "epoch": 220.97560975609755, + "grad_norm": 0.9055293798446655, + "learning_rate": 0.0001, + "loss": 53.7139, + "step": 13700 + }, + { + "epoch": 221.130081300813, + "grad_norm": 0.7490646839141846, + "learning_rate": 0.0001, + "loss": 52.4622, + "step": 13710 + }, + { + "epoch": 221.29268292682926, + "grad_norm": 0.7626569271087646, + "learning_rate": 0.0001, + "loss": 55.8454, + "step": 13720 + }, + { + "epoch": 221.45528455284554, + "grad_norm": 0.9692214131355286, + "learning_rate": 0.0001, + "loss": 55.0342, + "step": 13730 + }, + { + "epoch": 221.6178861788618, + "grad_norm": 0.8987938761711121, + "learning_rate": 0.0001, + "loss": 54.8382, + "step": 13740 + }, + { + "epoch": 221.78048780487805, + "grad_norm": 0.7676286101341248, + "learning_rate": 0.0001, + "loss": 55.5017, + "step": 13750 + }, + { + "epoch": 221.9430894308943, + "grad_norm": 0.8445277214050293, + "learning_rate": 0.0001, + "loss": 55.331, + "step": 13760 + }, + { + "epoch": 222.09756097560975, + "grad_norm": 0.7058215737342834, + "learning_rate": 0.0001, + "loss": 52.9581, + "step": 13770 + }, + { + "epoch": 222.260162601626, + "grad_norm": 0.7497864365577698, + "learning_rate": 0.0001, + "loss": 54.9104, + "step": 13780 + }, + { + "epoch": 222.4227642276423, + "grad_norm": 0.9665900468826294, + "learning_rate": 0.0001, + "loss": 55.9701, + "step": 13790 + }, + { + "epoch": 222.58536585365854, + "grad_norm": 0.9786218404769897, + "learning_rate": 0.0001, + "loss": 55.2512, + "step": 13800 + }, + { + "epoch": 222.7479674796748, + "grad_norm": 1.1519272327423096, + "learning_rate": 0.0001, + "loss": 56.0712, + "step": 13810 + }, + { + "epoch": 222.91056910569105, + "grad_norm": 0.8277210593223572, + "learning_rate": 0.0001, + "loss": 56.1046, + "step": 13820 + }, + { + "epoch": 223.0650406504065, + "grad_norm": 0.816068708896637, + "learning_rate": 0.0001, + "loss": 52.8655, + "step": 13830 + }, + { + "epoch": 223.22764227642276, + "grad_norm": 0.6874851584434509, + "learning_rate": 0.0001, + "loss": 54.6701, + "step": 13840 + }, + { + "epoch": 223.390243902439, + "grad_norm": 0.8549659848213196, + "learning_rate": 0.0001, + "loss": 55.4961, + "step": 13850 + }, + { + "epoch": 223.5528455284553, + "grad_norm": 0.9988876581192017, + "learning_rate": 0.0001, + "loss": 54.9155, + "step": 13860 + }, + { + "epoch": 223.71544715447155, + "grad_norm": 0.8321837782859802, + "learning_rate": 0.0001, + "loss": 54.0151, + "step": 13870 + }, + { + "epoch": 223.8780487804878, + "grad_norm": 0.730671763420105, + "learning_rate": 0.0001, + "loss": 53.601, + "step": 13880 + }, + { + "epoch": 224.03252032520325, + "grad_norm": 1.1248708963394165, + "learning_rate": 0.0001, + "loss": 51.4019, + "step": 13890 + }, + { + "epoch": 224.1951219512195, + "grad_norm": 1.034224033355713, + "learning_rate": 0.0001, + "loss": 54.3903, + "step": 13900 + }, + { + "epoch": 224.35772357723576, + "grad_norm": 1.0652328729629517, + "learning_rate": 0.0001, + "loss": 56.968, + "step": 13910 + }, + { + "epoch": 224.52032520325204, + "grad_norm": 0.8998368978500366, + "learning_rate": 0.0001, + "loss": 54.6571, + "step": 13920 + }, + { + "epoch": 224.6829268292683, + "grad_norm": 1.0750027894973755, + "learning_rate": 0.0001, + "loss": 53.7169, + "step": 13930 + }, + { + "epoch": 224.84552845528455, + "grad_norm": 1.1613131761550903, + "learning_rate": 0.0001, + "loss": 55.1695, + "step": 13940 + }, + { + "epoch": 225.0, + "grad_norm": 1.035953402519226, + "learning_rate": 0.0001, + "loss": 53.4469, + "step": 13950 + }, + { + "epoch": 225.16260162601625, + "grad_norm": 0.7781218886375427, + "learning_rate": 0.0001, + "loss": 55.1178, + "step": 13960 + }, + { + "epoch": 225.3252032520325, + "grad_norm": 0.9615271091461182, + "learning_rate": 0.0001, + "loss": 54.1518, + "step": 13970 + }, + { + "epoch": 225.4878048780488, + "grad_norm": 0.8908602595329285, + "learning_rate": 0.0001, + "loss": 56.4974, + "step": 13980 + }, + { + "epoch": 225.65040650406505, + "grad_norm": 1.3588823080062866, + "learning_rate": 0.0001, + "loss": 54.4211, + "step": 13990 + }, + { + "epoch": 225.8130081300813, + "grad_norm": 0.8439602255821228, + "learning_rate": 0.0001, + "loss": 53.8013, + "step": 14000 + }, + { + "epoch": 225.97560975609755, + "grad_norm": 0.8661660552024841, + "learning_rate": 0.0001, + "loss": 53.8126, + "step": 14010 + }, + { + "epoch": 226.130081300813, + "grad_norm": 1.3356671333312988, + "learning_rate": 0.0001, + "loss": 51.0263, + "step": 14020 + }, + { + "epoch": 226.29268292682926, + "grad_norm": 2.5738320350646973, + "learning_rate": 0.0001, + "loss": 55.0098, + "step": 14030 + }, + { + "epoch": 226.45528455284554, + "grad_norm": 0.9103851914405823, + "learning_rate": 0.0001, + "loss": 55.3346, + "step": 14040 + }, + { + "epoch": 226.6178861788618, + "grad_norm": 0.9767026901245117, + "learning_rate": 0.0001, + "loss": 51.9042, + "step": 14050 + }, + { + "epoch": 226.78048780487805, + "grad_norm": 0.7589291334152222, + "learning_rate": 0.0001, + "loss": 53.9771, + "step": 14060 + }, + { + "epoch": 226.9430894308943, + "grad_norm": 0.9664282202720642, + "learning_rate": 0.0001, + "loss": 54.2983, + "step": 14070 + }, + { + "epoch": 227.09756097560975, + "grad_norm": 0.7321500182151794, + "learning_rate": 0.0001, + "loss": 51.867, + "step": 14080 + }, + { + "epoch": 227.260162601626, + "grad_norm": 1.2257362604141235, + "learning_rate": 0.0001, + "loss": 53.7938, + "step": 14090 + }, + { + "epoch": 227.4227642276423, + "grad_norm": 1.1650798320770264, + "learning_rate": 0.0001, + "loss": 55.4182, + "step": 14100 + }, + { + "epoch": 227.58536585365854, + "grad_norm": 0.7246831059455872, + "learning_rate": 0.0001, + "loss": 55.1917, + "step": 14110 + }, + { + "epoch": 227.7479674796748, + "grad_norm": 0.8775220513343811, + "learning_rate": 0.0001, + "loss": 53.7243, + "step": 14120 + }, + { + "epoch": 227.91056910569105, + "grad_norm": 1.4759947061538696, + "learning_rate": 0.0001, + "loss": 54.2947, + "step": 14130 + }, + { + "epoch": 228.0650406504065, + "grad_norm": 1.6673812866210938, + "learning_rate": 0.0001, + "loss": 51.5433, + "step": 14140 + }, + { + "epoch": 228.22764227642276, + "grad_norm": 0.8332659006118774, + "learning_rate": 0.0001, + "loss": 56.2734, + "step": 14150 + }, + { + "epoch": 228.390243902439, + "grad_norm": 0.8206542134284973, + "learning_rate": 0.0001, + "loss": 55.9898, + "step": 14160 + }, + { + "epoch": 228.5528455284553, + "grad_norm": 0.7425916790962219, + "learning_rate": 0.0001, + "loss": 53.8255, + "step": 14170 + }, + { + "epoch": 228.71544715447155, + "grad_norm": 0.7315079569816589, + "learning_rate": 0.0001, + "loss": 54.7144, + "step": 14180 + }, + { + "epoch": 228.8780487804878, + "grad_norm": 0.9928258657455444, + "learning_rate": 0.0001, + "loss": 54.0064, + "step": 14190 + }, + { + "epoch": 229.03252032520325, + "grad_norm": 0.7837854623794556, + "learning_rate": 0.0001, + "loss": 51.6609, + "step": 14200 + }, + { + "epoch": 229.1951219512195, + "grad_norm": 0.90921550989151, + "learning_rate": 0.0001, + "loss": 55.787, + "step": 14210 + }, + { + "epoch": 229.35772357723576, + "grad_norm": 0.871849536895752, + "learning_rate": 0.0001, + "loss": 54.393, + "step": 14220 + }, + { + "epoch": 229.52032520325204, + "grad_norm": 1.0282469987869263, + "learning_rate": 0.0001, + "loss": 53.7822, + "step": 14230 + }, + { + "epoch": 229.6829268292683, + "grad_norm": 0.8177674412727356, + "learning_rate": 0.0001, + "loss": 53.9671, + "step": 14240 + }, + { + "epoch": 229.84552845528455, + "grad_norm": 0.993034303188324, + "learning_rate": 0.0001, + "loss": 54.4043, + "step": 14250 + }, + { + "epoch": 230.0, + "grad_norm": 0.8818158507347107, + "learning_rate": 0.0001, + "loss": 50.8848, + "step": 14260 + }, + { + "epoch": 230.16260162601625, + "grad_norm": 0.7449021935462952, + "learning_rate": 0.0001, + "loss": 54.0188, + "step": 14270 + }, + { + "epoch": 230.3252032520325, + "grad_norm": 0.7258269786834717, + "learning_rate": 0.0001, + "loss": 55.6941, + "step": 14280 + }, + { + "epoch": 230.4878048780488, + "grad_norm": 0.6683617234230042, + "learning_rate": 0.0001, + "loss": 55.1297, + "step": 14290 + }, + { + "epoch": 230.65040650406505, + "grad_norm": 1.3703839778900146, + "learning_rate": 0.0001, + "loss": 53.4986, + "step": 14300 + }, + { + "epoch": 230.8130081300813, + "grad_norm": 0.8369325995445251, + "learning_rate": 0.0001, + "loss": 54.2473, + "step": 14310 + }, + { + "epoch": 230.97560975609755, + "grad_norm": 0.8285283446311951, + "learning_rate": 0.0001, + "loss": 54.8075, + "step": 14320 + }, + { + "epoch": 231.130081300813, + "grad_norm": 0.9424141645431519, + "learning_rate": 0.0001, + "loss": 52.3665, + "step": 14330 + }, + { + "epoch": 231.29268292682926, + "grad_norm": 0.8255526423454285, + "learning_rate": 0.0001, + "loss": 54.6126, + "step": 14340 + }, + { + "epoch": 231.45528455284554, + "grad_norm": 1.006761074066162, + "learning_rate": 0.0001, + "loss": 53.7163, + "step": 14350 + }, + { + "epoch": 231.6178861788618, + "grad_norm": 0.9653576016426086, + "learning_rate": 0.0001, + "loss": 55.3647, + "step": 14360 + }, + { + "epoch": 231.78048780487805, + "grad_norm": 0.8040965795516968, + "learning_rate": 0.0001, + "loss": 55.3375, + "step": 14370 + }, + { + "epoch": 231.9430894308943, + "grad_norm": 0.9425880312919617, + "learning_rate": 0.0001, + "loss": 55.5746, + "step": 14380 + }, + { + "epoch": 232.09756097560975, + "grad_norm": 1.006316900253296, + "learning_rate": 0.0001, + "loss": 50.3978, + "step": 14390 + }, + { + "epoch": 232.260162601626, + "grad_norm": 1.136722207069397, + "learning_rate": 0.0001, + "loss": 54.1219, + "step": 14400 + }, + { + "epoch": 232.4227642276423, + "grad_norm": 0.8423946499824524, + "learning_rate": 0.0001, + "loss": 53.8679, + "step": 14410 + }, + { + "epoch": 232.58536585365854, + "grad_norm": 0.7676213979721069, + "learning_rate": 0.0001, + "loss": 54.5777, + "step": 14420 + }, + { + "epoch": 232.7479674796748, + "grad_norm": 2.3063645362854004, + "learning_rate": 0.0001, + "loss": 53.5352, + "step": 14430 + }, + { + "epoch": 232.91056910569105, + "grad_norm": 0.8056443333625793, + "learning_rate": 0.0001, + "loss": 55.2449, + "step": 14440 + }, + { + "epoch": 233.0650406504065, + "grad_norm": 1.1931108236312866, + "learning_rate": 0.0001, + "loss": 52.0586, + "step": 14450 + }, + { + "epoch": 233.22764227642276, + "grad_norm": 0.820271909236908, + "learning_rate": 0.0001, + "loss": 54.3902, + "step": 14460 + }, + { + "epoch": 233.390243902439, + "grad_norm": 1.0494563579559326, + "learning_rate": 0.0001, + "loss": 54.9187, + "step": 14470 + }, + { + "epoch": 233.5528455284553, + "grad_norm": 1.0478507280349731, + "learning_rate": 0.0001, + "loss": 55.2656, + "step": 14480 + }, + { + "epoch": 233.71544715447155, + "grad_norm": 0.8764968514442444, + "learning_rate": 0.0001, + "loss": 55.4161, + "step": 14490 + }, + { + "epoch": 233.8780487804878, + "grad_norm": 1.4083248376846313, + "learning_rate": 0.0001, + "loss": 55.1284, + "step": 14500 + }, + { + "epoch": 234.03252032520325, + "grad_norm": 0.9571195244789124, + "learning_rate": 0.0001, + "loss": 52.4321, + "step": 14510 + }, + { + "epoch": 234.1951219512195, + "grad_norm": 1.0617722272872925, + "learning_rate": 0.0001, + "loss": 54.0079, + "step": 14520 + }, + { + "epoch": 234.35772357723576, + "grad_norm": 0.8184088468551636, + "learning_rate": 0.0001, + "loss": 56.4753, + "step": 14530 + }, + { + "epoch": 234.52032520325204, + "grad_norm": 1.0400315523147583, + "learning_rate": 0.0001, + "loss": 54.4566, + "step": 14540 + }, + { + "epoch": 234.6829268292683, + "grad_norm": 0.9099937081336975, + "learning_rate": 0.0001, + "loss": 54.6234, + "step": 14550 + }, + { + "epoch": 234.84552845528455, + "grad_norm": 0.8048180341720581, + "learning_rate": 0.0001, + "loss": 56.1784, + "step": 14560 + }, + { + "epoch": 235.0, + "grad_norm": 0.888400673866272, + "learning_rate": 0.0001, + "loss": 50.9432, + "step": 14570 + }, + { + "epoch": 235.16260162601625, + "grad_norm": 0.8743310570716858, + "learning_rate": 0.0001, + "loss": 55.2681, + "step": 14580 + }, + { + "epoch": 235.3252032520325, + "grad_norm": 0.9016884565353394, + "learning_rate": 0.0001, + "loss": 53.566, + "step": 14590 + }, + { + "epoch": 235.4878048780488, + "grad_norm": 0.8748098015785217, + "learning_rate": 0.0001, + "loss": 52.3228, + "step": 14600 + }, + { + "epoch": 235.65040650406505, + "grad_norm": 1.3671938180923462, + "learning_rate": 0.0001, + "loss": 55.7807, + "step": 14610 + }, + { + "epoch": 235.8130081300813, + "grad_norm": 0.7671380043029785, + "learning_rate": 0.0001, + "loss": 53.2217, + "step": 14620 + }, + { + "epoch": 235.97560975609755, + "grad_norm": 1.0221309661865234, + "learning_rate": 0.0001, + "loss": 53.6954, + "step": 14630 + }, + { + "epoch": 236.130081300813, + "grad_norm": 1.9246286153793335, + "learning_rate": 0.0001, + "loss": 52.7675, + "step": 14640 + }, + { + "epoch": 236.29268292682926, + "grad_norm": 0.7186591029167175, + "learning_rate": 0.0001, + "loss": 54.508, + "step": 14650 + }, + { + "epoch": 236.45528455284554, + "grad_norm": 0.8058716654777527, + "learning_rate": 0.0001, + "loss": 54.0912, + "step": 14660 + }, + { + "epoch": 236.6178861788618, + "grad_norm": 0.7054926156997681, + "learning_rate": 0.0001, + "loss": 55.3318, + "step": 14670 + }, + { + "epoch": 236.78048780487805, + "grad_norm": 0.9968011379241943, + "learning_rate": 0.0001, + "loss": 55.3583, + "step": 14680 + }, + { + "epoch": 236.9430894308943, + "grad_norm": 1.5916420221328735, + "learning_rate": 0.0001, + "loss": 54.5985, + "step": 14690 + }, + { + "epoch": 237.09756097560975, + "grad_norm": 1.0716444253921509, + "learning_rate": 0.0001, + "loss": 53.3298, + "step": 14700 + }, + { + "epoch": 237.260162601626, + "grad_norm": 0.7287874221801758, + "learning_rate": 0.0001, + "loss": 54.7082, + "step": 14710 + }, + { + "epoch": 237.4227642276423, + "grad_norm": 0.860691487789154, + "learning_rate": 0.0001, + "loss": 52.5907, + "step": 14720 + }, + { + "epoch": 237.58536585365854, + "grad_norm": 0.8677813410758972, + "learning_rate": 0.0001, + "loss": 55.2442, + "step": 14730 + }, + { + "epoch": 237.7479674796748, + "grad_norm": 1.0112682580947876, + "learning_rate": 0.0001, + "loss": 54.6315, + "step": 14740 + }, + { + "epoch": 237.91056910569105, + "grad_norm": 0.6526017189025879, + "learning_rate": 0.0001, + "loss": 56.9737, + "step": 14750 + }, + { + "epoch": 238.0650406504065, + "grad_norm": 2.012220621109009, + "learning_rate": 0.0001, + "loss": 50.5104, + "step": 14760 + }, + { + "epoch": 238.22764227642276, + "grad_norm": 1.1385548114776611, + "learning_rate": 0.0001, + "loss": 54.953, + "step": 14770 + }, + { + "epoch": 238.390243902439, + "grad_norm": 0.915226936340332, + "learning_rate": 0.0001, + "loss": 53.3019, + "step": 14780 + }, + { + "epoch": 238.5528455284553, + "grad_norm": 1.0534974336624146, + "learning_rate": 0.0001, + "loss": 53.7377, + "step": 14790 + }, + { + "epoch": 238.71544715447155, + "grad_norm": 0.6639984846115112, + "learning_rate": 0.0001, + "loss": 51.6132, + "step": 14800 + }, + { + "epoch": 238.8780487804878, + "grad_norm": 1.1607130765914917, + "learning_rate": 0.0001, + "loss": 51.6656, + "step": 14810 + }, + { + "epoch": 239.03252032520325, + "grad_norm": 0.8796831965446472, + "learning_rate": 0.0001, + "loss": 53.0906, + "step": 14820 + }, + { + "epoch": 239.1951219512195, + "grad_norm": 1.3613159656524658, + "learning_rate": 0.0001, + "loss": 54.371, + "step": 14830 + }, + { + "epoch": 239.35772357723576, + "grad_norm": 1.1560760736465454, + "learning_rate": 0.0001, + "loss": 53.6194, + "step": 14840 + }, + { + "epoch": 239.52032520325204, + "grad_norm": 0.8476263880729675, + "learning_rate": 0.0001, + "loss": 52.5599, + "step": 14850 + }, + { + "epoch": 239.6829268292683, + "grad_norm": 0.7116367816925049, + "learning_rate": 0.0001, + "loss": 54.79, + "step": 14860 + }, + { + "epoch": 239.84552845528455, + "grad_norm": 1.1638411283493042, + "learning_rate": 0.0001, + "loss": 53.5391, + "step": 14870 + }, + { + "epoch": 240.0, + "grad_norm": 0.947271466255188, + "learning_rate": 0.0001, + "loss": 51.3252, + "step": 14880 + }, + { + "epoch": 240.16260162601625, + "grad_norm": 0.960483968257904, + "learning_rate": 0.0001, + "loss": 54.3772, + "step": 14890 + }, + { + "epoch": 240.3252032520325, + "grad_norm": 0.841806948184967, + "learning_rate": 0.0001, + "loss": 52.1828, + "step": 14900 + }, + { + "epoch": 240.4878048780488, + "grad_norm": 0.9069699645042419, + "learning_rate": 0.0001, + "loss": 52.5106, + "step": 14910 + }, + { + "epoch": 240.65040650406505, + "grad_norm": 1.1150813102722168, + "learning_rate": 0.0001, + "loss": 56.0098, + "step": 14920 + }, + { + "epoch": 240.8130081300813, + "grad_norm": 0.7638039588928223, + "learning_rate": 0.0001, + "loss": 53.7415, + "step": 14930 + }, + { + "epoch": 240.97560975609755, + "grad_norm": 1.1885515451431274, + "learning_rate": 0.0001, + "loss": 55.089, + "step": 14940 + }, + { + "epoch": 241.130081300813, + "grad_norm": 1.0159634351730347, + "learning_rate": 0.0001, + "loss": 51.1617, + "step": 14950 + }, + { + "epoch": 241.29268292682926, + "grad_norm": 1.0128728151321411, + "learning_rate": 0.0001, + "loss": 53.957, + "step": 14960 + }, + { + "epoch": 241.45528455284554, + "grad_norm": 0.9399421811103821, + "learning_rate": 0.0001, + "loss": 55.1805, + "step": 14970 + }, + { + "epoch": 241.6178861788618, + "grad_norm": 0.7499178051948547, + "learning_rate": 0.0001, + "loss": 53.4352, + "step": 14980 + }, + { + "epoch": 241.78048780487805, + "grad_norm": 0.6498700380325317, + "learning_rate": 0.0001, + "loss": 54.0107, + "step": 14990 + }, + { + "epoch": 241.9430894308943, + "grad_norm": 0.7191994190216064, + "learning_rate": 0.0001, + "loss": 56.2089, + "step": 15000 + }, + { + "epoch": 242.09756097560975, + "grad_norm": 1.18024742603302, + "learning_rate": 0.0001, + "loss": 51.6447, + "step": 15010 + }, + { + "epoch": 242.260162601626, + "grad_norm": 0.9039168953895569, + "learning_rate": 0.0001, + "loss": 53.4341, + "step": 15020 + }, + { + "epoch": 242.4227642276423, + "grad_norm": 1.0368373394012451, + "learning_rate": 0.0001, + "loss": 53.7973, + "step": 15030 + }, + { + "epoch": 242.58536585365854, + "grad_norm": 0.8648324012756348, + "learning_rate": 0.0001, + "loss": 55.773, + "step": 15040 + }, + { + "epoch": 242.7479674796748, + "grad_norm": 0.7960569262504578, + "learning_rate": 0.0001, + "loss": 53.6785, + "step": 15050 + }, + { + "epoch": 242.91056910569105, + "grad_norm": 0.771918773651123, + "learning_rate": 0.0001, + "loss": 54.7576, + "step": 15060 + }, + { + "epoch": 243.0650406504065, + "grad_norm": 1.105131983757019, + "learning_rate": 0.0001, + "loss": 51.4313, + "step": 15070 + }, + { + "epoch": 243.22764227642276, + "grad_norm": 1.053236722946167, + "learning_rate": 0.0001, + "loss": 54.1687, + "step": 15080 + }, + { + "epoch": 243.390243902439, + "grad_norm": 1.4011870622634888, + "learning_rate": 0.0001, + "loss": 55.7092, + "step": 15090 + }, + { + "epoch": 243.5528455284553, + "grad_norm": 0.8540129065513611, + "learning_rate": 0.0001, + "loss": 54.2341, + "step": 15100 + }, + { + "epoch": 243.71544715447155, + "grad_norm": 1.666799545288086, + "learning_rate": 0.0001, + "loss": 52.3724, + "step": 15110 + }, + { + "epoch": 243.8780487804878, + "grad_norm": 1.0199878215789795, + "learning_rate": 0.0001, + "loss": 54.5811, + "step": 15120 + }, + { + "epoch": 244.03252032520325, + "grad_norm": 0.8971478939056396, + "learning_rate": 0.0001, + "loss": 52.6352, + "step": 15130 + }, + { + "epoch": 244.1951219512195, + "grad_norm": 0.6896243691444397, + "learning_rate": 0.0001, + "loss": 53.9287, + "step": 15140 + }, + { + "epoch": 244.35772357723576, + "grad_norm": 0.6954209208488464, + "learning_rate": 0.0001, + "loss": 53.2609, + "step": 15150 + }, + { + "epoch": 244.52032520325204, + "grad_norm": 0.8604191541671753, + "learning_rate": 0.0001, + "loss": 55.3024, + "step": 15160 + }, + { + "epoch": 244.6829268292683, + "grad_norm": 1.3844572305679321, + "learning_rate": 0.0001, + "loss": 53.6013, + "step": 15170 + }, + { + "epoch": 244.84552845528455, + "grad_norm": 1.3308122158050537, + "learning_rate": 0.0001, + "loss": 53.0818, + "step": 15180 + }, + { + "epoch": 245.0, + "grad_norm": 0.9475764632225037, + "learning_rate": 0.0001, + "loss": 51.509, + "step": 15190 + }, + { + "epoch": 245.16260162601625, + "grad_norm": 0.8317999243736267, + "learning_rate": 0.0001, + "loss": 53.8405, + "step": 15200 + }, + { + "epoch": 245.3252032520325, + "grad_norm": 0.7647470235824585, + "learning_rate": 0.0001, + "loss": 55.3181, + "step": 15210 + }, + { + "epoch": 245.4878048780488, + "grad_norm": 0.8726988434791565, + "learning_rate": 0.0001, + "loss": 55.4537, + "step": 15220 + }, + { + "epoch": 245.65040650406505, + "grad_norm": 0.9479020237922668, + "learning_rate": 0.0001, + "loss": 53.913, + "step": 15230 + }, + { + "epoch": 245.8130081300813, + "grad_norm": 0.7102323770523071, + "learning_rate": 0.0001, + "loss": 54.3977, + "step": 15240 + }, + { + "epoch": 245.97560975609755, + "grad_norm": 0.7271092534065247, + "learning_rate": 0.0001, + "loss": 53.5612, + "step": 15250 + }, + { + "epoch": 246.130081300813, + "grad_norm": 0.9429534673690796, + "learning_rate": 0.0001, + "loss": 52.4, + "step": 15260 + }, + { + "epoch": 246.29268292682926, + "grad_norm": 1.7194772958755493, + "learning_rate": 0.0001, + "loss": 54.2578, + "step": 15270 + }, + { + "epoch": 246.45528455284554, + "grad_norm": 1.1360211372375488, + "learning_rate": 0.0001, + "loss": 52.4441, + "step": 15280 + }, + { + "epoch": 246.6178861788618, + "grad_norm": 0.8528667092323303, + "learning_rate": 0.0001, + "loss": 54.6957, + "step": 15290 + }, + { + "epoch": 246.78048780487805, + "grad_norm": 0.6910762786865234, + "learning_rate": 0.0001, + "loss": 54.7472, + "step": 15300 + }, + { + "epoch": 246.9430894308943, + "grad_norm": 0.6756943464279175, + "learning_rate": 0.0001, + "loss": 53.0433, + "step": 15310 + }, + { + "epoch": 247.09756097560975, + "grad_norm": 0.949343740940094, + "learning_rate": 0.0001, + "loss": 51.568, + "step": 15320 + }, + { + "epoch": 247.260162601626, + "grad_norm": 1.0063872337341309, + "learning_rate": 0.0001, + "loss": 55.162, + "step": 15330 + }, + { + "epoch": 247.4227642276423, + "grad_norm": 0.7869600057601929, + "learning_rate": 0.0001, + "loss": 53.789, + "step": 15340 + }, + { + "epoch": 247.58536585365854, + "grad_norm": 0.9456334114074707, + "learning_rate": 0.0001, + "loss": 54.3977, + "step": 15350 + }, + { + "epoch": 247.7479674796748, + "grad_norm": 0.6960943341255188, + "learning_rate": 0.0001, + "loss": 52.5222, + "step": 15360 + }, + { + "epoch": 247.91056910569105, + "grad_norm": 0.5755976438522339, + "learning_rate": 0.0001, + "loss": 53.8927, + "step": 15370 + }, + { + "epoch": 248.0650406504065, + "grad_norm": 2.368572235107422, + "learning_rate": 0.0001, + "loss": 51.5995, + "step": 15380 + }, + { + "epoch": 248.22764227642276, + "grad_norm": 0.8773524761199951, + "learning_rate": 0.0001, + "loss": 54.296, + "step": 15390 + }, + { + "epoch": 248.390243902439, + "grad_norm": 1.0561209917068481, + "learning_rate": 0.0001, + "loss": 51.7327, + "step": 15400 + }, + { + "epoch": 248.5528455284553, + "grad_norm": 1.0130037069320679, + "learning_rate": 0.0001, + "loss": 54.616, + "step": 15410 + }, + { + "epoch": 248.71544715447155, + "grad_norm": 0.8258432745933533, + "learning_rate": 0.0001, + "loss": 53.8808, + "step": 15420 + }, + { + "epoch": 248.8780487804878, + "grad_norm": 0.7326891422271729, + "learning_rate": 0.0001, + "loss": 53.3624, + "step": 15430 + }, + { + "epoch": 249.03252032520325, + "grad_norm": 1.2207914590835571, + "learning_rate": 0.0001, + "loss": 51.166, + "step": 15440 + }, + { + "epoch": 249.1951219512195, + "grad_norm": 0.9404299855232239, + "learning_rate": 0.0001, + "loss": 54.9002, + "step": 15450 + }, + { + "epoch": 249.35772357723576, + "grad_norm": 0.6494407057762146, + "learning_rate": 0.0001, + "loss": 54.4227, + "step": 15460 + }, + { + "epoch": 249.52032520325204, + "grad_norm": 1.112674355506897, + "learning_rate": 0.0001, + "loss": 52.1128, + "step": 15470 + }, + { + "epoch": 249.6829268292683, + "grad_norm": 0.8266693949699402, + "learning_rate": 0.0001, + "loss": 53.8957, + "step": 15480 + }, + { + "epoch": 249.84552845528455, + "grad_norm": 0.8604085445404053, + "learning_rate": 0.0001, + "loss": 54.8597, + "step": 15490 + }, + { + "epoch": 250.0, + "grad_norm": 0.8825786709785461, + "learning_rate": 0.0001, + "loss": 51.4052, + "step": 15500 + }, + { + "epoch": 250.16260162601625, + "grad_norm": 0.8595220446586609, + "learning_rate": 0.0001, + "loss": 56.6767, + "step": 15510 + }, + { + "epoch": 250.3252032520325, + "grad_norm": 0.9159016013145447, + "learning_rate": 0.0001, + "loss": 53.6622, + "step": 15520 + }, + { + "epoch": 250.4878048780488, + "grad_norm": 0.9752806425094604, + "learning_rate": 0.0001, + "loss": 53.7919, + "step": 15530 + }, + { + "epoch": 250.65040650406505, + "grad_norm": 1.299797773361206, + "learning_rate": 0.0001, + "loss": 52.582, + "step": 15540 + }, + { + "epoch": 250.8130081300813, + "grad_norm": 0.8046541213989258, + "learning_rate": 0.0001, + "loss": 56.1583, + "step": 15550 + }, + { + "epoch": 250.97560975609755, + "grad_norm": 0.8848250508308411, + "learning_rate": 0.0001, + "loss": 55.1164, + "step": 15560 + }, + { + "epoch": 251.130081300813, + "grad_norm": 0.9489118456840515, + "learning_rate": 0.0001, + "loss": 51.5053, + "step": 15570 + }, + { + "epoch": 251.29268292682926, + "grad_norm": 5.069701194763184, + "learning_rate": 0.0001, + "loss": 54.2582, + "step": 15580 + }, + { + "epoch": 251.45528455284554, + "grad_norm": 0.7503376603126526, + "learning_rate": 0.0001, + "loss": 52.8597, + "step": 15590 + }, + { + "epoch": 251.6178861788618, + "grad_norm": 0.7200372815132141, + "learning_rate": 0.0001, + "loss": 52.6846, + "step": 15600 + }, + { + "epoch": 251.78048780487805, + "grad_norm": 1.4221407175064087, + "learning_rate": 0.0001, + "loss": 52.2177, + "step": 15610 + }, + { + "epoch": 251.9430894308943, + "grad_norm": 0.7428510189056396, + "learning_rate": 0.0001, + "loss": 54.7994, + "step": 15620 + }, + { + "epoch": 252.09756097560975, + "grad_norm": 0.9019786715507507, + "learning_rate": 0.0001, + "loss": 51.3638, + "step": 15630 + }, + { + "epoch": 252.260162601626, + "grad_norm": 0.8459905385971069, + "learning_rate": 0.0001, + "loss": 55.4398, + "step": 15640 + }, + { + "epoch": 252.4227642276423, + "grad_norm": 1.040996789932251, + "learning_rate": 0.0001, + "loss": 51.7563, + "step": 15650 + }, + { + "epoch": 252.58536585365854, + "grad_norm": 0.9953641295433044, + "learning_rate": 0.0001, + "loss": 52.351, + "step": 15660 + }, + { + "epoch": 252.7479674796748, + "grad_norm": 1.0336418151855469, + "learning_rate": 0.0001, + "loss": 52.2774, + "step": 15670 + }, + { + "epoch": 252.91056910569105, + "grad_norm": 0.728695809841156, + "learning_rate": 0.0001, + "loss": 53.8307, + "step": 15680 + }, + { + "epoch": 253.0650406504065, + "grad_norm": 0.8020172715187073, + "learning_rate": 0.0001, + "loss": 50.6589, + "step": 15690 + }, + { + "epoch": 253.22764227642276, + "grad_norm": 1.0117321014404297, + "learning_rate": 0.0001, + "loss": 55.3505, + "step": 15700 + }, + { + "epoch": 253.390243902439, + "grad_norm": 0.8227602243423462, + "learning_rate": 0.0001, + "loss": 56.1819, + "step": 15710 + }, + { + "epoch": 253.5528455284553, + "grad_norm": 0.7399566173553467, + "learning_rate": 0.0001, + "loss": 54.7373, + "step": 15720 + }, + { + "epoch": 253.71544715447155, + "grad_norm": 1.1013562679290771, + "learning_rate": 0.0001, + "loss": 54.1074, + "step": 15730 + }, + { + "epoch": 253.8780487804878, + "grad_norm": 1.2620662450790405, + "learning_rate": 0.0001, + "loss": 53.1314, + "step": 15740 + }, + { + "epoch": 254.03252032520325, + "grad_norm": 0.713722825050354, + "learning_rate": 0.0001, + "loss": 50.3889, + "step": 15750 + }, + { + "epoch": 254.1951219512195, + "grad_norm": 1.7682756185531616, + "learning_rate": 0.0001, + "loss": 55.0939, + "step": 15760 + }, + { + "epoch": 254.35772357723576, + "grad_norm": 2.202893018722534, + "learning_rate": 0.0001, + "loss": 53.8187, + "step": 15770 + }, + { + "epoch": 254.52032520325204, + "grad_norm": 1.1321382522583008, + "learning_rate": 0.0001, + "loss": 53.4802, + "step": 15780 + }, + { + "epoch": 254.6829268292683, + "grad_norm": 0.7600924968719482, + "learning_rate": 0.0001, + "loss": 54.0337, + "step": 15790 + }, + { + "epoch": 254.84552845528455, + "grad_norm": 0.7733964323997498, + "learning_rate": 0.0001, + "loss": 55.608, + "step": 15800 + }, + { + "epoch": 255.0, + "grad_norm": 0.8145171403884888, + "learning_rate": 0.0001, + "loss": 50.7658, + "step": 15810 + }, + { + "epoch": 255.16260162601625, + "grad_norm": 0.8330771327018738, + "learning_rate": 0.0001, + "loss": 52.5933, + "step": 15820 + }, + { + "epoch": 255.3252032520325, + "grad_norm": 0.87321537733078, + "learning_rate": 0.0001, + "loss": 55.3489, + "step": 15830 + }, + { + "epoch": 255.4878048780488, + "grad_norm": 0.8081530928611755, + "learning_rate": 0.0001, + "loss": 55.5343, + "step": 15840 + }, + { + "epoch": 255.65040650406505, + "grad_norm": 1.308362603187561, + "learning_rate": 0.0001, + "loss": 54.2978, + "step": 15850 + }, + { + "epoch": 255.8130081300813, + "grad_norm": 0.8281819820404053, + "learning_rate": 0.0001, + "loss": 54.8497, + "step": 15860 + }, + { + "epoch": 255.97560975609755, + "grad_norm": 0.9533831477165222, + "learning_rate": 0.0001, + "loss": 52.4189, + "step": 15870 + }, + { + "epoch": 256.130081300813, + "grad_norm": 1.1174825429916382, + "learning_rate": 0.0001, + "loss": 51.8444, + "step": 15880 + }, + { + "epoch": 256.2926829268293, + "grad_norm": 0.6530044078826904, + "learning_rate": 0.0001, + "loss": 53.4939, + "step": 15890 + }, + { + "epoch": 256.4552845528455, + "grad_norm": 1.254204511642456, + "learning_rate": 0.0001, + "loss": 53.9468, + "step": 15900 + }, + { + "epoch": 256.6178861788618, + "grad_norm": 0.8999534845352173, + "learning_rate": 0.0001, + "loss": 55.0074, + "step": 15910 + }, + { + "epoch": 256.780487804878, + "grad_norm": 0.7126501798629761, + "learning_rate": 0.0001, + "loss": 53.0478, + "step": 15920 + }, + { + "epoch": 256.9430894308943, + "grad_norm": 0.7162296772003174, + "learning_rate": 0.0001, + "loss": 54.1921, + "step": 15930 + }, + { + "epoch": 257.0975609756098, + "grad_norm": 0.7281424403190613, + "learning_rate": 0.0001, + "loss": 50.6751, + "step": 15940 + }, + { + "epoch": 257.260162601626, + "grad_norm": 1.1298840045928955, + "learning_rate": 0.0001, + "loss": 54.8229, + "step": 15950 + }, + { + "epoch": 257.4227642276423, + "grad_norm": 0.6440666317939758, + "learning_rate": 0.0001, + "loss": 53.4585, + "step": 15960 + }, + { + "epoch": 257.5853658536585, + "grad_norm": 0.8525786995887756, + "learning_rate": 0.0001, + "loss": 52.6461, + "step": 15970 + }, + { + "epoch": 257.7479674796748, + "grad_norm": 1.1225342750549316, + "learning_rate": 0.0001, + "loss": 53.8119, + "step": 15980 + }, + { + "epoch": 257.9105691056911, + "grad_norm": 1.1013641357421875, + "learning_rate": 0.0001, + "loss": 52.155, + "step": 15990 + }, + { + "epoch": 258.0650406504065, + "grad_norm": 0.9112017154693604, + "learning_rate": 0.0001, + "loss": 50.1069, + "step": 16000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 323, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9434058260504222e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c7f229ebe1034caaf3e92006c8c7528c21af93d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517586849d044483051c7a59cec7c6ce389951878a7357ef837edb1cbc9983c8 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-16000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..801d78391c71b2ea88973da027455c40327f1971 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dca7824a80c994fb2211cb79f9a04218ca658b46f4c2393e4179d7810170e0a +size 1853729968 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..083fd488e4750535ffd9536eb20021fe55056cbb --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77fe286bf4ddd2b6d3b7aa5f750cd67ea774db67ce0680c7487f4f32d4975b71 +size 1853730480 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de1ac3f32fa813814de51e44d04b5e14a9764fc6 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b595275d2d1f9a14a92e8908605d6180355a1e04f1fa10b92e164f4a245b6b2 +size 1853731312 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e404b6bf0856800949de47748fcb2ce52946efb --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35aad193734064322bcd892af10d53fbab383600e908e8443482b03b14b1cc3a +size 1853731120 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59aeff22aa5846563b3ebb1efcc54598fc2ba298 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a591d2cb76c64a20f4acda03cee6381c18c28e2517c805f37df3acb0e7b90c79 +size 1853731312 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b562f93448213d009880767db47757642eca2e6 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72bd0c42150e09c0a79dc1885d2c468b155bb2b15958452aa36dd002d9727e7f +size 1853731120 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..10b3de521e1c3a114f0f8bd35035e4b6cbea6069 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fe208d8cfd104abe3346ba102eec7f7f23f1143bb9a5b6baedb17dec6d7c57d +size 1853731440 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f68bae38ddf8a071bfc1ee687472c0e5ec86aa47 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5ff4216a5b27ecd7bf61f7fa13b1682ec6c707c76009a526ac06b42aed51b87 +size 1853731056 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/mp_rank_00_model_states.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd9268254a2918a635bc15b547697335bb0b6226 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/global_step18000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c883b232f981898a23898b2f77c3e5d93ee55b14d8dd30bb379628d9979b0fa +size 2471676664 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/latest new file mode 100644 index 0000000000000000000000000000000000000000..b73f60d7a0f5e4a77d445820eafbd0d5e13129a2 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/latest @@ -0,0 +1 @@ +global_step18000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..623b4e9a10e7b1fbb10b648682813faea58a0b4f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8014e4fd2bebf48193b5168a3b95b348a168e2ece182165f55dae330c9510d88 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..3ecc2adadcb468ada5537f0d9f91add01b1220d8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58f32b9288ef4082106ce2813fea1100d2b1d0004d45aac0c1e565fd9d6654ca +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..7b1cff2df8a584a15dc863e517cf8a93329b98a7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:628bd5080c2d80ce8b94b1b9708e7f71ad6b6ad3b03f76853fe690f75f467eae +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..03a0366e285449b4c1cee0b60c99de214e5991d2 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9f8c6d15e6f085297e8d0e87135d316fcf72998e79b25ced6e5d905ca004a75 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..50489a53824bf2cf776b06bb01fb732d901bd19d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7cde1cb861cdf53191ec5ea6881959ed2e7d179fd46197f6301dd09e9e72d84 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..5dbac9d84091ff363f222126222d68ba08a86c73 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d13301c9934dc20d2b7f0062e3409fb7099ff51d69384549022c39f630a52572 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..459905f4cbbb10481c8d5806bc935c8b1881e89d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aec1b8c30417a51e6d92109c229dd4d7d520521c41d032a8002c65a0219eedf +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..61c8fc8f6ab1b1c1dc0469ad0b561ddb2f72676c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e1dda1d89cfe424dc689ff3338baac8b964b08b96034f79e059fec34e1773e7 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..3cea4c1e696f1233c3a122716881c6e51bfa732a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7ff1d6227cc26e4ad658890ce985838e52fa952e400ab383238d538559426e0 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..21565de482b40f006d37963ebafcdfe26c25d297 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6811f4eff35f13019fe7a0f3e11af3e201050fd538642f614a8e73a804630d2 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fbdadf4fd12eca2b33030455eb7e5e9a2ee13228 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/trainer_state.json @@ -0,0 +1,12634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 290.3252032520325, + "eval_steps": 2000, + "global_step": 18000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.16260162601626016, + "grad_norm": 23.72986602783203, + "learning_rate": 0.0001, + "loss": 271.6598, + "step": 10 + }, + { + "epoch": 0.3252032520325203, + "grad_norm": 18.99613380432129, + "learning_rate": 0.0001, + "loss": 233.029, + "step": 20 + }, + { + "epoch": 0.4878048780487805, + "grad_norm": 3.9815304279327393, + "learning_rate": 0.0001, + "loss": 231.0776, + "step": 30 + }, + { + "epoch": 0.6504065040650406, + "grad_norm": 4.641753196716309, + "learning_rate": 0.0001, + "loss": 228.6894, + "step": 40 + }, + { + "epoch": 0.8130081300813008, + "grad_norm": 3.000852108001709, + "learning_rate": 0.0001, + "loss": 228.4694, + "step": 50 + }, + { + "epoch": 0.975609756097561, + "grad_norm": 12.213018417358398, + "learning_rate": 0.0001, + "loss": 226.4258, + "step": 60 + }, + { + "epoch": 1.1300813008130082, + "grad_norm": 2.535187005996704, + "learning_rate": 0.0001, + "loss": 209.9772, + "step": 70 + }, + { + "epoch": 1.2926829268292683, + "grad_norm": 2.0984582901000977, + "learning_rate": 0.0001, + "loss": 217.2421, + "step": 80 + }, + { + "epoch": 1.4552845528455285, + "grad_norm": 3.4802305698394775, + "learning_rate": 0.0001, + "loss": 210.9241, + "step": 90 + }, + { + "epoch": 1.6178861788617886, + "grad_norm": 3.813626527786255, + "learning_rate": 0.0001, + "loss": 205.7566, + "step": 100 + }, + { + "epoch": 1.7804878048780488, + "grad_norm": 2.6378512382507324, + "learning_rate": 0.0001, + "loss": 201.8789, + "step": 110 + }, + { + "epoch": 1.943089430894309, + "grad_norm": 2.7502875328063965, + "learning_rate": 0.0001, + "loss": 196.373, + "step": 120 + }, + { + "epoch": 2.097560975609756, + "grad_norm": 2.659212112426758, + "learning_rate": 0.0001, + "loss": 183.1456, + "step": 130 + }, + { + "epoch": 2.2601626016260163, + "grad_norm": 2.513044595718384, + "learning_rate": 0.0001, + "loss": 190.1661, + "step": 140 + }, + { + "epoch": 2.4227642276422765, + "grad_norm": 2.3192145824432373, + "learning_rate": 0.0001, + "loss": 184.9997, + "step": 150 + }, + { + "epoch": 2.5853658536585367, + "grad_norm": 3.056708574295044, + "learning_rate": 0.0001, + "loss": 182.0164, + "step": 160 + }, + { + "epoch": 2.747967479674797, + "grad_norm": 2.9687108993530273, + "learning_rate": 0.0001, + "loss": 179.6539, + "step": 170 + }, + { + "epoch": 2.910569105691057, + "grad_norm": 2.1498093605041504, + "learning_rate": 0.0001, + "loss": 176.7159, + "step": 180 + }, + { + "epoch": 3.065040650406504, + "grad_norm": 3.0103771686553955, + "learning_rate": 0.0001, + "loss": 166.1703, + "step": 190 + }, + { + "epoch": 3.227642276422764, + "grad_norm": 2.992462635040283, + "learning_rate": 0.0001, + "loss": 171.124, + "step": 200 + }, + { + "epoch": 3.3902439024390243, + "grad_norm": 4.8617119789123535, + "learning_rate": 0.0001, + "loss": 168.2646, + "step": 210 + }, + { + "epoch": 3.5528455284552845, + "grad_norm": 3.1047444343566895, + "learning_rate": 0.0001, + "loss": 163.0889, + "step": 220 + }, + { + "epoch": 3.7154471544715446, + "grad_norm": 6.473227500915527, + "learning_rate": 0.0001, + "loss": 151.3667, + "step": 230 + }, + { + "epoch": 3.8780487804878048, + "grad_norm": 6.16747522354126, + "learning_rate": 0.0001, + "loss": 137.5915, + "step": 240 + }, + { + "epoch": 4.032520325203252, + "grad_norm": 7.994515419006348, + "learning_rate": 0.0001, + "loss": 118.8875, + "step": 250 + }, + { + "epoch": 4.195121951219512, + "grad_norm": 5.882101058959961, + "learning_rate": 0.0001, + "loss": 119.4793, + "step": 260 + }, + { + "epoch": 4.357723577235772, + "grad_norm": 12.32968521118164, + "learning_rate": 0.0001, + "loss": 113.1209, + "step": 270 + }, + { + "epoch": 4.520325203252033, + "grad_norm": 5.697225570678711, + "learning_rate": 0.0001, + "loss": 107.51, + "step": 280 + }, + { + "epoch": 4.682926829268292, + "grad_norm": 4.654167175292969, + "learning_rate": 0.0001, + "loss": 107.0172, + "step": 290 + }, + { + "epoch": 4.845528455284553, + "grad_norm": 2.9595484733581543, + "learning_rate": 0.0001, + "loss": 100.9164, + "step": 300 + }, + { + "epoch": 5.0, + "grad_norm": 21.242591857910156, + "learning_rate": 0.0001, + "loss": 94.9884, + "step": 310 + }, + { + "epoch": 5.16260162601626, + "grad_norm": 5.760314464569092, + "learning_rate": 0.0001, + "loss": 97.5901, + "step": 320 + }, + { + "epoch": 5.32520325203252, + "grad_norm": 3.1373329162597656, + "learning_rate": 0.0001, + "loss": 94.8485, + "step": 330 + }, + { + "epoch": 5.487804878048781, + "grad_norm": 2.3777568340301514, + "learning_rate": 0.0001, + "loss": 96.9982, + "step": 340 + }, + { + "epoch": 5.650406504065041, + "grad_norm": 3.0607948303222656, + "learning_rate": 0.0001, + "loss": 96.1586, + "step": 350 + }, + { + "epoch": 5.8130081300813, + "grad_norm": 4.168269634246826, + "learning_rate": 0.0001, + "loss": 94.5743, + "step": 360 + }, + { + "epoch": 5.975609756097561, + "grad_norm": 2.4383699893951416, + "learning_rate": 0.0001, + "loss": 95.5197, + "step": 370 + }, + { + "epoch": 6.130081300813008, + "grad_norm": 2.302259683609009, + "learning_rate": 0.0001, + "loss": 85.6599, + "step": 380 + }, + { + "epoch": 6.2926829268292686, + "grad_norm": 2.552753210067749, + "learning_rate": 0.0001, + "loss": 91.2999, + "step": 390 + }, + { + "epoch": 6.455284552845528, + "grad_norm": 2.2092080116271973, + "learning_rate": 0.0001, + "loss": 90.7041, + "step": 400 + }, + { + "epoch": 6.617886178861789, + "grad_norm": 2.2935690879821777, + "learning_rate": 0.0001, + "loss": 89.9949, + "step": 410 + }, + { + "epoch": 6.780487804878049, + "grad_norm": 2.487510919570923, + "learning_rate": 0.0001, + "loss": 86.9056, + "step": 420 + }, + { + "epoch": 6.943089430894309, + "grad_norm": 2.8630545139312744, + "learning_rate": 0.0001, + "loss": 88.3347, + "step": 430 + }, + { + "epoch": 7.097560975609756, + "grad_norm": 5.87056303024292, + "learning_rate": 0.0001, + "loss": 84.9632, + "step": 440 + }, + { + "epoch": 7.260162601626016, + "grad_norm": 2.587661027908325, + "learning_rate": 0.0001, + "loss": 86.2024, + "step": 450 + }, + { + "epoch": 7.4227642276422765, + "grad_norm": 3.5671255588531494, + "learning_rate": 0.0001, + "loss": 86.218, + "step": 460 + }, + { + "epoch": 7.585365853658536, + "grad_norm": 2.5036680698394775, + "learning_rate": 0.0001, + "loss": 86.5914, + "step": 470 + }, + { + "epoch": 7.747967479674797, + "grad_norm": 3.2656471729278564, + "learning_rate": 0.0001, + "loss": 82.6081, + "step": 480 + }, + { + "epoch": 7.9105691056910565, + "grad_norm": 2.5053915977478027, + "learning_rate": 0.0001, + "loss": 84.6116, + "step": 490 + }, + { + "epoch": 8.065040650406504, + "grad_norm": 2.1669957637786865, + "learning_rate": 0.0001, + "loss": 80.4354, + "step": 500 + }, + { + "epoch": 8.227642276422765, + "grad_norm": 4.2003703117370605, + "learning_rate": 0.0001, + "loss": 85.2286, + "step": 510 + }, + { + "epoch": 8.390243902439025, + "grad_norm": 3.1909608840942383, + "learning_rate": 0.0001, + "loss": 85.0358, + "step": 520 + }, + { + "epoch": 8.552845528455284, + "grad_norm": 1.8420121669769287, + "learning_rate": 0.0001, + "loss": 85.4647, + "step": 530 + }, + { + "epoch": 8.715447154471544, + "grad_norm": 1.9822801351547241, + "learning_rate": 0.0001, + "loss": 84.7134, + "step": 540 + }, + { + "epoch": 8.878048780487806, + "grad_norm": 2.936758518218994, + "learning_rate": 0.0001, + "loss": 83.8068, + "step": 550 + }, + { + "epoch": 9.032520325203253, + "grad_norm": 1.5713881254196167, + "learning_rate": 0.0001, + "loss": 80.0695, + "step": 560 + }, + { + "epoch": 9.195121951219512, + "grad_norm": 2.0584616661071777, + "learning_rate": 0.0001, + "loss": 84.5158, + "step": 570 + }, + { + "epoch": 9.357723577235772, + "grad_norm": 2.101391553878784, + "learning_rate": 0.0001, + "loss": 82.4719, + "step": 580 + }, + { + "epoch": 9.520325203252032, + "grad_norm": 1.9956543445587158, + "learning_rate": 0.0001, + "loss": 81.6715, + "step": 590 + }, + { + "epoch": 9.682926829268293, + "grad_norm": 2.7622063159942627, + "learning_rate": 0.0001, + "loss": 85.4178, + "step": 600 + }, + { + "epoch": 9.845528455284553, + "grad_norm": 2.2084789276123047, + "learning_rate": 0.0001, + "loss": 84.5702, + "step": 610 + }, + { + "epoch": 10.0, + "grad_norm": 1.9635663032531738, + "learning_rate": 0.0001, + "loss": 77.3838, + "step": 620 + }, + { + "epoch": 10.16260162601626, + "grad_norm": 2.5895581245422363, + "learning_rate": 0.0001, + "loss": 81.723, + "step": 630 + }, + { + "epoch": 10.32520325203252, + "grad_norm": 2.743727445602417, + "learning_rate": 0.0001, + "loss": 83.1332, + "step": 640 + }, + { + "epoch": 10.487804878048781, + "grad_norm": 2.024585008621216, + "learning_rate": 0.0001, + "loss": 81.5968, + "step": 650 + }, + { + "epoch": 10.65040650406504, + "grad_norm": 1.8254153728485107, + "learning_rate": 0.0001, + "loss": 78.9589, + "step": 660 + }, + { + "epoch": 10.8130081300813, + "grad_norm": 3.2015740871429443, + "learning_rate": 0.0001, + "loss": 79.1829, + "step": 670 + }, + { + "epoch": 10.975609756097562, + "grad_norm": 1.788453459739685, + "learning_rate": 0.0001, + "loss": 81.527, + "step": 680 + }, + { + "epoch": 11.130081300813009, + "grad_norm": 1.5744423866271973, + "learning_rate": 0.0001, + "loss": 74.9869, + "step": 690 + }, + { + "epoch": 11.292682926829269, + "grad_norm": 2.006002187728882, + "learning_rate": 0.0001, + "loss": 80.2677, + "step": 700 + }, + { + "epoch": 11.455284552845528, + "grad_norm": 1.8847148418426514, + "learning_rate": 0.0001, + "loss": 78.8709, + "step": 710 + }, + { + "epoch": 11.617886178861788, + "grad_norm": 1.8384478092193604, + "learning_rate": 0.0001, + "loss": 79.008, + "step": 720 + }, + { + "epoch": 11.78048780487805, + "grad_norm": 1.9486687183380127, + "learning_rate": 0.0001, + "loss": 76.1525, + "step": 730 + }, + { + "epoch": 11.94308943089431, + "grad_norm": 1.8644076585769653, + "learning_rate": 0.0001, + "loss": 77.5802, + "step": 740 + }, + { + "epoch": 12.097560975609756, + "grad_norm": 1.6035499572753906, + "learning_rate": 0.0001, + "loss": 75.2422, + "step": 750 + }, + { + "epoch": 12.260162601626016, + "grad_norm": 1.6935622692108154, + "learning_rate": 0.0001, + "loss": 77.5838, + "step": 760 + }, + { + "epoch": 12.422764227642276, + "grad_norm": 1.9327987432479858, + "learning_rate": 0.0001, + "loss": 78.3237, + "step": 770 + }, + { + "epoch": 12.585365853658537, + "grad_norm": 1.7903852462768555, + "learning_rate": 0.0001, + "loss": 78.7459, + "step": 780 + }, + { + "epoch": 12.747967479674797, + "grad_norm": 1.4962575435638428, + "learning_rate": 0.0001, + "loss": 78.9193, + "step": 790 + }, + { + "epoch": 12.910569105691057, + "grad_norm": 42.10435104370117, + "learning_rate": 0.0001, + "loss": 80.1429, + "step": 800 + }, + { + "epoch": 13.065040650406504, + "grad_norm": 1.904065728187561, + "learning_rate": 0.0001, + "loss": 73.7798, + "step": 810 + }, + { + "epoch": 13.227642276422765, + "grad_norm": 1.8768481016159058, + "learning_rate": 0.0001, + "loss": 76.8692, + "step": 820 + }, + { + "epoch": 13.390243902439025, + "grad_norm": 16.140565872192383, + "learning_rate": 0.0001, + "loss": 78.122, + "step": 830 + }, + { + "epoch": 13.552845528455284, + "grad_norm": 2.160288095474243, + "learning_rate": 0.0001, + "loss": 76.2169, + "step": 840 + }, + { + "epoch": 13.715447154471544, + "grad_norm": 27.682231903076172, + "learning_rate": 0.0001, + "loss": 76.6266, + "step": 850 + }, + { + "epoch": 13.878048780487806, + "grad_norm": 1.779772162437439, + "learning_rate": 0.0001, + "loss": 80.0434, + "step": 860 + }, + { + "epoch": 14.032520325203253, + "grad_norm": 1.6516485214233398, + "learning_rate": 0.0001, + "loss": 73.3943, + "step": 870 + }, + { + "epoch": 14.195121951219512, + "grad_norm": 1.846498966217041, + "learning_rate": 0.0001, + "loss": 77.5157, + "step": 880 + }, + { + "epoch": 14.357723577235772, + "grad_norm": 1.942946434020996, + "learning_rate": 0.0001, + "loss": 77.6944, + "step": 890 + }, + { + "epoch": 14.520325203252032, + "grad_norm": 2.0929224491119385, + "learning_rate": 0.0001, + "loss": 76.9411, + "step": 900 + }, + { + "epoch": 14.682926829268293, + "grad_norm": 3.4498767852783203, + "learning_rate": 0.0001, + "loss": 74.9234, + "step": 910 + }, + { + "epoch": 14.845528455284553, + "grad_norm": 1.6880728006362915, + "learning_rate": 0.0001, + "loss": 75.3617, + "step": 920 + }, + { + "epoch": 15.0, + "grad_norm": 1.8403468132019043, + "learning_rate": 0.0001, + "loss": 73.2543, + "step": 930 + }, + { + "epoch": 15.16260162601626, + "grad_norm": 1.4368852376937866, + "learning_rate": 0.0001, + "loss": 76.6271, + "step": 940 + }, + { + "epoch": 15.32520325203252, + "grad_norm": 2.9467780590057373, + "learning_rate": 0.0001, + "loss": 75.8744, + "step": 950 + }, + { + "epoch": 15.487804878048781, + "grad_norm": 1.567134141921997, + "learning_rate": 0.0001, + "loss": 76.6629, + "step": 960 + }, + { + "epoch": 15.65040650406504, + "grad_norm": 1.4503871202468872, + "learning_rate": 0.0001, + "loss": 76.3525, + "step": 970 + }, + { + "epoch": 15.8130081300813, + "grad_norm": 1.8871147632598877, + "learning_rate": 0.0001, + "loss": 77.4684, + "step": 980 + }, + { + "epoch": 15.975609756097562, + "grad_norm": 1.6707026958465576, + "learning_rate": 0.0001, + "loss": 76.3063, + "step": 990 + }, + { + "epoch": 16.130081300813007, + "grad_norm": 3.6300716400146484, + "learning_rate": 0.0001, + "loss": 73.0586, + "step": 1000 + }, + { + "epoch": 16.29268292682927, + "grad_norm": 1.6669234037399292, + "learning_rate": 0.0001, + "loss": 75.8027, + "step": 1010 + }, + { + "epoch": 16.45528455284553, + "grad_norm": 1.6920552253723145, + "learning_rate": 0.0001, + "loss": 74.7252, + "step": 1020 + }, + { + "epoch": 16.617886178861788, + "grad_norm": 1.8184431791305542, + "learning_rate": 0.0001, + "loss": 75.3625, + "step": 1030 + }, + { + "epoch": 16.78048780487805, + "grad_norm": 1.5828496217727661, + "learning_rate": 0.0001, + "loss": 78.0302, + "step": 1040 + }, + { + "epoch": 16.943089430894307, + "grad_norm": 1.7372915744781494, + "learning_rate": 0.0001, + "loss": 78.6949, + "step": 1050 + }, + { + "epoch": 17.097560975609756, + "grad_norm": 1.837161898612976, + "learning_rate": 0.0001, + "loss": 74.4726, + "step": 1060 + }, + { + "epoch": 17.260162601626018, + "grad_norm": 3.8733808994293213, + "learning_rate": 0.0001, + "loss": 76.1711, + "step": 1070 + }, + { + "epoch": 17.422764227642276, + "grad_norm": 1.5907851457595825, + "learning_rate": 0.0001, + "loss": 75.8292, + "step": 1080 + }, + { + "epoch": 17.585365853658537, + "grad_norm": 1.8271747827529907, + "learning_rate": 0.0001, + "loss": 74.5316, + "step": 1090 + }, + { + "epoch": 17.747967479674795, + "grad_norm": 1.827980875968933, + "learning_rate": 0.0001, + "loss": 74.7281, + "step": 1100 + }, + { + "epoch": 17.910569105691057, + "grad_norm": 1.4184074401855469, + "learning_rate": 0.0001, + "loss": 75.736, + "step": 1110 + }, + { + "epoch": 18.065040650406505, + "grad_norm": 2.3459699153900146, + "learning_rate": 0.0001, + "loss": 71.9586, + "step": 1120 + }, + { + "epoch": 18.227642276422763, + "grad_norm": 47.673641204833984, + "learning_rate": 0.0001, + "loss": 75.6649, + "step": 1130 + }, + { + "epoch": 18.390243902439025, + "grad_norm": 1.677398443222046, + "learning_rate": 0.0001, + "loss": 76.0574, + "step": 1140 + }, + { + "epoch": 18.552845528455286, + "grad_norm": 1.4180850982666016, + "learning_rate": 0.0001, + "loss": 74.3077, + "step": 1150 + }, + { + "epoch": 18.715447154471544, + "grad_norm": 2.0041465759277344, + "learning_rate": 0.0001, + "loss": 73.5916, + "step": 1160 + }, + { + "epoch": 18.878048780487806, + "grad_norm": 1.9098238945007324, + "learning_rate": 0.0001, + "loss": 74.5002, + "step": 1170 + }, + { + "epoch": 19.03252032520325, + "grad_norm": 1.405555248260498, + "learning_rate": 0.0001, + "loss": 69.8115, + "step": 1180 + }, + { + "epoch": 19.195121951219512, + "grad_norm": 1.424198865890503, + "learning_rate": 0.0001, + "loss": 73.7289, + "step": 1190 + }, + { + "epoch": 19.357723577235774, + "grad_norm": 7.542550563812256, + "learning_rate": 0.0001, + "loss": 73.7696, + "step": 1200 + }, + { + "epoch": 19.520325203252032, + "grad_norm": 1.425329566001892, + "learning_rate": 0.0001, + "loss": 72.1261, + "step": 1210 + }, + { + "epoch": 19.682926829268293, + "grad_norm": 1.525724172592163, + "learning_rate": 0.0001, + "loss": 73.6578, + "step": 1220 + }, + { + "epoch": 19.84552845528455, + "grad_norm": 1.6387193202972412, + "learning_rate": 0.0001, + "loss": 74.3027, + "step": 1230 + }, + { + "epoch": 20.0, + "grad_norm": 1.8429840803146362, + "learning_rate": 0.0001, + "loss": 69.5056, + "step": 1240 + }, + { + "epoch": 20.16260162601626, + "grad_norm": 2.5166006088256836, + "learning_rate": 0.0001, + "loss": 73.5965, + "step": 1250 + }, + { + "epoch": 20.32520325203252, + "grad_norm": 1.6006672382354736, + "learning_rate": 0.0001, + "loss": 72.4581, + "step": 1260 + }, + { + "epoch": 20.48780487804878, + "grad_norm": 1.6722878217697144, + "learning_rate": 0.0001, + "loss": 72.3097, + "step": 1270 + }, + { + "epoch": 20.65040650406504, + "grad_norm": 1.8715728521347046, + "learning_rate": 0.0001, + "loss": 73.4752, + "step": 1280 + }, + { + "epoch": 20.8130081300813, + "grad_norm": 1.3280900716781616, + "learning_rate": 0.0001, + "loss": 73.2145, + "step": 1290 + }, + { + "epoch": 20.975609756097562, + "grad_norm": 1.941451907157898, + "learning_rate": 0.0001, + "loss": 73.2085, + "step": 1300 + }, + { + "epoch": 21.130081300813007, + "grad_norm": 4.277497291564941, + "learning_rate": 0.0001, + "loss": 68.8305, + "step": 1310 + }, + { + "epoch": 21.29268292682927, + "grad_norm": 1.9949846267700195, + "learning_rate": 0.0001, + "loss": 72.6689, + "step": 1320 + }, + { + "epoch": 21.45528455284553, + "grad_norm": 1.524044394493103, + "learning_rate": 0.0001, + "loss": 72.9705, + "step": 1330 + }, + { + "epoch": 21.617886178861788, + "grad_norm": 1.384194016456604, + "learning_rate": 0.0001, + "loss": 72.5465, + "step": 1340 + }, + { + "epoch": 21.78048780487805, + "grad_norm": 1.724225640296936, + "learning_rate": 0.0001, + "loss": 72.9203, + "step": 1350 + }, + { + "epoch": 21.943089430894307, + "grad_norm": 2.0856704711914062, + "learning_rate": 0.0001, + "loss": 72.4034, + "step": 1360 + }, + { + "epoch": 22.097560975609756, + "grad_norm": 5.128044128417969, + "learning_rate": 0.0001, + "loss": 68.7754, + "step": 1370 + }, + { + "epoch": 22.260162601626018, + "grad_norm": 1.7385634183883667, + "learning_rate": 0.0001, + "loss": 73.9349, + "step": 1380 + }, + { + "epoch": 22.422764227642276, + "grad_norm": 1.3946012258529663, + "learning_rate": 0.0001, + "loss": 70.9627, + "step": 1390 + }, + { + "epoch": 22.585365853658537, + "grad_norm": 1.285414457321167, + "learning_rate": 0.0001, + "loss": 72.4578, + "step": 1400 + }, + { + "epoch": 22.747967479674795, + "grad_norm": 1.4197710752487183, + "learning_rate": 0.0001, + "loss": 72.5252, + "step": 1410 + }, + { + "epoch": 22.910569105691057, + "grad_norm": 3.048942804336548, + "learning_rate": 0.0001, + "loss": 71.1107, + "step": 1420 + }, + { + "epoch": 23.065040650406505, + "grad_norm": 1.3114765882492065, + "learning_rate": 0.0001, + "loss": 68.2725, + "step": 1430 + }, + { + "epoch": 23.227642276422763, + "grad_norm": 1.2763880491256714, + "learning_rate": 0.0001, + "loss": 71.5783, + "step": 1440 + }, + { + "epoch": 23.390243902439025, + "grad_norm": 1.3913089036941528, + "learning_rate": 0.0001, + "loss": 72.5986, + "step": 1450 + }, + { + "epoch": 23.552845528455286, + "grad_norm": 1.628751516342163, + "learning_rate": 0.0001, + "loss": 71.8383, + "step": 1460 + }, + { + "epoch": 23.715447154471544, + "grad_norm": 1.4331274032592773, + "learning_rate": 0.0001, + "loss": 70.8051, + "step": 1470 + }, + { + "epoch": 23.878048780487806, + "grad_norm": 1.4154727458953857, + "learning_rate": 0.0001, + "loss": 71.3283, + "step": 1480 + }, + { + "epoch": 24.03252032520325, + "grad_norm": 1.7208784818649292, + "learning_rate": 0.0001, + "loss": 68.4719, + "step": 1490 + }, + { + "epoch": 24.195121951219512, + "grad_norm": 1.466123342514038, + "learning_rate": 0.0001, + "loss": 71.9597, + "step": 1500 + }, + { + "epoch": 24.357723577235774, + "grad_norm": 1.4823886156082153, + "learning_rate": 0.0001, + "loss": 72.4533, + "step": 1510 + }, + { + "epoch": 24.520325203252032, + "grad_norm": 1.8407355546951294, + "learning_rate": 0.0001, + "loss": 71.1468, + "step": 1520 + }, + { + "epoch": 24.682926829268293, + "grad_norm": 1.630565881729126, + "learning_rate": 0.0001, + "loss": 71.5192, + "step": 1530 + }, + { + "epoch": 24.84552845528455, + "grad_norm": 1.5248785018920898, + "learning_rate": 0.0001, + "loss": 68.4804, + "step": 1540 + }, + { + "epoch": 25.0, + "grad_norm": 1.4690219163894653, + "learning_rate": 0.0001, + "loss": 67.5716, + "step": 1550 + }, + { + "epoch": 25.16260162601626, + "grad_norm": 2.0246646404266357, + "learning_rate": 0.0001, + "loss": 71.2554, + "step": 1560 + }, + { + "epoch": 25.32520325203252, + "grad_norm": 1.3611458539962769, + "learning_rate": 0.0001, + "loss": 69.8558, + "step": 1570 + }, + { + "epoch": 25.48780487804878, + "grad_norm": 2.0006039142608643, + "learning_rate": 0.0001, + "loss": 72.5344, + "step": 1580 + }, + { + "epoch": 25.65040650406504, + "grad_norm": 1.7373167276382446, + "learning_rate": 0.0001, + "loss": 70.6253, + "step": 1590 + }, + { + "epoch": 25.8130081300813, + "grad_norm": 1.2878378629684448, + "learning_rate": 0.0001, + "loss": 72.165, + "step": 1600 + }, + { + "epoch": 25.975609756097562, + "grad_norm": 1.5337013006210327, + "learning_rate": 0.0001, + "loss": 70.6989, + "step": 1610 + }, + { + "epoch": 26.130081300813007, + "grad_norm": 1.364615559577942, + "learning_rate": 0.0001, + "loss": 67.3898, + "step": 1620 + }, + { + "epoch": 26.29268292682927, + "grad_norm": 1.3249316215515137, + "learning_rate": 0.0001, + "loss": 72.7102, + "step": 1630 + }, + { + "epoch": 26.45528455284553, + "grad_norm": 4.209898471832275, + "learning_rate": 0.0001, + "loss": 70.3185, + "step": 1640 + }, + { + "epoch": 26.617886178861788, + "grad_norm": 1.5488810539245605, + "learning_rate": 0.0001, + "loss": 72.3678, + "step": 1650 + }, + { + "epoch": 26.78048780487805, + "grad_norm": 1.7743597030639648, + "learning_rate": 0.0001, + "loss": 71.0917, + "step": 1660 + }, + { + "epoch": 26.943089430894307, + "grad_norm": 1.5433533191680908, + "learning_rate": 0.0001, + "loss": 71.4001, + "step": 1670 + }, + { + "epoch": 27.097560975609756, + "grad_norm": 1.6320385932922363, + "learning_rate": 0.0001, + "loss": 66.7441, + "step": 1680 + }, + { + "epoch": 27.260162601626018, + "grad_norm": 1.3137969970703125, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 1690 + }, + { + "epoch": 27.422764227642276, + "grad_norm": 1.9150484800338745, + "learning_rate": 0.0001, + "loss": 69.8956, + "step": 1700 + }, + { + "epoch": 27.585365853658537, + "grad_norm": 1.4442802667617798, + "learning_rate": 0.0001, + "loss": 70.7185, + "step": 1710 + }, + { + "epoch": 27.747967479674795, + "grad_norm": 2.572497606277466, + "learning_rate": 0.0001, + "loss": 70.8934, + "step": 1720 + }, + { + "epoch": 27.910569105691057, + "grad_norm": 1.6669617891311646, + "learning_rate": 0.0001, + "loss": 71.8221, + "step": 1730 + }, + { + "epoch": 28.065040650406505, + "grad_norm": 1.8449876308441162, + "learning_rate": 0.0001, + "loss": 66.3923, + "step": 1740 + }, + { + "epoch": 28.227642276422763, + "grad_norm": 1.6600004434585571, + "learning_rate": 0.0001, + "loss": 69.5301, + "step": 1750 + }, + { + "epoch": 28.390243902439025, + "grad_norm": 1.4655731916427612, + "learning_rate": 0.0001, + "loss": 72.1924, + "step": 1760 + }, + { + "epoch": 28.552845528455286, + "grad_norm": 1.134505033493042, + "learning_rate": 0.0001, + "loss": 69.5855, + "step": 1770 + }, + { + "epoch": 28.715447154471544, + "grad_norm": 1.301984190940857, + "learning_rate": 0.0001, + "loss": 70.7999, + "step": 1780 + }, + { + "epoch": 28.878048780487806, + "grad_norm": 1.5641013383865356, + "learning_rate": 0.0001, + "loss": 69.9091, + "step": 1790 + }, + { + "epoch": 29.03252032520325, + "grad_norm": 1.4850797653198242, + "learning_rate": 0.0001, + "loss": 66.6266, + "step": 1800 + }, + { + "epoch": 29.195121951219512, + "grad_norm": 1.2069118022918701, + "learning_rate": 0.0001, + "loss": 69.3802, + "step": 1810 + }, + { + "epoch": 29.357723577235774, + "grad_norm": 1.2890352010726929, + "learning_rate": 0.0001, + "loss": 68.5044, + "step": 1820 + }, + { + "epoch": 29.520325203252032, + "grad_norm": 1.4150917530059814, + "learning_rate": 0.0001, + "loss": 69.807, + "step": 1830 + }, + { + "epoch": 29.682926829268293, + "grad_norm": 1.514807939529419, + "learning_rate": 0.0001, + "loss": 71.3858, + "step": 1840 + }, + { + "epoch": 29.84552845528455, + "grad_norm": 1.4246330261230469, + "learning_rate": 0.0001, + "loss": 70.494, + "step": 1850 + }, + { + "epoch": 30.0, + "grad_norm": 1.1789597272872925, + "learning_rate": 0.0001, + "loss": 67.1056, + "step": 1860 + }, + { + "epoch": 30.16260162601626, + "grad_norm": 1.23678719997406, + "learning_rate": 0.0001, + "loss": 70.336, + "step": 1870 + }, + { + "epoch": 30.32520325203252, + "grad_norm": 1.1756274700164795, + "learning_rate": 0.0001, + "loss": 67.9909, + "step": 1880 + }, + { + "epoch": 30.48780487804878, + "grad_norm": 1.4175690412521362, + "learning_rate": 0.0001, + "loss": 69.2617, + "step": 1890 + }, + { + "epoch": 30.65040650406504, + "grad_norm": 1.3084465265274048, + "learning_rate": 0.0001, + "loss": 70.2963, + "step": 1900 + }, + { + "epoch": 30.8130081300813, + "grad_norm": 1.486464500427246, + "learning_rate": 0.0001, + "loss": 68.6317, + "step": 1910 + }, + { + "epoch": 30.975609756097562, + "grad_norm": 10.415762901306152, + "learning_rate": 0.0001, + "loss": 69.7767, + "step": 1920 + }, + { + "epoch": 31.130081300813007, + "grad_norm": 1.4484038352966309, + "learning_rate": 0.0001, + "loss": 65.6345, + "step": 1930 + }, + { + "epoch": 31.29268292682927, + "grad_norm": 2.297306537628174, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 1940 + }, + { + "epoch": 31.45528455284553, + "grad_norm": 1.223490595817566, + "learning_rate": 0.0001, + "loss": 69.2139, + "step": 1950 + }, + { + "epoch": 31.617886178861788, + "grad_norm": 1.2144125699996948, + "learning_rate": 0.0001, + "loss": 70.3061, + "step": 1960 + }, + { + "epoch": 31.78048780487805, + "grad_norm": 2.633528470993042, + "learning_rate": 0.0001, + "loss": 68.7096, + "step": 1970 + }, + { + "epoch": 31.943089430894307, + "grad_norm": 1.5048350095748901, + "learning_rate": 0.0001, + "loss": 68.6711, + "step": 1980 + }, + { + "epoch": 32.09756097560975, + "grad_norm": 1.5154467821121216, + "learning_rate": 0.0001, + "loss": 66.7989, + "step": 1990 + }, + { + "epoch": 32.260162601626014, + "grad_norm": 2.721277952194214, + "learning_rate": 0.0001, + "loss": 70.4987, + "step": 2000 + }, + { + "epoch": 32.422764227642276, + "grad_norm": 1.44464910030365, + "learning_rate": 0.0001, + "loss": 69.9712, + "step": 2010 + }, + { + "epoch": 32.58536585365854, + "grad_norm": 1.3462835550308228, + "learning_rate": 0.0001, + "loss": 68.9381, + "step": 2020 + }, + { + "epoch": 32.7479674796748, + "grad_norm": 1.3283029794692993, + "learning_rate": 0.0001, + "loss": 68.5301, + "step": 2030 + }, + { + "epoch": 32.91056910569106, + "grad_norm": 1.5172960758209229, + "learning_rate": 0.0001, + "loss": 70.1894, + "step": 2040 + }, + { + "epoch": 33.0650406504065, + "grad_norm": 1.1872576475143433, + "learning_rate": 0.0001, + "loss": 65.0158, + "step": 2050 + }, + { + "epoch": 33.22764227642276, + "grad_norm": 1.5414419174194336, + "learning_rate": 0.0001, + "loss": 69.1537, + "step": 2060 + }, + { + "epoch": 33.390243902439025, + "grad_norm": 1.3143430948257446, + "learning_rate": 0.0001, + "loss": 68.6792, + "step": 2070 + }, + { + "epoch": 33.552845528455286, + "grad_norm": 1.3863723278045654, + "learning_rate": 0.0001, + "loss": 70.3867, + "step": 2080 + }, + { + "epoch": 33.71544715447155, + "grad_norm": 1.4060479402542114, + "learning_rate": 0.0001, + "loss": 68.4981, + "step": 2090 + }, + { + "epoch": 33.8780487804878, + "grad_norm": 1.34132981300354, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 2100 + }, + { + "epoch": 34.03252032520325, + "grad_norm": 1.8303673267364502, + "learning_rate": 0.0001, + "loss": 65.819, + "step": 2110 + }, + { + "epoch": 34.19512195121951, + "grad_norm": 1.387339472770691, + "learning_rate": 0.0001, + "loss": 69.9825, + "step": 2120 + }, + { + "epoch": 34.357723577235774, + "grad_norm": 1.4125195741653442, + "learning_rate": 0.0001, + "loss": 67.5149, + "step": 2130 + }, + { + "epoch": 34.520325203252035, + "grad_norm": 1.2909412384033203, + "learning_rate": 0.0001, + "loss": 69.7848, + "step": 2140 + }, + { + "epoch": 34.68292682926829, + "grad_norm": 1.8860795497894287, + "learning_rate": 0.0001, + "loss": 68.6145, + "step": 2150 + }, + { + "epoch": 34.84552845528455, + "grad_norm": 1.5111079216003418, + "learning_rate": 0.0001, + "loss": 68.2612, + "step": 2160 + }, + { + "epoch": 35.0, + "grad_norm": 1.3275264501571655, + "learning_rate": 0.0001, + "loss": 64.0298, + "step": 2170 + }, + { + "epoch": 35.16260162601626, + "grad_norm": 1.174864411354065, + "learning_rate": 0.0001, + "loss": 66.6153, + "step": 2180 + }, + { + "epoch": 35.32520325203252, + "grad_norm": 1.6873915195465088, + "learning_rate": 0.0001, + "loss": 69.6278, + "step": 2190 + }, + { + "epoch": 35.48780487804878, + "grad_norm": 2.4457507133483887, + "learning_rate": 0.0001, + "loss": 68.3611, + "step": 2200 + }, + { + "epoch": 35.65040650406504, + "grad_norm": 1.4779958724975586, + "learning_rate": 0.0001, + "loss": 69.2609, + "step": 2210 + }, + { + "epoch": 35.8130081300813, + "grad_norm": 2.273709774017334, + "learning_rate": 0.0001, + "loss": 69.1312, + "step": 2220 + }, + { + "epoch": 35.97560975609756, + "grad_norm": 1.5741862058639526, + "learning_rate": 0.0001, + "loss": 69.1191, + "step": 2230 + }, + { + "epoch": 36.13008130081301, + "grad_norm": 1.2730036973953247, + "learning_rate": 0.0001, + "loss": 65.692, + "step": 2240 + }, + { + "epoch": 36.292682926829265, + "grad_norm": 1.2936429977416992, + "learning_rate": 0.0001, + "loss": 68.2962, + "step": 2250 + }, + { + "epoch": 36.45528455284553, + "grad_norm": 1.354279637336731, + "learning_rate": 0.0001, + "loss": 70.5507, + "step": 2260 + }, + { + "epoch": 36.61788617886179, + "grad_norm": 1.4165853261947632, + "learning_rate": 0.0001, + "loss": 68.0048, + "step": 2270 + }, + { + "epoch": 36.78048780487805, + "grad_norm": 1.2992445230484009, + "learning_rate": 0.0001, + "loss": 67.7072, + "step": 2280 + }, + { + "epoch": 36.94308943089431, + "grad_norm": 1.5053461790084839, + "learning_rate": 0.0001, + "loss": 66.7587, + "step": 2290 + }, + { + "epoch": 37.09756097560975, + "grad_norm": 1.337390661239624, + "learning_rate": 0.0001, + "loss": 63.786, + "step": 2300 + }, + { + "epoch": 37.260162601626014, + "grad_norm": 1.0407774448394775, + "learning_rate": 0.0001, + "loss": 66.6253, + "step": 2310 + }, + { + "epoch": 37.422764227642276, + "grad_norm": 1.564152479171753, + "learning_rate": 0.0001, + "loss": 68.3606, + "step": 2320 + }, + { + "epoch": 37.58536585365854, + "grad_norm": 1.2028454542160034, + "learning_rate": 0.0001, + "loss": 66.2841, + "step": 2330 + }, + { + "epoch": 37.7479674796748, + "grad_norm": 1.4528603553771973, + "learning_rate": 0.0001, + "loss": 67.4277, + "step": 2340 + }, + { + "epoch": 37.91056910569106, + "grad_norm": 1.4954335689544678, + "learning_rate": 0.0001, + "loss": 68.9954, + "step": 2350 + }, + { + "epoch": 38.0650406504065, + "grad_norm": 1.2820416688919067, + "learning_rate": 0.0001, + "loss": 65.2449, + "step": 2360 + }, + { + "epoch": 38.22764227642276, + "grad_norm": 1.2392579317092896, + "learning_rate": 0.0001, + "loss": 67.5006, + "step": 2370 + }, + { + "epoch": 38.390243902439025, + "grad_norm": 1.2135238647460938, + "learning_rate": 0.0001, + "loss": 69.2214, + "step": 2380 + }, + { + "epoch": 38.552845528455286, + "grad_norm": 1.5964744091033936, + "learning_rate": 0.0001, + "loss": 67.934, + "step": 2390 + }, + { + "epoch": 38.71544715447155, + "grad_norm": 1.2841702699661255, + "learning_rate": 0.0001, + "loss": 67.0848, + "step": 2400 + }, + { + "epoch": 38.8780487804878, + "grad_norm": 1.8026081323623657, + "learning_rate": 0.0001, + "loss": 67.5358, + "step": 2410 + }, + { + "epoch": 39.03252032520325, + "grad_norm": 1.4178807735443115, + "learning_rate": 0.0001, + "loss": 64.771, + "step": 2420 + }, + { + "epoch": 39.19512195121951, + "grad_norm": 1.3671854734420776, + "learning_rate": 0.0001, + "loss": 68.0531, + "step": 2430 + }, + { + "epoch": 39.357723577235774, + "grad_norm": 1.43596613407135, + "learning_rate": 0.0001, + "loss": 67.7735, + "step": 2440 + }, + { + "epoch": 39.520325203252035, + "grad_norm": 1.464250922203064, + "learning_rate": 0.0001, + "loss": 67.8331, + "step": 2450 + }, + { + "epoch": 39.68292682926829, + "grad_norm": 3.101461410522461, + "learning_rate": 0.0001, + "loss": 65.8956, + "step": 2460 + }, + { + "epoch": 39.84552845528455, + "grad_norm": 1.8332289457321167, + "learning_rate": 0.0001, + "loss": 66.8218, + "step": 2470 + }, + { + "epoch": 40.0, + "grad_norm": 1.2660984992980957, + "learning_rate": 0.0001, + "loss": 63.9969, + "step": 2480 + }, + { + "epoch": 40.16260162601626, + "grad_norm": 2.0309898853302, + "learning_rate": 0.0001, + "loss": 67.3538, + "step": 2490 + }, + { + "epoch": 40.32520325203252, + "grad_norm": 1.207000494003296, + "learning_rate": 0.0001, + "loss": 68.0898, + "step": 2500 + }, + { + "epoch": 40.48780487804878, + "grad_norm": 1.3609018325805664, + "learning_rate": 0.0001, + "loss": 67.0381, + "step": 2510 + }, + { + "epoch": 40.65040650406504, + "grad_norm": 1.310826301574707, + "learning_rate": 0.0001, + "loss": 69.0267, + "step": 2520 + }, + { + "epoch": 40.8130081300813, + "grad_norm": 1.4150960445404053, + "learning_rate": 0.0001, + "loss": 69.1934, + "step": 2530 + }, + { + "epoch": 40.97560975609756, + "grad_norm": 1.5414319038391113, + "learning_rate": 0.0001, + "loss": 68.0604, + "step": 2540 + }, + { + "epoch": 41.13008130081301, + "grad_norm": 2.3246140480041504, + "learning_rate": 0.0001, + "loss": 61.5698, + "step": 2550 + }, + { + "epoch": 41.292682926829265, + "grad_norm": 1.206296443939209, + "learning_rate": 0.0001, + "loss": 64.9898, + "step": 2560 + }, + { + "epoch": 41.45528455284553, + "grad_norm": 1.6080626249313354, + "learning_rate": 0.0001, + "loss": 68.2685, + "step": 2570 + }, + { + "epoch": 41.61788617886179, + "grad_norm": 1.2404112815856934, + "learning_rate": 0.0001, + "loss": 67.8457, + "step": 2580 + }, + { + "epoch": 41.78048780487805, + "grad_norm": 1.2304514646530151, + "learning_rate": 0.0001, + "loss": 67.9052, + "step": 2590 + }, + { + "epoch": 41.94308943089431, + "grad_norm": 1.5806187391281128, + "learning_rate": 0.0001, + "loss": 66.0663, + "step": 2600 + }, + { + "epoch": 42.09756097560975, + "grad_norm": 1.2564151287078857, + "learning_rate": 0.0001, + "loss": 65.7763, + "step": 2610 + }, + { + "epoch": 42.260162601626014, + "grad_norm": 1.4972114562988281, + "learning_rate": 0.0001, + "loss": 66.4269, + "step": 2620 + }, + { + "epoch": 42.422764227642276, + "grad_norm": 1.2913843393325806, + "learning_rate": 0.0001, + "loss": 67.7554, + "step": 2630 + }, + { + "epoch": 42.58536585365854, + "grad_norm": 1.3743863105773926, + "learning_rate": 0.0001, + "loss": 67.2627, + "step": 2640 + }, + { + "epoch": 42.7479674796748, + "grad_norm": 1.1267883777618408, + "learning_rate": 0.0001, + "loss": 67.4597, + "step": 2650 + }, + { + "epoch": 42.91056910569106, + "grad_norm": 2.0194661617279053, + "learning_rate": 0.0001, + "loss": 66.9295, + "step": 2660 + }, + { + "epoch": 43.0650406504065, + "grad_norm": 1.273779273033142, + "learning_rate": 0.0001, + "loss": 60.7906, + "step": 2670 + }, + { + "epoch": 43.22764227642276, + "grad_norm": 1.1498609781265259, + "learning_rate": 0.0001, + "loss": 68.0405, + "step": 2680 + }, + { + "epoch": 43.390243902439025, + "grad_norm": 1.4689130783081055, + "learning_rate": 0.0001, + "loss": 66.2358, + "step": 2690 + }, + { + "epoch": 43.552845528455286, + "grad_norm": 1.4495564699172974, + "learning_rate": 0.0001, + "loss": 67.4267, + "step": 2700 + }, + { + "epoch": 43.71544715447155, + "grad_norm": 2.5769612789154053, + "learning_rate": 0.0001, + "loss": 66.953, + "step": 2710 + }, + { + "epoch": 43.8780487804878, + "grad_norm": 2.1684205532073975, + "learning_rate": 0.0001, + "loss": 65.6393, + "step": 2720 + }, + { + "epoch": 44.03252032520325, + "grad_norm": 1.4968472719192505, + "learning_rate": 0.0001, + "loss": 60.8256, + "step": 2730 + }, + { + "epoch": 44.19512195121951, + "grad_norm": 1.1874353885650635, + "learning_rate": 0.0001, + "loss": 66.818, + "step": 2740 + }, + { + "epoch": 44.357723577235774, + "grad_norm": 2.271235227584839, + "learning_rate": 0.0001, + "loss": 66.5048, + "step": 2750 + }, + { + "epoch": 44.520325203252035, + "grad_norm": 1.20489501953125, + "learning_rate": 0.0001, + "loss": 67.0487, + "step": 2760 + }, + { + "epoch": 44.68292682926829, + "grad_norm": 1.6273691654205322, + "learning_rate": 0.0001, + "loss": 66.2228, + "step": 2770 + }, + { + "epoch": 44.84552845528455, + "grad_norm": 1.4935433864593506, + "learning_rate": 0.0001, + "loss": 67.2039, + "step": 2780 + }, + { + "epoch": 45.0, + "grad_norm": 1.2668659687042236, + "learning_rate": 0.0001, + "loss": 62.7855, + "step": 2790 + }, + { + "epoch": 45.16260162601626, + "grad_norm": 1.6439573764801025, + "learning_rate": 0.0001, + "loss": 67.919, + "step": 2800 + }, + { + "epoch": 45.32520325203252, + "grad_norm": 1.311854600906372, + "learning_rate": 0.0001, + "loss": 65.1797, + "step": 2810 + }, + { + "epoch": 45.48780487804878, + "grad_norm": 1.3596776723861694, + "learning_rate": 0.0001, + "loss": 67.469, + "step": 2820 + }, + { + "epoch": 45.65040650406504, + "grad_norm": 1.5618443489074707, + "learning_rate": 0.0001, + "loss": 66.1808, + "step": 2830 + }, + { + "epoch": 45.8130081300813, + "grad_norm": 1.7551205158233643, + "learning_rate": 0.0001, + "loss": 65.3803, + "step": 2840 + }, + { + "epoch": 45.97560975609756, + "grad_norm": 1.433794379234314, + "learning_rate": 0.0001, + "loss": 67.8314, + "step": 2850 + }, + { + "epoch": 46.13008130081301, + "grad_norm": 1.4634851217269897, + "learning_rate": 0.0001, + "loss": 63.4219, + "step": 2860 + }, + { + "epoch": 46.292682926829265, + "grad_norm": 1.083052396774292, + "learning_rate": 0.0001, + "loss": 68.0719, + "step": 2870 + }, + { + "epoch": 46.45528455284553, + "grad_norm": 1.1784703731536865, + "learning_rate": 0.0001, + "loss": 66.1391, + "step": 2880 + }, + { + "epoch": 46.61788617886179, + "grad_norm": 1.2552465200424194, + "learning_rate": 0.0001, + "loss": 65.6006, + "step": 2890 + }, + { + "epoch": 46.78048780487805, + "grad_norm": 1.4684522151947021, + "learning_rate": 0.0001, + "loss": 66.8314, + "step": 2900 + }, + { + "epoch": 46.94308943089431, + "grad_norm": 1.741514801979065, + "learning_rate": 0.0001, + "loss": 67.5056, + "step": 2910 + }, + { + "epoch": 47.09756097560975, + "grad_norm": 5.696534156799316, + "learning_rate": 0.0001, + "loss": 62.8016, + "step": 2920 + }, + { + "epoch": 47.260162601626014, + "grad_norm": 1.5000969171524048, + "learning_rate": 0.0001, + "loss": 68.0272, + "step": 2930 + }, + { + "epoch": 47.422764227642276, + "grad_norm": 1.1044933795928955, + "learning_rate": 0.0001, + "loss": 67.0905, + "step": 2940 + }, + { + "epoch": 47.58536585365854, + "grad_norm": 0.9933589100837708, + "learning_rate": 0.0001, + "loss": 67.6077, + "step": 2950 + }, + { + "epoch": 47.7479674796748, + "grad_norm": 1.2908599376678467, + "learning_rate": 0.0001, + "loss": 65.869, + "step": 2960 + }, + { + "epoch": 47.91056910569106, + "grad_norm": 1.0470529794692993, + "learning_rate": 0.0001, + "loss": 66.5353, + "step": 2970 + }, + { + "epoch": 48.0650406504065, + "grad_norm": 1.0927976369857788, + "learning_rate": 0.0001, + "loss": 62.7541, + "step": 2980 + }, + { + "epoch": 48.22764227642276, + "grad_norm": 1.0870513916015625, + "learning_rate": 0.0001, + "loss": 64.172, + "step": 2990 + }, + { + "epoch": 48.390243902439025, + "grad_norm": 1.3492090702056885, + "learning_rate": 0.0001, + "loss": 67.6642, + "step": 3000 + }, + { + "epoch": 48.552845528455286, + "grad_norm": 1.4024657011032104, + "learning_rate": 0.0001, + "loss": 65.2722, + "step": 3010 + }, + { + "epoch": 48.71544715447155, + "grad_norm": 1.1165252923965454, + "learning_rate": 0.0001, + "loss": 66.7233, + "step": 3020 + }, + { + "epoch": 48.8780487804878, + "grad_norm": 1.2147928476333618, + "learning_rate": 0.0001, + "loss": 65.7277, + "step": 3030 + }, + { + "epoch": 49.03252032520325, + "grad_norm": 0.925310492515564, + "learning_rate": 0.0001, + "loss": 62.9256, + "step": 3040 + }, + { + "epoch": 49.19512195121951, + "grad_norm": 1.2353442907333374, + "learning_rate": 0.0001, + "loss": 66.1218, + "step": 3050 + }, + { + "epoch": 49.357723577235774, + "grad_norm": 1.2177367210388184, + "learning_rate": 0.0001, + "loss": 66.7978, + "step": 3060 + }, + { + "epoch": 49.520325203252035, + "grad_norm": 1.248044729232788, + "learning_rate": 0.0001, + "loss": 66.2132, + "step": 3070 + }, + { + "epoch": 49.68292682926829, + "grad_norm": 1.366050124168396, + "learning_rate": 0.0001, + "loss": 67.0118, + "step": 3080 + }, + { + "epoch": 49.84552845528455, + "grad_norm": 1.3701151609420776, + "learning_rate": 0.0001, + "loss": 67.6099, + "step": 3090 + }, + { + "epoch": 50.0, + "grad_norm": 1.0420141220092773, + "learning_rate": 0.0001, + "loss": 61.5289, + "step": 3100 + }, + { + "epoch": 50.16260162601626, + "grad_norm": 1.0957746505737305, + "learning_rate": 0.0001, + "loss": 66.7348, + "step": 3110 + }, + { + "epoch": 50.32520325203252, + "grad_norm": 1.108628749847412, + "learning_rate": 0.0001, + "loss": 66.6199, + "step": 3120 + }, + { + "epoch": 50.48780487804878, + "grad_norm": 1.3680156469345093, + "learning_rate": 0.0001, + "loss": 67.0825, + "step": 3130 + }, + { + "epoch": 50.65040650406504, + "grad_norm": 1.169851303100586, + "learning_rate": 0.0001, + "loss": 65.4901, + "step": 3140 + }, + { + "epoch": 50.8130081300813, + "grad_norm": 2.1822028160095215, + "learning_rate": 0.0001, + "loss": 65.6841, + "step": 3150 + }, + { + "epoch": 50.97560975609756, + "grad_norm": 1.053402304649353, + "learning_rate": 0.0001, + "loss": 66.3119, + "step": 3160 + }, + { + "epoch": 51.13008130081301, + "grad_norm": 1.0627789497375488, + "learning_rate": 0.0001, + "loss": 61.3642, + "step": 3170 + }, + { + "epoch": 51.292682926829265, + "grad_norm": 1.926291823387146, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 3180 + }, + { + "epoch": 51.45528455284553, + "grad_norm": 1.5791983604431152, + "learning_rate": 0.0001, + "loss": 63.913, + "step": 3190 + }, + { + "epoch": 51.61788617886179, + "grad_norm": 1.210265040397644, + "learning_rate": 0.0001, + "loss": 64.5261, + "step": 3200 + }, + { + "epoch": 51.78048780487805, + "grad_norm": 1.0720808506011963, + "learning_rate": 0.0001, + "loss": 67.0194, + "step": 3210 + }, + { + "epoch": 51.94308943089431, + "grad_norm": 1.94563889503479, + "learning_rate": 0.0001, + "loss": 68.4662, + "step": 3220 + }, + { + "epoch": 52.09756097560975, + "grad_norm": 1.598980188369751, + "learning_rate": 0.0001, + "loss": 62.4946, + "step": 3230 + }, + { + "epoch": 52.260162601626014, + "grad_norm": 1.0564576387405396, + "learning_rate": 0.0001, + "loss": 65.7912, + "step": 3240 + }, + { + "epoch": 52.422764227642276, + "grad_norm": 1.2189645767211914, + "learning_rate": 0.0001, + "loss": 65.5772, + "step": 3250 + }, + { + "epoch": 52.58536585365854, + "grad_norm": 1.1357049942016602, + "learning_rate": 0.0001, + "loss": 65.1559, + "step": 3260 + }, + { + "epoch": 52.7479674796748, + "grad_norm": 1.316867709159851, + "learning_rate": 0.0001, + "loss": 64.7897, + "step": 3270 + }, + { + "epoch": 52.91056910569106, + "grad_norm": 1.3870197534561157, + "learning_rate": 0.0001, + "loss": 65.5195, + "step": 3280 + }, + { + "epoch": 53.0650406504065, + "grad_norm": 1.2467927932739258, + "learning_rate": 0.0001, + "loss": 61.3392, + "step": 3290 + }, + { + "epoch": 53.22764227642276, + "grad_norm": 1.1661109924316406, + "learning_rate": 0.0001, + "loss": 66.0117, + "step": 3300 + }, + { + "epoch": 53.390243902439025, + "grad_norm": 1.3509628772735596, + "learning_rate": 0.0001, + "loss": 64.7228, + "step": 3310 + }, + { + "epoch": 53.552845528455286, + "grad_norm": 1.1093642711639404, + "learning_rate": 0.0001, + "loss": 65.2534, + "step": 3320 + }, + { + "epoch": 53.71544715447155, + "grad_norm": 1.1801207065582275, + "learning_rate": 0.0001, + "loss": 65.7097, + "step": 3330 + }, + { + "epoch": 53.8780487804878, + "grad_norm": 1.175884485244751, + "learning_rate": 0.0001, + "loss": 64.4368, + "step": 3340 + }, + { + "epoch": 54.03252032520325, + "grad_norm": 1.1727889776229858, + "learning_rate": 0.0001, + "loss": 62.9373, + "step": 3350 + }, + { + "epoch": 54.19512195121951, + "grad_norm": 1.5330945253372192, + "learning_rate": 0.0001, + "loss": 66.7763, + "step": 3360 + }, + { + "epoch": 54.357723577235774, + "grad_norm": 1.2992504835128784, + "learning_rate": 0.0001, + "loss": 65.7854, + "step": 3370 + }, + { + "epoch": 54.520325203252035, + "grad_norm": 1.0185420513153076, + "learning_rate": 0.0001, + "loss": 65.7046, + "step": 3380 + }, + { + "epoch": 54.68292682926829, + "grad_norm": 1.426229476928711, + "learning_rate": 0.0001, + "loss": 64.0286, + "step": 3390 + }, + { + "epoch": 54.84552845528455, + "grad_norm": 1.2600868940353394, + "learning_rate": 0.0001, + "loss": 66.1738, + "step": 3400 + }, + { + "epoch": 55.0, + "grad_norm": 1.917455792427063, + "learning_rate": 0.0001, + "loss": 61.9381, + "step": 3410 + }, + { + "epoch": 55.16260162601626, + "grad_norm": 1.2000917196273804, + "learning_rate": 0.0001, + "loss": 65.5626, + "step": 3420 + }, + { + "epoch": 55.32520325203252, + "grad_norm": 1.362176775932312, + "learning_rate": 0.0001, + "loss": 66.8012, + "step": 3430 + }, + { + "epoch": 55.48780487804878, + "grad_norm": 1.1611402034759521, + "learning_rate": 0.0001, + "loss": 64.5227, + "step": 3440 + }, + { + "epoch": 55.65040650406504, + "grad_norm": 1.2066621780395508, + "learning_rate": 0.0001, + "loss": 63.4351, + "step": 3450 + }, + { + "epoch": 55.8130081300813, + "grad_norm": 1.005340814590454, + "learning_rate": 0.0001, + "loss": 65.9261, + "step": 3460 + }, + { + "epoch": 55.97560975609756, + "grad_norm": 1.0906637907028198, + "learning_rate": 0.0001, + "loss": 64.6635, + "step": 3470 + }, + { + "epoch": 56.13008130081301, + "grad_norm": 1.0608748197555542, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 3480 + }, + { + "epoch": 56.292682926829265, + "grad_norm": 1.025319218635559, + "learning_rate": 0.0001, + "loss": 64.7783, + "step": 3490 + }, + { + "epoch": 56.45528455284553, + "grad_norm": 1.1596784591674805, + "learning_rate": 0.0001, + "loss": 66.4629, + "step": 3500 + }, + { + "epoch": 56.61788617886179, + "grad_norm": 0.9318640232086182, + "learning_rate": 0.0001, + "loss": 65.5533, + "step": 3510 + }, + { + "epoch": 56.78048780487805, + "grad_norm": 1.0977418422698975, + "learning_rate": 0.0001, + "loss": 63.647, + "step": 3520 + }, + { + "epoch": 56.94308943089431, + "grad_norm": 1.3912436962127686, + "learning_rate": 0.0001, + "loss": 62.8273, + "step": 3530 + }, + { + "epoch": 57.09756097560975, + "grad_norm": 1.0921591520309448, + "learning_rate": 0.0001, + "loss": 61.5968, + "step": 3540 + }, + { + "epoch": 57.260162601626014, + "grad_norm": 1.17819344997406, + "learning_rate": 0.0001, + "loss": 64.9381, + "step": 3550 + }, + { + "epoch": 57.422764227642276, + "grad_norm": 1.188829779624939, + "learning_rate": 0.0001, + "loss": 64.6498, + "step": 3560 + }, + { + "epoch": 57.58536585365854, + "grad_norm": 3.664839506149292, + "learning_rate": 0.0001, + "loss": 65.3834, + "step": 3570 + }, + { + "epoch": 57.7479674796748, + "grad_norm": 1.57958984375, + "learning_rate": 0.0001, + "loss": 63.8808, + "step": 3580 + }, + { + "epoch": 57.91056910569106, + "grad_norm": 1.1837552785873413, + "learning_rate": 0.0001, + "loss": 64.1545, + "step": 3590 + }, + { + "epoch": 58.0650406504065, + "grad_norm": 1.5986013412475586, + "learning_rate": 0.0001, + "loss": 60.5742, + "step": 3600 + }, + { + "epoch": 58.22764227642276, + "grad_norm": 1.1551810503005981, + "learning_rate": 0.0001, + "loss": 66.2378, + "step": 3610 + }, + { + "epoch": 58.390243902439025, + "grad_norm": 1.0869863033294678, + "learning_rate": 0.0001, + "loss": 65.3034, + "step": 3620 + }, + { + "epoch": 58.552845528455286, + "grad_norm": 1.2263212203979492, + "learning_rate": 0.0001, + "loss": 63.976, + "step": 3630 + }, + { + "epoch": 58.71544715447155, + "grad_norm": 1.4410817623138428, + "learning_rate": 0.0001, + "loss": 63.4778, + "step": 3640 + }, + { + "epoch": 58.8780487804878, + "grad_norm": 1.9633759260177612, + "learning_rate": 0.0001, + "loss": 65.5946, + "step": 3650 + }, + { + "epoch": 59.03252032520325, + "grad_norm": 1.1729766130447388, + "learning_rate": 0.0001, + "loss": 62.2171, + "step": 3660 + }, + { + "epoch": 59.19512195121951, + "grad_norm": 1.0049556493759155, + "learning_rate": 0.0001, + "loss": 65.1854, + "step": 3670 + }, + { + "epoch": 59.357723577235774, + "grad_norm": 1.1727055311203003, + "learning_rate": 0.0001, + "loss": 65.4839, + "step": 3680 + }, + { + "epoch": 59.520325203252035, + "grad_norm": 1.5394458770751953, + "learning_rate": 0.0001, + "loss": 64.9341, + "step": 3690 + }, + { + "epoch": 59.68292682926829, + "grad_norm": 1.1583774089813232, + "learning_rate": 0.0001, + "loss": 63.9197, + "step": 3700 + }, + { + "epoch": 59.84552845528455, + "grad_norm": 1.0835607051849365, + "learning_rate": 0.0001, + "loss": 64.0494, + "step": 3710 + }, + { + "epoch": 60.0, + "grad_norm": 1.7504246234893799, + "learning_rate": 0.0001, + "loss": 59.4134, + "step": 3720 + }, + { + "epoch": 60.16260162601626, + "grad_norm": 1.1870343685150146, + "learning_rate": 0.0001, + "loss": 63.2158, + "step": 3730 + }, + { + "epoch": 60.32520325203252, + "grad_norm": 1.1983181238174438, + "learning_rate": 0.0001, + "loss": 64.7144, + "step": 3740 + }, + { + "epoch": 60.48780487804878, + "grad_norm": 1.3014581203460693, + "learning_rate": 0.0001, + "loss": 64.6219, + "step": 3750 + }, + { + "epoch": 60.65040650406504, + "grad_norm": 1.4954087734222412, + "learning_rate": 0.0001, + "loss": 63.3785, + "step": 3760 + }, + { + "epoch": 60.8130081300813, + "grad_norm": 1.2236649990081787, + "learning_rate": 0.0001, + "loss": 64.6933, + "step": 3770 + }, + { + "epoch": 60.97560975609756, + "grad_norm": 1.0796610116958618, + "learning_rate": 0.0001, + "loss": 63.2704, + "step": 3780 + }, + { + "epoch": 61.13008130081301, + "grad_norm": 0.9969968795776367, + "learning_rate": 0.0001, + "loss": 62.1838, + "step": 3790 + }, + { + "epoch": 61.292682926829265, + "grad_norm": 1.1624361276626587, + "learning_rate": 0.0001, + "loss": 63.6249, + "step": 3800 + }, + { + "epoch": 61.45528455284553, + "grad_norm": 1.3849223852157593, + "learning_rate": 0.0001, + "loss": 64.5321, + "step": 3810 + }, + { + "epoch": 61.61788617886179, + "grad_norm": 1.6607458591461182, + "learning_rate": 0.0001, + "loss": 64.651, + "step": 3820 + }, + { + "epoch": 61.78048780487805, + "grad_norm": 1.09684419631958, + "learning_rate": 0.0001, + "loss": 64.1129, + "step": 3830 + }, + { + "epoch": 61.94308943089431, + "grad_norm": 1.406246304512024, + "learning_rate": 0.0001, + "loss": 64.3642, + "step": 3840 + }, + { + "epoch": 62.09756097560975, + "grad_norm": 0.9850211143493652, + "learning_rate": 0.0001, + "loss": 60.8703, + "step": 3850 + }, + { + "epoch": 62.260162601626014, + "grad_norm": 1.001344084739685, + "learning_rate": 0.0001, + "loss": 67.5523, + "step": 3860 + }, + { + "epoch": 62.422764227642276, + "grad_norm": 2.435969591140747, + "learning_rate": 0.0001, + "loss": 62.7363, + "step": 3870 + }, + { + "epoch": 62.58536585365854, + "grad_norm": 1.0675824880599976, + "learning_rate": 0.0001, + "loss": 64.8513, + "step": 3880 + }, + { + "epoch": 62.7479674796748, + "grad_norm": 0.9793754816055298, + "learning_rate": 0.0001, + "loss": 64.9904, + "step": 3890 + }, + { + "epoch": 62.91056910569106, + "grad_norm": 1.2187775373458862, + "learning_rate": 0.0001, + "loss": 63.2821, + "step": 3900 + }, + { + "epoch": 63.0650406504065, + "grad_norm": 1.130772590637207, + "learning_rate": 0.0001, + "loss": 61.5957, + "step": 3910 + }, + { + "epoch": 63.22764227642276, + "grad_norm": 1.0573055744171143, + "learning_rate": 0.0001, + "loss": 63.6337, + "step": 3920 + }, + { + "epoch": 63.390243902439025, + "grad_norm": 2.055490255355835, + "learning_rate": 0.0001, + "loss": 63.9598, + "step": 3930 + }, + { + "epoch": 63.552845528455286, + "grad_norm": 1.0780516862869263, + "learning_rate": 0.0001, + "loss": 62.9303, + "step": 3940 + }, + { + "epoch": 63.71544715447155, + "grad_norm": 1.8136590719223022, + "learning_rate": 0.0001, + "loss": 64.4588, + "step": 3950 + }, + { + "epoch": 63.8780487804878, + "grad_norm": 1.0525016784667969, + "learning_rate": 0.0001, + "loss": 64.192, + "step": 3960 + }, + { + "epoch": 64.03252032520325, + "grad_norm": 1.12627375125885, + "learning_rate": 0.0001, + "loss": 59.1823, + "step": 3970 + }, + { + "epoch": 64.1951219512195, + "grad_norm": 1.2558730840682983, + "learning_rate": 0.0001, + "loss": 65.126, + "step": 3980 + }, + { + "epoch": 64.35772357723577, + "grad_norm": 1.2304726839065552, + "learning_rate": 0.0001, + "loss": 62.0868, + "step": 3990 + }, + { + "epoch": 64.52032520325203, + "grad_norm": 1.4179201126098633, + "learning_rate": 0.0001, + "loss": 62.3114, + "step": 4000 + }, + { + "epoch": 64.6829268292683, + "grad_norm": 1.125758409500122, + "learning_rate": 0.0001, + "loss": 63.4103, + "step": 4010 + }, + { + "epoch": 64.84552845528455, + "grad_norm": 2.321124315261841, + "learning_rate": 0.0001, + "loss": 62.9032, + "step": 4020 + }, + { + "epoch": 65.0, + "grad_norm": 1.4433993101119995, + "learning_rate": 0.0001, + "loss": 61.8766, + "step": 4030 + }, + { + "epoch": 65.16260162601625, + "grad_norm": 1.157878041267395, + "learning_rate": 0.0001, + "loss": 62.4833, + "step": 4040 + }, + { + "epoch": 65.32520325203252, + "grad_norm": 1.4127166271209717, + "learning_rate": 0.0001, + "loss": 64.768, + "step": 4050 + }, + { + "epoch": 65.48780487804878, + "grad_norm": 1.2575284242630005, + "learning_rate": 0.0001, + "loss": 62.9401, + "step": 4060 + }, + { + "epoch": 65.65040650406505, + "grad_norm": 1.0957911014556885, + "learning_rate": 0.0001, + "loss": 64.6061, + "step": 4070 + }, + { + "epoch": 65.8130081300813, + "grad_norm": 1.1773414611816406, + "learning_rate": 0.0001, + "loss": 63.9469, + "step": 4080 + }, + { + "epoch": 65.97560975609755, + "grad_norm": 2.1189942359924316, + "learning_rate": 0.0001, + "loss": 65.1422, + "step": 4090 + }, + { + "epoch": 66.130081300813, + "grad_norm": 1.1064234972000122, + "learning_rate": 0.0001, + "loss": 59.6108, + "step": 4100 + }, + { + "epoch": 66.29268292682927, + "grad_norm": 1.1135871410369873, + "learning_rate": 0.0001, + "loss": 63.4036, + "step": 4110 + }, + { + "epoch": 66.45528455284553, + "grad_norm": 4.167781829833984, + "learning_rate": 0.0001, + "loss": 63.1174, + "step": 4120 + }, + { + "epoch": 66.6178861788618, + "grad_norm": 1.1429376602172852, + "learning_rate": 0.0001, + "loss": 64.4085, + "step": 4130 + }, + { + "epoch": 66.78048780487805, + "grad_norm": 1.3188339471817017, + "learning_rate": 0.0001, + "loss": 62.5997, + "step": 4140 + }, + { + "epoch": 66.9430894308943, + "grad_norm": 1.160335659980774, + "learning_rate": 0.0001, + "loss": 64.2911, + "step": 4150 + }, + { + "epoch": 67.09756097560975, + "grad_norm": 0.9261873960494995, + "learning_rate": 0.0001, + "loss": 61.9036, + "step": 4160 + }, + { + "epoch": 67.26016260162602, + "grad_norm": 1.2254258394241333, + "learning_rate": 0.0001, + "loss": 64.1092, + "step": 4170 + }, + { + "epoch": 67.42276422764228, + "grad_norm": 0.9519097208976746, + "learning_rate": 0.0001, + "loss": 61.6428, + "step": 4180 + }, + { + "epoch": 67.58536585365853, + "grad_norm": 1.2290887832641602, + "learning_rate": 0.0001, + "loss": 64.9842, + "step": 4190 + }, + { + "epoch": 67.7479674796748, + "grad_norm": 1.0995723009109497, + "learning_rate": 0.0001, + "loss": 64.0677, + "step": 4200 + }, + { + "epoch": 67.91056910569105, + "grad_norm": 2.727957010269165, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 4210 + }, + { + "epoch": 68.0650406504065, + "grad_norm": 1.1222411394119263, + "learning_rate": 0.0001, + "loss": 61.0693, + "step": 4220 + }, + { + "epoch": 68.22764227642277, + "grad_norm": 1.159303069114685, + "learning_rate": 0.0001, + "loss": 64.2469, + "step": 4230 + }, + { + "epoch": 68.39024390243902, + "grad_norm": 1.2205990552902222, + "learning_rate": 0.0001, + "loss": 63.8831, + "step": 4240 + }, + { + "epoch": 68.55284552845528, + "grad_norm": 1.2351205348968506, + "learning_rate": 0.0001, + "loss": 63.4294, + "step": 4250 + }, + { + "epoch": 68.71544715447155, + "grad_norm": 1.4483757019042969, + "learning_rate": 0.0001, + "loss": 63.5345, + "step": 4260 + }, + { + "epoch": 68.8780487804878, + "grad_norm": 1.0808624029159546, + "learning_rate": 0.0001, + "loss": 63.6885, + "step": 4270 + }, + { + "epoch": 69.03252032520325, + "grad_norm": 1.7775999307632446, + "learning_rate": 0.0001, + "loss": 60.7266, + "step": 4280 + }, + { + "epoch": 69.1951219512195, + "grad_norm": 1.3214768171310425, + "learning_rate": 0.0001, + "loss": 63.3496, + "step": 4290 + }, + { + "epoch": 69.35772357723577, + "grad_norm": 1.1698002815246582, + "learning_rate": 0.0001, + "loss": 63.5284, + "step": 4300 + }, + { + "epoch": 69.52032520325203, + "grad_norm": 1.2050056457519531, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 4310 + }, + { + "epoch": 69.6829268292683, + "grad_norm": 1.0873243808746338, + "learning_rate": 0.0001, + "loss": 65.1709, + "step": 4320 + }, + { + "epoch": 69.84552845528455, + "grad_norm": 1.14133620262146, + "learning_rate": 0.0001, + "loss": 63.1872, + "step": 4330 + }, + { + "epoch": 70.0, + "grad_norm": 0.9684088230133057, + "learning_rate": 0.0001, + "loss": 60.1486, + "step": 4340 + }, + { + "epoch": 70.16260162601625, + "grad_norm": 1.1206653118133545, + "learning_rate": 0.0001, + "loss": 61.117, + "step": 4350 + }, + { + "epoch": 70.32520325203252, + "grad_norm": 1.2754334211349487, + "learning_rate": 0.0001, + "loss": 62.5409, + "step": 4360 + }, + { + "epoch": 70.48780487804878, + "grad_norm": 1.0154718160629272, + "learning_rate": 0.0001, + "loss": 62.9428, + "step": 4370 + }, + { + "epoch": 70.65040650406505, + "grad_norm": 1.0597270727157593, + "learning_rate": 0.0001, + "loss": 63.079, + "step": 4380 + }, + { + "epoch": 70.8130081300813, + "grad_norm": 1.254758596420288, + "learning_rate": 0.0001, + "loss": 63.3035, + "step": 4390 + }, + { + "epoch": 70.97560975609755, + "grad_norm": 1.0185120105743408, + "learning_rate": 0.0001, + "loss": 63.8922, + "step": 4400 + }, + { + "epoch": 71.130081300813, + "grad_norm": 1.007300853729248, + "learning_rate": 0.0001, + "loss": 58.7735, + "step": 4410 + }, + { + "epoch": 71.29268292682927, + "grad_norm": 1.2012463808059692, + "learning_rate": 0.0001, + "loss": 63.1782, + "step": 4420 + }, + { + "epoch": 71.45528455284553, + "grad_norm": 1.0258845090866089, + "learning_rate": 0.0001, + "loss": 63.2483, + "step": 4430 + }, + { + "epoch": 71.6178861788618, + "grad_norm": 1.0611220598220825, + "learning_rate": 0.0001, + "loss": 63.2247, + "step": 4440 + }, + { + "epoch": 71.78048780487805, + "grad_norm": 1.060114860534668, + "learning_rate": 0.0001, + "loss": 61.8352, + "step": 4450 + }, + { + "epoch": 71.9430894308943, + "grad_norm": 1.0540589094161987, + "learning_rate": 0.0001, + "loss": 62.1156, + "step": 4460 + }, + { + "epoch": 72.09756097560975, + "grad_norm": 1.4923726320266724, + "learning_rate": 0.0001, + "loss": 59.3863, + "step": 4470 + }, + { + "epoch": 72.26016260162602, + "grad_norm": 1.13791024684906, + "learning_rate": 0.0001, + "loss": 62.203, + "step": 4480 + }, + { + "epoch": 72.42276422764228, + "grad_norm": 0.9737065434455872, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 4490 + }, + { + "epoch": 72.58536585365853, + "grad_norm": 1.3980190753936768, + "learning_rate": 0.0001, + "loss": 63.5627, + "step": 4500 + }, + { + "epoch": 72.7479674796748, + "grad_norm": 1.0274118185043335, + "learning_rate": 0.0001, + "loss": 63.6129, + "step": 4510 + }, + { + "epoch": 72.91056910569105, + "grad_norm": 1.3148925304412842, + "learning_rate": 0.0001, + "loss": 63.6506, + "step": 4520 + }, + { + "epoch": 73.0650406504065, + "grad_norm": 1.4182451963424683, + "learning_rate": 0.0001, + "loss": 60.3616, + "step": 4530 + }, + { + "epoch": 73.22764227642277, + "grad_norm": 0.9235166907310486, + "learning_rate": 0.0001, + "loss": 63.6465, + "step": 4540 + }, + { + "epoch": 73.39024390243902, + "grad_norm": 0.9293469786643982, + "learning_rate": 0.0001, + "loss": 61.0984, + "step": 4550 + }, + { + "epoch": 73.55284552845528, + "grad_norm": 1.1075892448425293, + "learning_rate": 0.0001, + "loss": 62.9356, + "step": 4560 + }, + { + "epoch": 73.71544715447155, + "grad_norm": 1.4955459833145142, + "learning_rate": 0.0001, + "loss": 61.3811, + "step": 4570 + }, + { + "epoch": 73.8780487804878, + "grad_norm": 1.220595359802246, + "learning_rate": 0.0001, + "loss": 62.4364, + "step": 4580 + }, + { + "epoch": 74.03252032520325, + "grad_norm": 0.9975704550743103, + "learning_rate": 0.0001, + "loss": 58.5194, + "step": 4590 + }, + { + "epoch": 74.1951219512195, + "grad_norm": 1.0160857439041138, + "learning_rate": 0.0001, + "loss": 64.6989, + "step": 4600 + }, + { + "epoch": 74.35772357723577, + "grad_norm": 1.0052287578582764, + "learning_rate": 0.0001, + "loss": 62.526, + "step": 4610 + }, + { + "epoch": 74.52032520325203, + "grad_norm": 1.015514612197876, + "learning_rate": 0.0001, + "loss": 63.8627, + "step": 4620 + }, + { + "epoch": 74.6829268292683, + "grad_norm": 1.139051914215088, + "learning_rate": 0.0001, + "loss": 62.4525, + "step": 4630 + }, + { + "epoch": 74.84552845528455, + "grad_norm": 1.1727714538574219, + "learning_rate": 0.0001, + "loss": 64.0046, + "step": 4640 + }, + { + "epoch": 75.0, + "grad_norm": 1.1611062288284302, + "learning_rate": 0.0001, + "loss": 57.6443, + "step": 4650 + }, + { + "epoch": 75.16260162601625, + "grad_norm": 0.9893108606338501, + "learning_rate": 0.0001, + "loss": 64.1109, + "step": 4660 + }, + { + "epoch": 75.32520325203252, + "grad_norm": 1.190146803855896, + "learning_rate": 0.0001, + "loss": 62.1988, + "step": 4670 + }, + { + "epoch": 75.48780487804878, + "grad_norm": 1.1648579835891724, + "learning_rate": 0.0001, + "loss": 63.1393, + "step": 4680 + }, + { + "epoch": 75.65040650406505, + "grad_norm": 1.3303166627883911, + "learning_rate": 0.0001, + "loss": 63.7, + "step": 4690 + }, + { + "epoch": 75.8130081300813, + "grad_norm": 1.407772421836853, + "learning_rate": 0.0001, + "loss": 60.6798, + "step": 4700 + }, + { + "epoch": 75.97560975609755, + "grad_norm": 1.039741039276123, + "learning_rate": 0.0001, + "loss": 61.6126, + "step": 4710 + }, + { + "epoch": 76.130081300813, + "grad_norm": 1.013515830039978, + "learning_rate": 0.0001, + "loss": 57.8452, + "step": 4720 + }, + { + "epoch": 76.29268292682927, + "grad_norm": 1.1279789209365845, + "learning_rate": 0.0001, + "loss": 63.0592, + "step": 4730 + }, + { + "epoch": 76.45528455284553, + "grad_norm": 1.158605694770813, + "learning_rate": 0.0001, + "loss": 62.0836, + "step": 4740 + }, + { + "epoch": 76.6178861788618, + "grad_norm": 1.0200021266937256, + "learning_rate": 0.0001, + "loss": 62.1775, + "step": 4750 + }, + { + "epoch": 76.78048780487805, + "grad_norm": 1.216412901878357, + "learning_rate": 0.0001, + "loss": 62.6316, + "step": 4760 + }, + { + "epoch": 76.9430894308943, + "grad_norm": 1.0513523817062378, + "learning_rate": 0.0001, + "loss": 60.1037, + "step": 4770 + }, + { + "epoch": 77.09756097560975, + "grad_norm": 1.10075843334198, + "learning_rate": 0.0001, + "loss": 58.7599, + "step": 4780 + }, + { + "epoch": 77.26016260162602, + "grad_norm": 1.1728724241256714, + "learning_rate": 0.0001, + "loss": 63.3476, + "step": 4790 + }, + { + "epoch": 77.42276422764228, + "grad_norm": 1.3511605262756348, + "learning_rate": 0.0001, + "loss": 60.9885, + "step": 4800 + }, + { + "epoch": 77.58536585365853, + "grad_norm": 0.9733042120933533, + "learning_rate": 0.0001, + "loss": 62.4245, + "step": 4810 + }, + { + "epoch": 77.7479674796748, + "grad_norm": 1.0380535125732422, + "learning_rate": 0.0001, + "loss": 64.0755, + "step": 4820 + }, + { + "epoch": 77.91056910569105, + "grad_norm": 1.1594985723495483, + "learning_rate": 0.0001, + "loss": 62.1099, + "step": 4830 + }, + { + "epoch": 78.0650406504065, + "grad_norm": 1.0392569303512573, + "learning_rate": 0.0001, + "loss": 58.3778, + "step": 4840 + }, + { + "epoch": 78.22764227642277, + "grad_norm": 1.0032405853271484, + "learning_rate": 0.0001, + "loss": 62.1384, + "step": 4850 + }, + { + "epoch": 78.39024390243902, + "grad_norm": 1.0576013326644897, + "learning_rate": 0.0001, + "loss": 63.3308, + "step": 4860 + }, + { + "epoch": 78.55284552845528, + "grad_norm": 1.34328031539917, + "learning_rate": 0.0001, + "loss": 62.5351, + "step": 4870 + }, + { + "epoch": 78.71544715447155, + "grad_norm": 1.1040822267532349, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 4880 + }, + { + "epoch": 78.8780487804878, + "grad_norm": 1.1554768085479736, + "learning_rate": 0.0001, + "loss": 62.7308, + "step": 4890 + }, + { + "epoch": 79.03252032520325, + "grad_norm": 1.1864408254623413, + "learning_rate": 0.0001, + "loss": 60.0741, + "step": 4900 + }, + { + "epoch": 79.1951219512195, + "grad_norm": 0.9181821346282959, + "learning_rate": 0.0001, + "loss": 63.1682, + "step": 4910 + }, + { + "epoch": 79.35772357723577, + "grad_norm": 1.1400946378707886, + "learning_rate": 0.0001, + "loss": 62.0736, + "step": 4920 + }, + { + "epoch": 79.52032520325203, + "grad_norm": 1.245202660560608, + "learning_rate": 0.0001, + "loss": 60.51, + "step": 4930 + }, + { + "epoch": 79.6829268292683, + "grad_norm": 1.033347725868225, + "learning_rate": 0.0001, + "loss": 61.7845, + "step": 4940 + }, + { + "epoch": 79.84552845528455, + "grad_norm": 1.6206163167953491, + "learning_rate": 0.0001, + "loss": 61.5232, + "step": 4950 + }, + { + "epoch": 80.0, + "grad_norm": 1.0924937725067139, + "learning_rate": 0.0001, + "loss": 57.24, + "step": 4960 + }, + { + "epoch": 80.16260162601625, + "grad_norm": 1.075985074043274, + "learning_rate": 0.0001, + "loss": 61.2615, + "step": 4970 + }, + { + "epoch": 80.32520325203252, + "grad_norm": 0.9964771866798401, + "learning_rate": 0.0001, + "loss": 62.287, + "step": 4980 + }, + { + "epoch": 80.48780487804878, + "grad_norm": 1.3707735538482666, + "learning_rate": 0.0001, + "loss": 60.9383, + "step": 4990 + }, + { + "epoch": 80.65040650406505, + "grad_norm": 0.9395170211791992, + "learning_rate": 0.0001, + "loss": 61.2697, + "step": 5000 + }, + { + "epoch": 80.8130081300813, + "grad_norm": 2.0208873748779297, + "learning_rate": 0.0001, + "loss": 61.7219, + "step": 5010 + }, + { + "epoch": 80.97560975609755, + "grad_norm": 1.468702793121338, + "learning_rate": 0.0001, + "loss": 62.6967, + "step": 5020 + }, + { + "epoch": 81.130081300813, + "grad_norm": 1.0525966882705688, + "learning_rate": 0.0001, + "loss": 59.9793, + "step": 5030 + }, + { + "epoch": 81.29268292682927, + "grad_norm": 1.0951132774353027, + "learning_rate": 0.0001, + "loss": 60.9807, + "step": 5040 + }, + { + "epoch": 81.45528455284553, + "grad_norm": 1.3457857370376587, + "learning_rate": 0.0001, + "loss": 61.4137, + "step": 5050 + }, + { + "epoch": 81.6178861788618, + "grad_norm": 1.0895649194717407, + "learning_rate": 0.0001, + "loss": 61.2747, + "step": 5060 + }, + { + "epoch": 81.78048780487805, + "grad_norm": 1.0552371740341187, + "learning_rate": 0.0001, + "loss": 62.0947, + "step": 5070 + }, + { + "epoch": 81.9430894308943, + "grad_norm": 1.039038896560669, + "learning_rate": 0.0001, + "loss": 61.2571, + "step": 5080 + }, + { + "epoch": 82.09756097560975, + "grad_norm": 0.970783531665802, + "learning_rate": 0.0001, + "loss": 57.4157, + "step": 5090 + }, + { + "epoch": 82.26016260162602, + "grad_norm": 1.0276020765304565, + "learning_rate": 0.0001, + "loss": 62.1324, + "step": 5100 + }, + { + "epoch": 82.42276422764228, + "grad_norm": 1.2092448472976685, + "learning_rate": 0.0001, + "loss": 60.6448, + "step": 5110 + }, + { + "epoch": 82.58536585365853, + "grad_norm": 0.9352007508277893, + "learning_rate": 0.0001, + "loss": 61.9299, + "step": 5120 + }, + { + "epoch": 82.7479674796748, + "grad_norm": 1.2678358554840088, + "learning_rate": 0.0001, + "loss": 61.6622, + "step": 5130 + }, + { + "epoch": 82.91056910569105, + "grad_norm": 1.200494647026062, + "learning_rate": 0.0001, + "loss": 62.3169, + "step": 5140 + }, + { + "epoch": 83.0650406504065, + "grad_norm": 1.1284401416778564, + "learning_rate": 0.0001, + "loss": 58.7768, + "step": 5150 + }, + { + "epoch": 83.22764227642277, + "grad_norm": 1.1578079462051392, + "learning_rate": 0.0001, + "loss": 62.8423, + "step": 5160 + }, + { + "epoch": 83.39024390243902, + "grad_norm": 1.3426504135131836, + "learning_rate": 0.0001, + "loss": 61.5332, + "step": 5170 + }, + { + "epoch": 83.55284552845528, + "grad_norm": 0.9397301077842712, + "learning_rate": 0.0001, + "loss": 60.1043, + "step": 5180 + }, + { + "epoch": 83.71544715447155, + "grad_norm": 1.0728143453598022, + "learning_rate": 0.0001, + "loss": 60.5927, + "step": 5190 + }, + { + "epoch": 83.8780487804878, + "grad_norm": 1.2620211839675903, + "learning_rate": 0.0001, + "loss": 60.9224, + "step": 5200 + }, + { + "epoch": 84.03252032520325, + "grad_norm": 0.8232231140136719, + "learning_rate": 0.0001, + "loss": 58.8128, + "step": 5210 + }, + { + "epoch": 84.1951219512195, + "grad_norm": 1.0222564935684204, + "learning_rate": 0.0001, + "loss": 59.6995, + "step": 5220 + }, + { + "epoch": 84.35772357723577, + "grad_norm": 0.9078415632247925, + "learning_rate": 0.0001, + "loss": 61.1521, + "step": 5230 + }, + { + "epoch": 84.52032520325203, + "grad_norm": 1.0269399881362915, + "learning_rate": 0.0001, + "loss": 61.0136, + "step": 5240 + }, + { + "epoch": 84.6829268292683, + "grad_norm": 1.2108154296875, + "learning_rate": 0.0001, + "loss": 61.3564, + "step": 5250 + }, + { + "epoch": 84.84552845528455, + "grad_norm": 1.1170622110366821, + "learning_rate": 0.0001, + "loss": 61.7047, + "step": 5260 + }, + { + "epoch": 85.0, + "grad_norm": 1.467298150062561, + "learning_rate": 0.0001, + "loss": 58.1667, + "step": 5270 + }, + { + "epoch": 85.16260162601625, + "grad_norm": 0.908743143081665, + "learning_rate": 0.0001, + "loss": 61.5694, + "step": 5280 + }, + { + "epoch": 85.32520325203252, + "grad_norm": 0.9148766398429871, + "learning_rate": 0.0001, + "loss": 61.5325, + "step": 5290 + }, + { + "epoch": 85.48780487804878, + "grad_norm": 0.940021276473999, + "learning_rate": 0.0001, + "loss": 62.8807, + "step": 5300 + }, + { + "epoch": 85.65040650406505, + "grad_norm": 1.1239454746246338, + "learning_rate": 0.0001, + "loss": 62.361, + "step": 5310 + }, + { + "epoch": 85.8130081300813, + "grad_norm": 0.840437114238739, + "learning_rate": 0.0001, + "loss": 61.9057, + "step": 5320 + }, + { + "epoch": 85.97560975609755, + "grad_norm": 0.9764263033866882, + "learning_rate": 0.0001, + "loss": 61.0075, + "step": 5330 + }, + { + "epoch": 86.130081300813, + "grad_norm": 0.998540997505188, + "learning_rate": 0.0001, + "loss": 57.7146, + "step": 5340 + }, + { + "epoch": 86.29268292682927, + "grad_norm": 1.121551513671875, + "learning_rate": 0.0001, + "loss": 62.2278, + "step": 5350 + }, + { + "epoch": 86.45528455284553, + "grad_norm": 1.101842999458313, + "learning_rate": 0.0001, + "loss": 60.8837, + "step": 5360 + }, + { + "epoch": 86.6178861788618, + "grad_norm": 1.1724908351898193, + "learning_rate": 0.0001, + "loss": 61.6788, + "step": 5370 + }, + { + "epoch": 86.78048780487805, + "grad_norm": 1.3108958005905151, + "learning_rate": 0.0001, + "loss": 60.1461, + "step": 5380 + }, + { + "epoch": 86.9430894308943, + "grad_norm": 1.482298731803894, + "learning_rate": 0.0001, + "loss": 61.0457, + "step": 5390 + }, + { + "epoch": 87.09756097560975, + "grad_norm": 1.1471034288406372, + "learning_rate": 0.0001, + "loss": 57.6835, + "step": 5400 + }, + { + "epoch": 87.26016260162602, + "grad_norm": 0.9349374175071716, + "learning_rate": 0.0001, + "loss": 62.7018, + "step": 5410 + }, + { + "epoch": 87.42276422764228, + "grad_norm": 0.9917856454849243, + "learning_rate": 0.0001, + "loss": 60.2201, + "step": 5420 + }, + { + "epoch": 87.58536585365853, + "grad_norm": 1.0112310647964478, + "learning_rate": 0.0001, + "loss": 62.1185, + "step": 5430 + }, + { + "epoch": 87.7479674796748, + "grad_norm": 1.288894534111023, + "learning_rate": 0.0001, + "loss": 61.4348, + "step": 5440 + }, + { + "epoch": 87.91056910569105, + "grad_norm": 0.8577322363853455, + "learning_rate": 0.0001, + "loss": 60.7867, + "step": 5450 + }, + { + "epoch": 88.0650406504065, + "grad_norm": 0.8162860870361328, + "learning_rate": 0.0001, + "loss": 59.5224, + "step": 5460 + }, + { + "epoch": 88.22764227642277, + "grad_norm": 0.9491421580314636, + "learning_rate": 0.0001, + "loss": 60.4936, + "step": 5470 + }, + { + "epoch": 88.39024390243902, + "grad_norm": 4.164788246154785, + "learning_rate": 0.0001, + "loss": 61.8397, + "step": 5480 + }, + { + "epoch": 88.55284552845528, + "grad_norm": 0.9954016804695129, + "learning_rate": 0.0001, + "loss": 61.286, + "step": 5490 + }, + { + "epoch": 88.71544715447155, + "grad_norm": 1.4991120100021362, + "learning_rate": 0.0001, + "loss": 60.6157, + "step": 5500 + }, + { + "epoch": 88.8780487804878, + "grad_norm": 1.3678480386734009, + "learning_rate": 0.0001, + "loss": 62.5187, + "step": 5510 + }, + { + "epoch": 89.03252032520325, + "grad_norm": 1.057661771774292, + "learning_rate": 0.0001, + "loss": 57.5134, + "step": 5520 + }, + { + "epoch": 89.1951219512195, + "grad_norm": 0.8331280946731567, + "learning_rate": 0.0001, + "loss": 62.007, + "step": 5530 + }, + { + "epoch": 89.35772357723577, + "grad_norm": 0.9842398762702942, + "learning_rate": 0.0001, + "loss": 60.6178, + "step": 5540 + }, + { + "epoch": 89.52032520325203, + "grad_norm": 1.0510483980178833, + "learning_rate": 0.0001, + "loss": 59.6382, + "step": 5550 + }, + { + "epoch": 89.6829268292683, + "grad_norm": 1.013115406036377, + "learning_rate": 0.0001, + "loss": 61.015, + "step": 5560 + }, + { + "epoch": 89.84552845528455, + "grad_norm": 1.1595944166183472, + "learning_rate": 0.0001, + "loss": 62.5374, + "step": 5570 + }, + { + "epoch": 90.0, + "grad_norm": 1.1506270170211792, + "learning_rate": 0.0001, + "loss": 61.0063, + "step": 5580 + }, + { + "epoch": 90.16260162601625, + "grad_norm": 0.9940153360366821, + "learning_rate": 0.0001, + "loss": 62.0944, + "step": 5590 + }, + { + "epoch": 90.32520325203252, + "grad_norm": 1.1302493810653687, + "learning_rate": 0.0001, + "loss": 61.616, + "step": 5600 + }, + { + "epoch": 90.48780487804878, + "grad_norm": 1.2370656728744507, + "learning_rate": 0.0001, + "loss": 61.291, + "step": 5610 + }, + { + "epoch": 90.65040650406505, + "grad_norm": 1.228763461112976, + "learning_rate": 0.0001, + "loss": 60.0629, + "step": 5620 + }, + { + "epoch": 90.8130081300813, + "grad_norm": 1.2230069637298584, + "learning_rate": 0.0001, + "loss": 61.5391, + "step": 5630 + }, + { + "epoch": 90.97560975609755, + "grad_norm": 0.9849008917808533, + "learning_rate": 0.0001, + "loss": 63.1633, + "step": 5640 + }, + { + "epoch": 91.130081300813, + "grad_norm": 1.408632755279541, + "learning_rate": 0.0001, + "loss": 58.4195, + "step": 5650 + }, + { + "epoch": 91.29268292682927, + "grad_norm": 0.929027795791626, + "learning_rate": 0.0001, + "loss": 60.9862, + "step": 5660 + }, + { + "epoch": 91.45528455284553, + "grad_norm": 0.9228377342224121, + "learning_rate": 0.0001, + "loss": 60.8337, + "step": 5670 + }, + { + "epoch": 91.6178861788618, + "grad_norm": 1.0354220867156982, + "learning_rate": 0.0001, + "loss": 59.4667, + "step": 5680 + }, + { + "epoch": 91.78048780487805, + "grad_norm": 1.0338927507400513, + "learning_rate": 0.0001, + "loss": 62.0064, + "step": 5690 + }, + { + "epoch": 91.9430894308943, + "grad_norm": 1.101967215538025, + "learning_rate": 0.0001, + "loss": 61.6317, + "step": 5700 + }, + { + "epoch": 92.09756097560975, + "grad_norm": 1.370689034461975, + "learning_rate": 0.0001, + "loss": 57.2157, + "step": 5710 + }, + { + "epoch": 92.26016260162602, + "grad_norm": 0.9904115796089172, + "learning_rate": 0.0001, + "loss": 61.2786, + "step": 5720 + }, + { + "epoch": 92.42276422764228, + "grad_norm": 1.0545448064804077, + "learning_rate": 0.0001, + "loss": 63.2384, + "step": 5730 + }, + { + "epoch": 92.58536585365853, + "grad_norm": 1.0657731294631958, + "learning_rate": 0.0001, + "loss": 60.582, + "step": 5740 + }, + { + "epoch": 92.7479674796748, + "grad_norm": 0.9333183765411377, + "learning_rate": 0.0001, + "loss": 62.4627, + "step": 5750 + }, + { + "epoch": 92.91056910569105, + "grad_norm": 0.9810492396354675, + "learning_rate": 0.0001, + "loss": 60.0055, + "step": 5760 + }, + { + "epoch": 93.0650406504065, + "grad_norm": 1.6973884105682373, + "learning_rate": 0.0001, + "loss": 57.5381, + "step": 5770 + }, + { + "epoch": 93.22764227642277, + "grad_norm": 1.0075498819351196, + "learning_rate": 0.0001, + "loss": 61.2272, + "step": 5780 + }, + { + "epoch": 93.39024390243902, + "grad_norm": 0.980633556842804, + "learning_rate": 0.0001, + "loss": 60.4794, + "step": 5790 + }, + { + "epoch": 93.55284552845528, + "grad_norm": 1.128214955329895, + "learning_rate": 0.0001, + "loss": 60.4825, + "step": 5800 + }, + { + "epoch": 93.71544715447155, + "grad_norm": 1.302827000617981, + "learning_rate": 0.0001, + "loss": 62.4397, + "step": 5810 + }, + { + "epoch": 93.8780487804878, + "grad_norm": 0.826600193977356, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 5820 + }, + { + "epoch": 94.03252032520325, + "grad_norm": 11.418835639953613, + "learning_rate": 0.0001, + "loss": 58.5383, + "step": 5830 + }, + { + "epoch": 94.1951219512195, + "grad_norm": 1.0686477422714233, + "learning_rate": 0.0001, + "loss": 60.5696, + "step": 5840 + }, + { + "epoch": 94.35772357723577, + "grad_norm": 1.1232597827911377, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 5850 + }, + { + "epoch": 94.52032520325203, + "grad_norm": 1.1051944494247437, + "learning_rate": 0.0001, + "loss": 60.5387, + "step": 5860 + }, + { + "epoch": 94.6829268292683, + "grad_norm": 0.9954739809036255, + "learning_rate": 0.0001, + "loss": 61.3899, + "step": 5870 + }, + { + "epoch": 94.84552845528455, + "grad_norm": 1.9687999486923218, + "learning_rate": 0.0001, + "loss": 60.7711, + "step": 5880 + }, + { + "epoch": 95.0, + "grad_norm": 1.3172541856765747, + "learning_rate": 0.0001, + "loss": 58.5394, + "step": 5890 + }, + { + "epoch": 95.16260162601625, + "grad_norm": 1.2251427173614502, + "learning_rate": 0.0001, + "loss": 61.8319, + "step": 5900 + }, + { + "epoch": 95.32520325203252, + "grad_norm": 1.1701544523239136, + "learning_rate": 0.0001, + "loss": 60.1928, + "step": 5910 + }, + { + "epoch": 95.48780487804878, + "grad_norm": 1.209375262260437, + "learning_rate": 0.0001, + "loss": 61.1347, + "step": 5920 + }, + { + "epoch": 95.65040650406505, + "grad_norm": 0.9307044148445129, + "learning_rate": 0.0001, + "loss": 62.6676, + "step": 5930 + }, + { + "epoch": 95.8130081300813, + "grad_norm": 0.9679144024848938, + "learning_rate": 0.0001, + "loss": 60.7623, + "step": 5940 + }, + { + "epoch": 95.97560975609755, + "grad_norm": 1.398711085319519, + "learning_rate": 0.0001, + "loss": 59.0407, + "step": 5950 + }, + { + "epoch": 96.130081300813, + "grad_norm": 1.9118247032165527, + "learning_rate": 0.0001, + "loss": 59.9417, + "step": 5960 + }, + { + "epoch": 96.29268292682927, + "grad_norm": 1.036737084388733, + "learning_rate": 0.0001, + "loss": 60.6891, + "step": 5970 + }, + { + "epoch": 96.45528455284553, + "grad_norm": 1.5916132926940918, + "learning_rate": 0.0001, + "loss": 59.44, + "step": 5980 + }, + { + "epoch": 96.6178861788618, + "grad_norm": 1.526591181755066, + "learning_rate": 0.0001, + "loss": 58.9081, + "step": 5990 + }, + { + "epoch": 96.78048780487805, + "grad_norm": 0.9556435942649841, + "learning_rate": 0.0001, + "loss": 61.5743, + "step": 6000 + }, + { + "epoch": 96.9430894308943, + "grad_norm": 1.0041887760162354, + "learning_rate": 0.0001, + "loss": 60.0724, + "step": 6010 + }, + { + "epoch": 97.09756097560975, + "grad_norm": 0.9792081117630005, + "learning_rate": 0.0001, + "loss": 60.0613, + "step": 6020 + }, + { + "epoch": 97.26016260162602, + "grad_norm": 1.029634714126587, + "learning_rate": 0.0001, + "loss": 60.3526, + "step": 6030 + }, + { + "epoch": 97.42276422764228, + "grad_norm": 0.9812865257263184, + "learning_rate": 0.0001, + "loss": 60.9818, + "step": 6040 + }, + { + "epoch": 97.58536585365853, + "grad_norm": 0.8826327323913574, + "learning_rate": 0.0001, + "loss": 61.6375, + "step": 6050 + }, + { + "epoch": 97.7479674796748, + "grad_norm": 1.2197599411010742, + "learning_rate": 0.0001, + "loss": 60.2698, + "step": 6060 + }, + { + "epoch": 97.91056910569105, + "grad_norm": 0.9068922400474548, + "learning_rate": 0.0001, + "loss": 58.8698, + "step": 6070 + }, + { + "epoch": 98.0650406504065, + "grad_norm": 0.9008221626281738, + "learning_rate": 0.0001, + "loss": 56.4687, + "step": 6080 + }, + { + "epoch": 98.22764227642277, + "grad_norm": 1.8165878057479858, + "learning_rate": 0.0001, + "loss": 61.4213, + "step": 6090 + }, + { + "epoch": 98.39024390243902, + "grad_norm": 0.9541304111480713, + "learning_rate": 0.0001, + "loss": 62.8381, + "step": 6100 + }, + { + "epoch": 98.55284552845528, + "grad_norm": 1.293872594833374, + "learning_rate": 0.0001, + "loss": 61.3292, + "step": 6110 + }, + { + "epoch": 98.71544715447155, + "grad_norm": 0.9569422006607056, + "learning_rate": 0.0001, + "loss": 61.436, + "step": 6120 + }, + { + "epoch": 98.8780487804878, + "grad_norm": 1.160468578338623, + "learning_rate": 0.0001, + "loss": 60.8419, + "step": 6130 + }, + { + "epoch": 99.03252032520325, + "grad_norm": 1.3102303743362427, + "learning_rate": 0.0001, + "loss": 59.0064, + "step": 6140 + }, + { + "epoch": 99.1951219512195, + "grad_norm": 0.9553463459014893, + "learning_rate": 0.0001, + "loss": 59.925, + "step": 6150 + }, + { + "epoch": 99.35772357723577, + "grad_norm": 1.1993688344955444, + "learning_rate": 0.0001, + "loss": 62.3683, + "step": 6160 + }, + { + "epoch": 99.52032520325203, + "grad_norm": 1.0867350101470947, + "learning_rate": 0.0001, + "loss": 61.8848, + "step": 6170 + }, + { + "epoch": 99.6829268292683, + "grad_norm": 0.9924171566963196, + "learning_rate": 0.0001, + "loss": 60.9243, + "step": 6180 + }, + { + "epoch": 99.84552845528455, + "grad_norm": 0.9548614025115967, + "learning_rate": 0.0001, + "loss": 61.0234, + "step": 6190 + }, + { + "epoch": 100.0, + "grad_norm": 1.0070925951004028, + "learning_rate": 0.0001, + "loss": 56.6554, + "step": 6200 + }, + { + "epoch": 100.16260162601625, + "grad_norm": 1.202168583869934, + "learning_rate": 0.0001, + "loss": 61.3235, + "step": 6210 + }, + { + "epoch": 100.32520325203252, + "grad_norm": 0.9931091070175171, + "learning_rate": 0.0001, + "loss": 59.651, + "step": 6220 + }, + { + "epoch": 100.48780487804878, + "grad_norm": 0.9679157137870789, + "learning_rate": 0.0001, + "loss": 61.5785, + "step": 6230 + }, + { + "epoch": 100.65040650406505, + "grad_norm": 1.049842119216919, + "learning_rate": 0.0001, + "loss": 58.9646, + "step": 6240 + }, + { + "epoch": 100.8130081300813, + "grad_norm": 1.0772044658660889, + "learning_rate": 0.0001, + "loss": 62.3309, + "step": 6250 + }, + { + "epoch": 100.97560975609755, + "grad_norm": 0.9494168162345886, + "learning_rate": 0.0001, + "loss": 61.8663, + "step": 6260 + }, + { + "epoch": 101.130081300813, + "grad_norm": 1.0491962432861328, + "learning_rate": 0.0001, + "loss": 56.5872, + "step": 6270 + }, + { + "epoch": 101.29268292682927, + "grad_norm": 1.118611216545105, + "learning_rate": 0.0001, + "loss": 58.1338, + "step": 6280 + }, + { + "epoch": 101.45528455284553, + "grad_norm": 1.0419825315475464, + "learning_rate": 0.0001, + "loss": 59.6661, + "step": 6290 + }, + { + "epoch": 101.6178861788618, + "grad_norm": 1.2351056337356567, + "learning_rate": 0.0001, + "loss": 61.1697, + "step": 6300 + }, + { + "epoch": 101.78048780487805, + "grad_norm": 1.0556049346923828, + "learning_rate": 0.0001, + "loss": 60.463, + "step": 6310 + }, + { + "epoch": 101.9430894308943, + "grad_norm": 1.090761661529541, + "learning_rate": 0.0001, + "loss": 61.983, + "step": 6320 + }, + { + "epoch": 102.09756097560975, + "grad_norm": 1.4305297136306763, + "learning_rate": 0.0001, + "loss": 55.559, + "step": 6330 + }, + { + "epoch": 102.26016260162602, + "grad_norm": 1.1773794889450073, + "learning_rate": 0.0001, + "loss": 59.5931, + "step": 6340 + }, + { + "epoch": 102.42276422764228, + "grad_norm": 1.0028730630874634, + "learning_rate": 0.0001, + "loss": 60.2714, + "step": 6350 + }, + { + "epoch": 102.58536585365853, + "grad_norm": 0.9494019150733948, + "learning_rate": 0.0001, + "loss": 59.7757, + "step": 6360 + }, + { + "epoch": 102.7479674796748, + "grad_norm": 1.0415751934051514, + "learning_rate": 0.0001, + "loss": 60.7123, + "step": 6370 + }, + { + "epoch": 102.91056910569105, + "grad_norm": 1.2036449909210205, + "learning_rate": 0.0001, + "loss": 60.1201, + "step": 6380 + }, + { + "epoch": 103.0650406504065, + "grad_norm": 0.8801053166389465, + "learning_rate": 0.0001, + "loss": 56.4943, + "step": 6390 + }, + { + "epoch": 103.22764227642277, + "grad_norm": 0.9018735289573669, + "learning_rate": 0.0001, + "loss": 62.4522, + "step": 6400 + }, + { + "epoch": 103.39024390243902, + "grad_norm": 0.971685528755188, + "learning_rate": 0.0001, + "loss": 62.2367, + "step": 6410 + }, + { + "epoch": 103.55284552845528, + "grad_norm": 0.8531002402305603, + "learning_rate": 0.0001, + "loss": 59.5436, + "step": 6420 + }, + { + "epoch": 103.71544715447155, + "grad_norm": 2.123033285140991, + "learning_rate": 0.0001, + "loss": 59.0421, + "step": 6430 + }, + { + "epoch": 103.8780487804878, + "grad_norm": 1.209804654121399, + "learning_rate": 0.0001, + "loss": 60.7791, + "step": 6440 + }, + { + "epoch": 104.03252032520325, + "grad_norm": 0.8219161033630371, + "learning_rate": 0.0001, + "loss": 55.7522, + "step": 6450 + }, + { + "epoch": 104.1951219512195, + "grad_norm": 1.8217495679855347, + "learning_rate": 0.0001, + "loss": 60.6916, + "step": 6460 + }, + { + "epoch": 104.35772357723577, + "grad_norm": 1.060073971748352, + "learning_rate": 0.0001, + "loss": 59.971, + "step": 6470 + }, + { + "epoch": 104.52032520325203, + "grad_norm": 0.9892823696136475, + "learning_rate": 0.0001, + "loss": 61.467, + "step": 6480 + }, + { + "epoch": 104.6829268292683, + "grad_norm": 1.128934621810913, + "learning_rate": 0.0001, + "loss": 62.0451, + "step": 6490 + }, + { + "epoch": 104.84552845528455, + "grad_norm": 0.9627387523651123, + "learning_rate": 0.0001, + "loss": 61.532, + "step": 6500 + }, + { + "epoch": 105.0, + "grad_norm": 1.5344375371932983, + "learning_rate": 0.0001, + "loss": 56.4323, + "step": 6510 + }, + { + "epoch": 105.16260162601625, + "grad_norm": 0.9524087309837341, + "learning_rate": 0.0001, + "loss": 59.6628, + "step": 6520 + }, + { + "epoch": 105.32520325203252, + "grad_norm": 1.0469943284988403, + "learning_rate": 0.0001, + "loss": 59.0605, + "step": 6530 + }, + { + "epoch": 105.48780487804878, + "grad_norm": 2.7462990283966064, + "learning_rate": 0.0001, + "loss": 61.1429, + "step": 6540 + }, + { + "epoch": 105.65040650406505, + "grad_norm": 1.3140382766723633, + "learning_rate": 0.0001, + "loss": 58.2711, + "step": 6550 + }, + { + "epoch": 105.8130081300813, + "grad_norm": 1.1528750658035278, + "learning_rate": 0.0001, + "loss": 59.9937, + "step": 6560 + }, + { + "epoch": 105.97560975609755, + "grad_norm": 1.4449951648712158, + "learning_rate": 0.0001, + "loss": 59.1526, + "step": 6570 + }, + { + "epoch": 106.130081300813, + "grad_norm": 1.1350126266479492, + "learning_rate": 0.0001, + "loss": 55.1142, + "step": 6580 + }, + { + "epoch": 106.29268292682927, + "grad_norm": 0.9332845211029053, + "learning_rate": 0.0001, + "loss": 59.8049, + "step": 6590 + }, + { + "epoch": 106.45528455284553, + "grad_norm": 0.9678266644477844, + "learning_rate": 0.0001, + "loss": 60.9627, + "step": 6600 + }, + { + "epoch": 106.6178861788618, + "grad_norm": 1.1794427633285522, + "learning_rate": 0.0001, + "loss": 59.5372, + "step": 6610 + }, + { + "epoch": 106.78048780487805, + "grad_norm": 1.0404181480407715, + "learning_rate": 0.0001, + "loss": 59.2715, + "step": 6620 + }, + { + "epoch": 106.9430894308943, + "grad_norm": 0.8667073845863342, + "learning_rate": 0.0001, + "loss": 60.2625, + "step": 6630 + }, + { + "epoch": 107.09756097560975, + "grad_norm": 0.9940314888954163, + "learning_rate": 0.0001, + "loss": 55.6906, + "step": 6640 + }, + { + "epoch": 107.26016260162602, + "grad_norm": 1.3880919218063354, + "learning_rate": 0.0001, + "loss": 59.8103, + "step": 6650 + }, + { + "epoch": 107.42276422764228, + "grad_norm": 0.900931179523468, + "learning_rate": 0.0001, + "loss": 59.1929, + "step": 6660 + }, + { + "epoch": 107.58536585365853, + "grad_norm": 0.9374089241027832, + "learning_rate": 0.0001, + "loss": 60.1047, + "step": 6670 + }, + { + "epoch": 107.7479674796748, + "grad_norm": 1.0847140550613403, + "learning_rate": 0.0001, + "loss": 58.6943, + "step": 6680 + }, + { + "epoch": 107.91056910569105, + "grad_norm": 1.086506962776184, + "learning_rate": 0.0001, + "loss": 60.0341, + "step": 6690 + }, + { + "epoch": 108.0650406504065, + "grad_norm": 0.8984777331352234, + "learning_rate": 0.0001, + "loss": 57.0422, + "step": 6700 + }, + { + "epoch": 108.22764227642277, + "grad_norm": 1.187198519706726, + "learning_rate": 0.0001, + "loss": 60.7935, + "step": 6710 + }, + { + "epoch": 108.39024390243902, + "grad_norm": 0.8393696546554565, + "learning_rate": 0.0001, + "loss": 59.7896, + "step": 6720 + }, + { + "epoch": 108.55284552845528, + "grad_norm": 1.0898207426071167, + "learning_rate": 0.0001, + "loss": 60.2579, + "step": 6730 + }, + { + "epoch": 108.71544715447155, + "grad_norm": 1.016926646232605, + "learning_rate": 0.0001, + "loss": 60.9018, + "step": 6740 + }, + { + "epoch": 108.8780487804878, + "grad_norm": 1.3991293907165527, + "learning_rate": 0.0001, + "loss": 60.889, + "step": 6750 + }, + { + "epoch": 109.03252032520325, + "grad_norm": 0.9950843453407288, + "learning_rate": 0.0001, + "loss": 55.1098, + "step": 6760 + }, + { + "epoch": 109.1951219512195, + "grad_norm": 1.0843164920806885, + "learning_rate": 0.0001, + "loss": 60.3208, + "step": 6770 + }, + { + "epoch": 109.35772357723577, + "grad_norm": 0.9067014455795288, + "learning_rate": 0.0001, + "loss": 59.1254, + "step": 6780 + }, + { + "epoch": 109.52032520325203, + "grad_norm": 1.2166807651519775, + "learning_rate": 0.0001, + "loss": 59.9978, + "step": 6790 + }, + { + "epoch": 109.6829268292683, + "grad_norm": 1.256101131439209, + "learning_rate": 0.0001, + "loss": 60.2544, + "step": 6800 + }, + { + "epoch": 109.84552845528455, + "grad_norm": 1.312761664390564, + "learning_rate": 0.0001, + "loss": 59.1564, + "step": 6810 + }, + { + "epoch": 110.0, + "grad_norm": 0.9668724536895752, + "learning_rate": 0.0001, + "loss": 58.067, + "step": 6820 + }, + { + "epoch": 110.16260162601625, + "grad_norm": 1.1048885583877563, + "learning_rate": 0.0001, + "loss": 59.1506, + "step": 6830 + }, + { + "epoch": 110.32520325203252, + "grad_norm": 1.498469352722168, + "learning_rate": 0.0001, + "loss": 59.1366, + "step": 6840 + }, + { + "epoch": 110.48780487804878, + "grad_norm": 1.1747376918792725, + "learning_rate": 0.0001, + "loss": 59.8449, + "step": 6850 + }, + { + "epoch": 110.65040650406505, + "grad_norm": 1.1041772365570068, + "learning_rate": 0.0001, + "loss": 61.0879, + "step": 6860 + }, + { + "epoch": 110.8130081300813, + "grad_norm": 0.9452431797981262, + "learning_rate": 0.0001, + "loss": 60.8911, + "step": 6870 + }, + { + "epoch": 110.97560975609755, + "grad_norm": 1.0108182430267334, + "learning_rate": 0.0001, + "loss": 58.7924, + "step": 6880 + }, + { + "epoch": 111.130081300813, + "grad_norm": 0.9914487600326538, + "learning_rate": 0.0001, + "loss": 57.4749, + "step": 6890 + }, + { + "epoch": 111.29268292682927, + "grad_norm": 1.0404850244522095, + "learning_rate": 0.0001, + "loss": 59.5329, + "step": 6900 + }, + { + "epoch": 111.45528455284553, + "grad_norm": 1.0363966226577759, + "learning_rate": 0.0001, + "loss": 58.8972, + "step": 6910 + }, + { + "epoch": 111.6178861788618, + "grad_norm": 1.0189651250839233, + "learning_rate": 0.0001, + "loss": 59.3613, + "step": 6920 + }, + { + "epoch": 111.78048780487805, + "grad_norm": 0.9696914553642273, + "learning_rate": 0.0001, + "loss": 59.8001, + "step": 6930 + }, + { + "epoch": 111.9430894308943, + "grad_norm": 1.406821370124817, + "learning_rate": 0.0001, + "loss": 58.6405, + "step": 6940 + }, + { + "epoch": 112.09756097560975, + "grad_norm": 1.0969383716583252, + "learning_rate": 0.0001, + "loss": 57.4458, + "step": 6950 + }, + { + "epoch": 112.26016260162602, + "grad_norm": 1.117593765258789, + "learning_rate": 0.0001, + "loss": 62.599, + "step": 6960 + }, + { + "epoch": 112.42276422764228, + "grad_norm": 0.9327741861343384, + "learning_rate": 0.0001, + "loss": 60.632, + "step": 6970 + }, + { + "epoch": 112.58536585365853, + "grad_norm": 1.1198201179504395, + "learning_rate": 0.0001, + "loss": 59.6119, + "step": 6980 + }, + { + "epoch": 112.7479674796748, + "grad_norm": 0.8709354400634766, + "learning_rate": 0.0001, + "loss": 60.831, + "step": 6990 + }, + { + "epoch": 112.91056910569105, + "grad_norm": 0.9589444994926453, + "learning_rate": 0.0001, + "loss": 58.0002, + "step": 7000 + }, + { + "epoch": 113.0650406504065, + "grad_norm": 1.0971899032592773, + "learning_rate": 0.0001, + "loss": 56.8572, + "step": 7010 + }, + { + "epoch": 113.22764227642277, + "grad_norm": 0.9764513969421387, + "learning_rate": 0.0001, + "loss": 59.3038, + "step": 7020 + }, + { + "epoch": 113.39024390243902, + "grad_norm": 0.9050093293190002, + "learning_rate": 0.0001, + "loss": 59.9146, + "step": 7030 + }, + { + "epoch": 113.55284552845528, + "grad_norm": 0.9506774544715881, + "learning_rate": 0.0001, + "loss": 59.7523, + "step": 7040 + }, + { + "epoch": 113.71544715447155, + "grad_norm": 0.9678346514701843, + "learning_rate": 0.0001, + "loss": 61.3103, + "step": 7050 + }, + { + "epoch": 113.8780487804878, + "grad_norm": 0.9859515428543091, + "learning_rate": 0.0001, + "loss": 56.3866, + "step": 7060 + }, + { + "epoch": 114.03252032520325, + "grad_norm": 1.2696164846420288, + "learning_rate": 0.0001, + "loss": 58.16, + "step": 7070 + }, + { + "epoch": 114.1951219512195, + "grad_norm": 0.9323938488960266, + "learning_rate": 0.0001, + "loss": 58.9483, + "step": 7080 + }, + { + "epoch": 114.35772357723577, + "grad_norm": 1.4123785495758057, + "learning_rate": 0.0001, + "loss": 60.4666, + "step": 7090 + }, + { + "epoch": 114.52032520325203, + "grad_norm": 1.009020447731018, + "learning_rate": 0.0001, + "loss": 58.7152, + "step": 7100 + }, + { + "epoch": 114.6829268292683, + "grad_norm": 1.219383955001831, + "learning_rate": 0.0001, + "loss": 59.1494, + "step": 7110 + }, + { + "epoch": 114.84552845528455, + "grad_norm": 0.9579033255577087, + "learning_rate": 0.0001, + "loss": 58.9879, + "step": 7120 + }, + { + "epoch": 115.0, + "grad_norm": 0.899074912071228, + "learning_rate": 0.0001, + "loss": 55.2031, + "step": 7130 + }, + { + "epoch": 115.16260162601625, + "grad_norm": 0.9924830794334412, + "learning_rate": 0.0001, + "loss": 58.583, + "step": 7140 + }, + { + "epoch": 115.32520325203252, + "grad_norm": 1.0280295610427856, + "learning_rate": 0.0001, + "loss": 60.0948, + "step": 7150 + }, + { + "epoch": 115.48780487804878, + "grad_norm": 0.8108797669410706, + "learning_rate": 0.0001, + "loss": 60.9433, + "step": 7160 + }, + { + "epoch": 115.65040650406505, + "grad_norm": 1.2597944736480713, + "learning_rate": 0.0001, + "loss": 59.1905, + "step": 7170 + }, + { + "epoch": 115.8130081300813, + "grad_norm": 1.2565767765045166, + "learning_rate": 0.0001, + "loss": 59.3545, + "step": 7180 + }, + { + "epoch": 115.97560975609755, + "grad_norm": 1.2012240886688232, + "learning_rate": 0.0001, + "loss": 58.348, + "step": 7190 + }, + { + "epoch": 116.130081300813, + "grad_norm": 1.2883665561676025, + "learning_rate": 0.0001, + "loss": 57.462, + "step": 7200 + }, + { + "epoch": 116.29268292682927, + "grad_norm": 0.904152512550354, + "learning_rate": 0.0001, + "loss": 58.958, + "step": 7210 + }, + { + "epoch": 116.45528455284553, + "grad_norm": 1.1928468942642212, + "learning_rate": 0.0001, + "loss": 58.6106, + "step": 7220 + }, + { + "epoch": 116.6178861788618, + "grad_norm": 1.0655683279037476, + "learning_rate": 0.0001, + "loss": 58.4948, + "step": 7230 + }, + { + "epoch": 116.78048780487805, + "grad_norm": 1.0527434349060059, + "learning_rate": 0.0001, + "loss": 60.1771, + "step": 7240 + }, + { + "epoch": 116.9430894308943, + "grad_norm": 1.1439871788024902, + "learning_rate": 0.0001, + "loss": 58.6737, + "step": 7250 + }, + { + "epoch": 117.09756097560975, + "grad_norm": 1.506571650505066, + "learning_rate": 0.0001, + "loss": 58.0901, + "step": 7260 + }, + { + "epoch": 117.26016260162602, + "grad_norm": 1.1419955492019653, + "learning_rate": 0.0001, + "loss": 59.0705, + "step": 7270 + }, + { + "epoch": 117.42276422764228, + "grad_norm": 1.1211196184158325, + "learning_rate": 0.0001, + "loss": 57.801, + "step": 7280 + }, + { + "epoch": 117.58536585365853, + "grad_norm": 1.1424155235290527, + "learning_rate": 0.0001, + "loss": 59.7449, + "step": 7290 + }, + { + "epoch": 117.7479674796748, + "grad_norm": 1.0593281984329224, + "learning_rate": 0.0001, + "loss": 60.1789, + "step": 7300 + }, + { + "epoch": 117.91056910569105, + "grad_norm": 1.3680177927017212, + "learning_rate": 0.0001, + "loss": 58.4225, + "step": 7310 + }, + { + "epoch": 118.0650406504065, + "grad_norm": 0.8391700983047485, + "learning_rate": 0.0001, + "loss": 56.2102, + "step": 7320 + }, + { + "epoch": 118.22764227642277, + "grad_norm": 1.3166592121124268, + "learning_rate": 0.0001, + "loss": 61.0399, + "step": 7330 + }, + { + "epoch": 118.39024390243902, + "grad_norm": 1.929949402809143, + "learning_rate": 0.0001, + "loss": 59.6514, + "step": 7340 + }, + { + "epoch": 118.55284552845528, + "grad_norm": 0.9464254975318909, + "learning_rate": 0.0001, + "loss": 58.6488, + "step": 7350 + }, + { + "epoch": 118.71544715447155, + "grad_norm": 1.1066193580627441, + "learning_rate": 0.0001, + "loss": 61.5959, + "step": 7360 + }, + { + "epoch": 118.8780487804878, + "grad_norm": 1.216125726699829, + "learning_rate": 0.0001, + "loss": 58.8231, + "step": 7370 + }, + { + "epoch": 119.03252032520325, + "grad_norm": 1.1182583570480347, + "learning_rate": 0.0001, + "loss": 56.9496, + "step": 7380 + }, + { + "epoch": 119.1951219512195, + "grad_norm": 1.105729341506958, + "learning_rate": 0.0001, + "loss": 58.4291, + "step": 7390 + }, + { + "epoch": 119.35772357723577, + "grad_norm": 1.017879843711853, + "learning_rate": 0.0001, + "loss": 59.4963, + "step": 7400 + }, + { + "epoch": 119.52032520325203, + "grad_norm": 0.7882587909698486, + "learning_rate": 0.0001, + "loss": 57.7243, + "step": 7410 + }, + { + "epoch": 119.6829268292683, + "grad_norm": 0.8356695175170898, + "learning_rate": 0.0001, + "loss": 60.4962, + "step": 7420 + }, + { + "epoch": 119.84552845528455, + "grad_norm": 1.1358702182769775, + "learning_rate": 0.0001, + "loss": 60.7173, + "step": 7430 + }, + { + "epoch": 120.0, + "grad_norm": 1.116322636604309, + "learning_rate": 0.0001, + "loss": 53.8387, + "step": 7440 + }, + { + "epoch": 120.16260162601625, + "grad_norm": 0.8522642254829407, + "learning_rate": 0.0001, + "loss": 57.9095, + "step": 7450 + }, + { + "epoch": 120.32520325203252, + "grad_norm": 0.890731692314148, + "learning_rate": 0.0001, + "loss": 57.2296, + "step": 7460 + }, + { + "epoch": 120.48780487804878, + "grad_norm": 1.6366336345672607, + "learning_rate": 0.0001, + "loss": 58.9417, + "step": 7470 + }, + { + "epoch": 120.65040650406505, + "grad_norm": 0.9774733781814575, + "learning_rate": 0.0001, + "loss": 58.8446, + "step": 7480 + }, + { + "epoch": 120.8130081300813, + "grad_norm": 1.6188803911209106, + "learning_rate": 0.0001, + "loss": 59.9794, + "step": 7490 + }, + { + "epoch": 120.97560975609755, + "grad_norm": 1.146599292755127, + "learning_rate": 0.0001, + "loss": 59.8794, + "step": 7500 + }, + { + "epoch": 121.130081300813, + "grad_norm": 0.9472863078117371, + "learning_rate": 0.0001, + "loss": 55.2454, + "step": 7510 + }, + { + "epoch": 121.29268292682927, + "grad_norm": 0.8604205250740051, + "learning_rate": 0.0001, + "loss": 57.6314, + "step": 7520 + }, + { + "epoch": 121.45528455284553, + "grad_norm": 1.0755794048309326, + "learning_rate": 0.0001, + "loss": 60.909, + "step": 7530 + }, + { + "epoch": 121.6178861788618, + "grad_norm": 0.7967071533203125, + "learning_rate": 0.0001, + "loss": 60.1755, + "step": 7540 + }, + { + "epoch": 121.78048780487805, + "grad_norm": 1.037655234336853, + "learning_rate": 0.0001, + "loss": 59.283, + "step": 7550 + }, + { + "epoch": 121.9430894308943, + "grad_norm": 1.070142388343811, + "learning_rate": 0.0001, + "loss": 57.0941, + "step": 7560 + }, + { + "epoch": 122.09756097560975, + "grad_norm": 1.4201022386550903, + "learning_rate": 0.0001, + "loss": 56.2891, + "step": 7570 + }, + { + "epoch": 122.26016260162602, + "grad_norm": 1.0909277200698853, + "learning_rate": 0.0001, + "loss": 57.5213, + "step": 7580 + }, + { + "epoch": 122.42276422764228, + "grad_norm": 0.7573383450508118, + "learning_rate": 0.0001, + "loss": 59.0847, + "step": 7590 + }, + { + "epoch": 122.58536585365853, + "grad_norm": 0.8931725025177002, + "learning_rate": 0.0001, + "loss": 58.9706, + "step": 7600 + }, + { + "epoch": 122.7479674796748, + "grad_norm": 0.9459935426712036, + "learning_rate": 0.0001, + "loss": 58.6267, + "step": 7610 + }, + { + "epoch": 122.91056910569105, + "grad_norm": 0.9463319778442383, + "learning_rate": 0.0001, + "loss": 60.8536, + "step": 7620 + }, + { + "epoch": 123.0650406504065, + "grad_norm": 0.9286382794380188, + "learning_rate": 0.0001, + "loss": 55.2711, + "step": 7630 + }, + { + "epoch": 123.22764227642277, + "grad_norm": 0.9317494630813599, + "learning_rate": 0.0001, + "loss": 60.1578, + "step": 7640 + }, + { + "epoch": 123.39024390243902, + "grad_norm": 0.9834237098693848, + "learning_rate": 0.0001, + "loss": 58.8121, + "step": 7650 + }, + { + "epoch": 123.55284552845528, + "grad_norm": 0.9992889761924744, + "learning_rate": 0.0001, + "loss": 59.8061, + "step": 7660 + }, + { + "epoch": 123.71544715447155, + "grad_norm": 1.0354453325271606, + "learning_rate": 0.0001, + "loss": 59.5853, + "step": 7670 + }, + { + "epoch": 123.8780487804878, + "grad_norm": 1.0609121322631836, + "learning_rate": 0.0001, + "loss": 59.6556, + "step": 7680 + }, + { + "epoch": 124.03252032520325, + "grad_norm": 0.9742912650108337, + "learning_rate": 0.0001, + "loss": 57.2294, + "step": 7690 + }, + { + "epoch": 124.1951219512195, + "grad_norm": 0.9758710265159607, + "learning_rate": 0.0001, + "loss": 59.8581, + "step": 7700 + }, + { + "epoch": 124.35772357723577, + "grad_norm": 0.9492778182029724, + "learning_rate": 0.0001, + "loss": 60.0624, + "step": 7710 + }, + { + "epoch": 124.52032520325203, + "grad_norm": 0.8311835527420044, + "learning_rate": 0.0001, + "loss": 60.0849, + "step": 7720 + }, + { + "epoch": 124.6829268292683, + "grad_norm": 0.8652944564819336, + "learning_rate": 0.0001, + "loss": 58.2321, + "step": 7730 + }, + { + "epoch": 124.84552845528455, + "grad_norm": 1.2980835437774658, + "learning_rate": 0.0001, + "loss": 57.5315, + "step": 7740 + }, + { + "epoch": 125.0, + "grad_norm": 1.0550237894058228, + "learning_rate": 0.0001, + "loss": 56.9615, + "step": 7750 + }, + { + "epoch": 125.16260162601625, + "grad_norm": 1.1248353719711304, + "learning_rate": 0.0001, + "loss": 58.4853, + "step": 7760 + }, + { + "epoch": 125.32520325203252, + "grad_norm": 1.338152289390564, + "learning_rate": 0.0001, + "loss": 57.8604, + "step": 7770 + }, + { + "epoch": 125.48780487804878, + "grad_norm": 0.8162221908569336, + "learning_rate": 0.0001, + "loss": 56.6396, + "step": 7780 + }, + { + "epoch": 125.65040650406505, + "grad_norm": 1.0431848764419556, + "learning_rate": 0.0001, + "loss": 58.9896, + "step": 7790 + }, + { + "epoch": 125.8130081300813, + "grad_norm": 0.848052978515625, + "learning_rate": 0.0001, + "loss": 57.7459, + "step": 7800 + }, + { + "epoch": 125.97560975609755, + "grad_norm": 0.8441061973571777, + "learning_rate": 0.0001, + "loss": 57.9488, + "step": 7810 + }, + { + "epoch": 126.130081300813, + "grad_norm": 0.774566650390625, + "learning_rate": 0.0001, + "loss": 55.9949, + "step": 7820 + }, + { + "epoch": 126.29268292682927, + "grad_norm": 1.178572416305542, + "learning_rate": 0.0001, + "loss": 58.3554, + "step": 7830 + }, + { + "epoch": 126.45528455284553, + "grad_norm": 0.9363889098167419, + "learning_rate": 0.0001, + "loss": 60.6901, + "step": 7840 + }, + { + "epoch": 126.6178861788618, + "grad_norm": 1.0010961294174194, + "learning_rate": 0.0001, + "loss": 57.0078, + "step": 7850 + }, + { + "epoch": 126.78048780487805, + "grad_norm": 0.9573602080345154, + "learning_rate": 0.0001, + "loss": 58.0567, + "step": 7860 + }, + { + "epoch": 126.9430894308943, + "grad_norm": 1.1903672218322754, + "learning_rate": 0.0001, + "loss": 58.3699, + "step": 7870 + }, + { + "epoch": 127.09756097560975, + "grad_norm": 1.2158106565475464, + "learning_rate": 0.0001, + "loss": 57.1154, + "step": 7880 + }, + { + "epoch": 127.26016260162602, + "grad_norm": 1.198789119720459, + "learning_rate": 0.0001, + "loss": 59.1004, + "step": 7890 + }, + { + "epoch": 127.42276422764228, + "grad_norm": 0.8893532752990723, + "learning_rate": 0.0001, + "loss": 59.6481, + "step": 7900 + }, + { + "epoch": 127.58536585365853, + "grad_norm": 0.9834772348403931, + "learning_rate": 0.0001, + "loss": 57.4394, + "step": 7910 + }, + { + "epoch": 127.7479674796748, + "grad_norm": 1.1957768201828003, + "learning_rate": 0.0001, + "loss": 59.1622, + "step": 7920 + }, + { + "epoch": 127.91056910569105, + "grad_norm": 0.822515070438385, + "learning_rate": 0.0001, + "loss": 58.4967, + "step": 7930 + }, + { + "epoch": 128.0650406504065, + "grad_norm": 1.0374864339828491, + "learning_rate": 0.0001, + "loss": 53.9391, + "step": 7940 + }, + { + "epoch": 128.22764227642276, + "grad_norm": 1.1767654418945312, + "learning_rate": 0.0001, + "loss": 59.4305, + "step": 7950 + }, + { + "epoch": 128.390243902439, + "grad_norm": 1.0048503875732422, + "learning_rate": 0.0001, + "loss": 59.7954, + "step": 7960 + }, + { + "epoch": 128.5528455284553, + "grad_norm": 0.9039503931999207, + "learning_rate": 0.0001, + "loss": 58.6945, + "step": 7970 + }, + { + "epoch": 128.71544715447155, + "grad_norm": 1.047418236732483, + "learning_rate": 0.0001, + "loss": 58.6324, + "step": 7980 + }, + { + "epoch": 128.8780487804878, + "grad_norm": 1.7829573154449463, + "learning_rate": 0.0001, + "loss": 57.3699, + "step": 7990 + }, + { + "epoch": 129.03252032520325, + "grad_norm": 0.8874698281288147, + "learning_rate": 0.0001, + "loss": 57.8226, + "step": 8000 + }, + { + "epoch": 129.1951219512195, + "grad_norm": 0.8585357069969177, + "learning_rate": 0.0001, + "loss": 59.1828, + "step": 8010 + }, + { + "epoch": 129.35772357723576, + "grad_norm": 0.7905164957046509, + "learning_rate": 0.0001, + "loss": 58.4257, + "step": 8020 + }, + { + "epoch": 129.52032520325204, + "grad_norm": 1.15513014793396, + "learning_rate": 0.0001, + "loss": 57.712, + "step": 8030 + }, + { + "epoch": 129.6829268292683, + "grad_norm": 1.4509782791137695, + "learning_rate": 0.0001, + "loss": 59.965, + "step": 8040 + }, + { + "epoch": 129.84552845528455, + "grad_norm": 0.8998057246208191, + "learning_rate": 0.0001, + "loss": 58.6614, + "step": 8050 + }, + { + "epoch": 130.0, + "grad_norm": 1.3381586074829102, + "learning_rate": 0.0001, + "loss": 55.2756, + "step": 8060 + }, + { + "epoch": 130.16260162601625, + "grad_norm": 0.9927449226379395, + "learning_rate": 0.0001, + "loss": 59.6809, + "step": 8070 + }, + { + "epoch": 130.3252032520325, + "grad_norm": 0.9900999665260315, + "learning_rate": 0.0001, + "loss": 60.0017, + "step": 8080 + }, + { + "epoch": 130.4878048780488, + "grad_norm": 0.8317362070083618, + "learning_rate": 0.0001, + "loss": 58.2334, + "step": 8090 + }, + { + "epoch": 130.65040650406505, + "grad_norm": 1.0413036346435547, + "learning_rate": 0.0001, + "loss": 58.987, + "step": 8100 + }, + { + "epoch": 130.8130081300813, + "grad_norm": 1.0915170907974243, + "learning_rate": 0.0001, + "loss": 58.9502, + "step": 8110 + }, + { + "epoch": 130.97560975609755, + "grad_norm": 0.9717564582824707, + "learning_rate": 0.0001, + "loss": 58.6578, + "step": 8120 + }, + { + "epoch": 131.130081300813, + "grad_norm": 0.9847933650016785, + "learning_rate": 0.0001, + "loss": 55.0067, + "step": 8130 + }, + { + "epoch": 131.29268292682926, + "grad_norm": 1.2512580156326294, + "learning_rate": 0.0001, + "loss": 56.0223, + "step": 8140 + }, + { + "epoch": 131.45528455284554, + "grad_norm": 0.8859831094741821, + "learning_rate": 0.0001, + "loss": 57.8123, + "step": 8150 + }, + { + "epoch": 131.6178861788618, + "grad_norm": 1.1545639038085938, + "learning_rate": 0.0001, + "loss": 60.2219, + "step": 8160 + }, + { + "epoch": 131.78048780487805, + "grad_norm": 1.3861064910888672, + "learning_rate": 0.0001, + "loss": 60.2156, + "step": 8170 + }, + { + "epoch": 131.9430894308943, + "grad_norm": 1.2621673345565796, + "learning_rate": 0.0001, + "loss": 58.6422, + "step": 8180 + }, + { + "epoch": 132.09756097560975, + "grad_norm": 1.2075496912002563, + "learning_rate": 0.0001, + "loss": 55.4202, + "step": 8190 + }, + { + "epoch": 132.260162601626, + "grad_norm": 1.1960432529449463, + "learning_rate": 0.0001, + "loss": 59.6405, + "step": 8200 + }, + { + "epoch": 132.4227642276423, + "grad_norm": 1.683369755744934, + "learning_rate": 0.0001, + "loss": 60.1361, + "step": 8210 + }, + { + "epoch": 132.58536585365854, + "grad_norm": 0.9033818244934082, + "learning_rate": 0.0001, + "loss": 58.2383, + "step": 8220 + }, + { + "epoch": 132.7479674796748, + "grad_norm": 1.0529452562332153, + "learning_rate": 0.0001, + "loss": 58.9644, + "step": 8230 + }, + { + "epoch": 132.91056910569105, + "grad_norm": 1.0898828506469727, + "learning_rate": 0.0001, + "loss": 57.0898, + "step": 8240 + }, + { + "epoch": 133.0650406504065, + "grad_norm": 1.0203288793563843, + "learning_rate": 0.0001, + "loss": 56.8206, + "step": 8250 + }, + { + "epoch": 133.22764227642276, + "grad_norm": 0.9973819255828857, + "learning_rate": 0.0001, + "loss": 59.2097, + "step": 8260 + }, + { + "epoch": 133.390243902439, + "grad_norm": 0.9728502035140991, + "learning_rate": 0.0001, + "loss": 57.9959, + "step": 8270 + }, + { + "epoch": 133.5528455284553, + "grad_norm": 0.8645833134651184, + "learning_rate": 0.0001, + "loss": 58.9452, + "step": 8280 + }, + { + "epoch": 133.71544715447155, + "grad_norm": 1.0179742574691772, + "learning_rate": 0.0001, + "loss": 57.8577, + "step": 8290 + }, + { + "epoch": 133.8780487804878, + "grad_norm": 0.8542449474334717, + "learning_rate": 0.0001, + "loss": 57.5798, + "step": 8300 + }, + { + "epoch": 134.03252032520325, + "grad_norm": 0.9958596229553223, + "learning_rate": 0.0001, + "loss": 56.6277, + "step": 8310 + }, + { + "epoch": 134.1951219512195, + "grad_norm": 1.1476788520812988, + "learning_rate": 0.0001, + "loss": 60.8397, + "step": 8320 + }, + { + "epoch": 134.35772357723576, + "grad_norm": 0.9098069071769714, + "learning_rate": 0.0001, + "loss": 59.6647, + "step": 8330 + }, + { + "epoch": 134.52032520325204, + "grad_norm": 0.8126556277275085, + "learning_rate": 0.0001, + "loss": 57.147, + "step": 8340 + }, + { + "epoch": 134.6829268292683, + "grad_norm": 1.4846469163894653, + "learning_rate": 0.0001, + "loss": 58.1245, + "step": 8350 + }, + { + "epoch": 134.84552845528455, + "grad_norm": 1.0117210149765015, + "learning_rate": 0.0001, + "loss": 58.7183, + "step": 8360 + }, + { + "epoch": 135.0, + "grad_norm": 0.9517931938171387, + "learning_rate": 0.0001, + "loss": 57.0423, + "step": 8370 + }, + { + "epoch": 135.16260162601625, + "grad_norm": 1.0269144773483276, + "learning_rate": 0.0001, + "loss": 59.6931, + "step": 8380 + }, + { + "epoch": 135.3252032520325, + "grad_norm": 1.1544915437698364, + "learning_rate": 0.0001, + "loss": 57.4079, + "step": 8390 + }, + { + "epoch": 135.4878048780488, + "grad_norm": 1.1180702447891235, + "learning_rate": 0.0001, + "loss": 57.855, + "step": 8400 + }, + { + "epoch": 135.65040650406505, + "grad_norm": 1.2864829301834106, + "learning_rate": 0.0001, + "loss": 57.677, + "step": 8410 + }, + { + "epoch": 135.8130081300813, + "grad_norm": 1.1786772012710571, + "learning_rate": 0.0001, + "loss": 59.5238, + "step": 8420 + }, + { + "epoch": 135.97560975609755, + "grad_norm": 1.8596229553222656, + "learning_rate": 0.0001, + "loss": 57.1596, + "step": 8430 + }, + { + "epoch": 136.130081300813, + "grad_norm": 0.8087555170059204, + "learning_rate": 0.0001, + "loss": 55.8342, + "step": 8440 + }, + { + "epoch": 136.29268292682926, + "grad_norm": 0.8107356429100037, + "learning_rate": 0.0001, + "loss": 58.5965, + "step": 8450 + }, + { + "epoch": 136.45528455284554, + "grad_norm": 1.2416011095046997, + "learning_rate": 0.0001, + "loss": 57.5909, + "step": 8460 + }, + { + "epoch": 136.6178861788618, + "grad_norm": 0.9763500690460205, + "learning_rate": 0.0001, + "loss": 58.1822, + "step": 8470 + }, + { + "epoch": 136.78048780487805, + "grad_norm": 1.1418349742889404, + "learning_rate": 0.0001, + "loss": 59.3942, + "step": 8480 + }, + { + "epoch": 136.9430894308943, + "grad_norm": 1.206878900527954, + "learning_rate": 0.0001, + "loss": 58.4128, + "step": 8490 + }, + { + "epoch": 137.09756097560975, + "grad_norm": 0.861142098903656, + "learning_rate": 0.0001, + "loss": 55.8831, + "step": 8500 + }, + { + "epoch": 137.260162601626, + "grad_norm": 1.0507313013076782, + "learning_rate": 0.0001, + "loss": 59.755, + "step": 8510 + }, + { + "epoch": 137.4227642276423, + "grad_norm": 0.7070153951644897, + "learning_rate": 0.0001, + "loss": 55.6448, + "step": 8520 + }, + { + "epoch": 137.58536585365854, + "grad_norm": 0.9338926672935486, + "learning_rate": 0.0001, + "loss": 58.4242, + "step": 8530 + }, + { + "epoch": 137.7479674796748, + "grad_norm": 1.0600488185882568, + "learning_rate": 0.0001, + "loss": 59.4947, + "step": 8540 + }, + { + "epoch": 137.91056910569105, + "grad_norm": 1.069604754447937, + "learning_rate": 0.0001, + "loss": 57.5164, + "step": 8550 + }, + { + "epoch": 138.0650406504065, + "grad_norm": 0.872332751750946, + "learning_rate": 0.0001, + "loss": 56.453, + "step": 8560 + }, + { + "epoch": 138.22764227642276, + "grad_norm": 0.9670246243476868, + "learning_rate": 0.0001, + "loss": 59.6467, + "step": 8570 + }, + { + "epoch": 138.390243902439, + "grad_norm": 1.1045281887054443, + "learning_rate": 0.0001, + "loss": 59.3334, + "step": 8580 + }, + { + "epoch": 138.5528455284553, + "grad_norm": 1.0553908348083496, + "learning_rate": 0.0001, + "loss": 59.9081, + "step": 8590 + }, + { + "epoch": 138.71544715447155, + "grad_norm": 0.7342197895050049, + "learning_rate": 0.0001, + "loss": 60.2319, + "step": 8600 + }, + { + "epoch": 138.8780487804878, + "grad_norm": 0.9574354290962219, + "learning_rate": 0.0001, + "loss": 56.7984, + "step": 8610 + }, + { + "epoch": 139.03252032520325, + "grad_norm": 0.9238156676292419, + "learning_rate": 0.0001, + "loss": 56.0007, + "step": 8620 + }, + { + "epoch": 139.1951219512195, + "grad_norm": 1.0704458951950073, + "learning_rate": 0.0001, + "loss": 59.4573, + "step": 8630 + }, + { + "epoch": 139.35772357723576, + "grad_norm": 0.8392401337623596, + "learning_rate": 0.0001, + "loss": 57.7589, + "step": 8640 + }, + { + "epoch": 139.52032520325204, + "grad_norm": 0.9042978286743164, + "learning_rate": 0.0001, + "loss": 59.0548, + "step": 8650 + }, + { + "epoch": 139.6829268292683, + "grad_norm": 2.024853467941284, + "learning_rate": 0.0001, + "loss": 58.0019, + "step": 8660 + }, + { + "epoch": 139.84552845528455, + "grad_norm": 1.9823323488235474, + "learning_rate": 0.0001, + "loss": 57.4398, + "step": 8670 + }, + { + "epoch": 140.0, + "grad_norm": 1.626451849937439, + "learning_rate": 0.0001, + "loss": 54.969, + "step": 8680 + }, + { + "epoch": 140.16260162601625, + "grad_norm": 2.435973644256592, + "learning_rate": 0.0001, + "loss": 57.8945, + "step": 8690 + }, + { + "epoch": 140.3252032520325, + "grad_norm": 1.0417332649230957, + "learning_rate": 0.0001, + "loss": 59.9172, + "step": 8700 + }, + { + "epoch": 140.4878048780488, + "grad_norm": 1.0246574878692627, + "learning_rate": 0.0001, + "loss": 57.7337, + "step": 8710 + }, + { + "epoch": 140.65040650406505, + "grad_norm": 0.9492414593696594, + "learning_rate": 0.0001, + "loss": 57.6049, + "step": 8720 + }, + { + "epoch": 140.8130081300813, + "grad_norm": 1.054689645767212, + "learning_rate": 0.0001, + "loss": 56.4394, + "step": 8730 + }, + { + "epoch": 140.97560975609755, + "grad_norm": 0.7142108082771301, + "learning_rate": 0.0001, + "loss": 58.0006, + "step": 8740 + }, + { + "epoch": 141.130081300813, + "grad_norm": 1.1387385129928589, + "learning_rate": 0.0001, + "loss": 55.2639, + "step": 8750 + }, + { + "epoch": 141.29268292682926, + "grad_norm": 2.105823040008545, + "learning_rate": 0.0001, + "loss": 59.3938, + "step": 8760 + }, + { + "epoch": 141.45528455284554, + "grad_norm": 0.9188179969787598, + "learning_rate": 0.0001, + "loss": 58.731, + "step": 8770 + }, + { + "epoch": 141.6178861788618, + "grad_norm": 1.043778896331787, + "learning_rate": 0.0001, + "loss": 56.2486, + "step": 8780 + }, + { + "epoch": 141.78048780487805, + "grad_norm": 1.438591718673706, + "learning_rate": 0.0001, + "loss": 58.4228, + "step": 8790 + }, + { + "epoch": 141.9430894308943, + "grad_norm": 1.116957187652588, + "learning_rate": 0.0001, + "loss": 59.2317, + "step": 8800 + }, + { + "epoch": 142.09756097560975, + "grad_norm": 0.8644933104515076, + "learning_rate": 0.0001, + "loss": 55.3905, + "step": 8810 + }, + { + "epoch": 142.260162601626, + "grad_norm": 1.0249485969543457, + "learning_rate": 0.0001, + "loss": 59.0372, + "step": 8820 + }, + { + "epoch": 142.4227642276423, + "grad_norm": 0.9939082264900208, + "learning_rate": 0.0001, + "loss": 57.3947, + "step": 8830 + }, + { + "epoch": 142.58536585365854, + "grad_norm": 0.9836716055870056, + "learning_rate": 0.0001, + "loss": 57.9229, + "step": 8840 + }, + { + "epoch": 142.7479674796748, + "grad_norm": 0.9596876502037048, + "learning_rate": 0.0001, + "loss": 57.0285, + "step": 8850 + }, + { + "epoch": 142.91056910569105, + "grad_norm": 1.052899956703186, + "learning_rate": 0.0001, + "loss": 60.1384, + "step": 8860 + }, + { + "epoch": 143.0650406504065, + "grad_norm": 1.1220297813415527, + "learning_rate": 0.0001, + "loss": 55.3928, + "step": 8870 + }, + { + "epoch": 143.22764227642276, + "grad_norm": 0.8395596742630005, + "learning_rate": 0.0001, + "loss": 57.8185, + "step": 8880 + }, + { + "epoch": 143.390243902439, + "grad_norm": 1.1940062046051025, + "learning_rate": 0.0001, + "loss": 58.0322, + "step": 8890 + }, + { + "epoch": 143.5528455284553, + "grad_norm": 1.1662484407424927, + "learning_rate": 0.0001, + "loss": 58.2746, + "step": 8900 + }, + { + "epoch": 143.71544715447155, + "grad_norm": 0.9028235077857971, + "learning_rate": 0.0001, + "loss": 56.5817, + "step": 8910 + }, + { + "epoch": 143.8780487804878, + "grad_norm": 1.0597296953201294, + "learning_rate": 0.0001, + "loss": 56.8426, + "step": 8920 + }, + { + "epoch": 144.03252032520325, + "grad_norm": 0.9968701004981995, + "learning_rate": 0.0001, + "loss": 54.9727, + "step": 8930 + }, + { + "epoch": 144.1951219512195, + "grad_norm": 0.9306491613388062, + "learning_rate": 0.0001, + "loss": 56.5321, + "step": 8940 + }, + { + "epoch": 144.35772357723576, + "grad_norm": 1.1079609394073486, + "learning_rate": 0.0001, + "loss": 56.9714, + "step": 8950 + }, + { + "epoch": 144.52032520325204, + "grad_norm": 0.9644935131072998, + "learning_rate": 0.0001, + "loss": 58.613, + "step": 8960 + }, + { + "epoch": 144.6829268292683, + "grad_norm": 1.2037341594696045, + "learning_rate": 0.0001, + "loss": 56.7768, + "step": 8970 + }, + { + "epoch": 144.84552845528455, + "grad_norm": 0.7347294092178345, + "learning_rate": 0.0001, + "loss": 58.5653, + "step": 8980 + }, + { + "epoch": 145.0, + "grad_norm": 0.8877835273742676, + "learning_rate": 0.0001, + "loss": 53.7768, + "step": 8990 + }, + { + "epoch": 145.16260162601625, + "grad_norm": 0.9649038910865784, + "learning_rate": 0.0001, + "loss": 57.7391, + "step": 9000 + }, + { + "epoch": 145.3252032520325, + "grad_norm": 1.1804879903793335, + "learning_rate": 0.0001, + "loss": 60.3727, + "step": 9010 + }, + { + "epoch": 145.4878048780488, + "grad_norm": 0.8406301140785217, + "learning_rate": 0.0001, + "loss": 57.9417, + "step": 9020 + }, + { + "epoch": 145.65040650406505, + "grad_norm": 0.9555912613868713, + "learning_rate": 0.0001, + "loss": 58.0936, + "step": 9030 + }, + { + "epoch": 145.8130081300813, + "grad_norm": 1.20796537399292, + "learning_rate": 0.0001, + "loss": 57.0784, + "step": 9040 + }, + { + "epoch": 145.97560975609755, + "grad_norm": 1.0527915954589844, + "learning_rate": 0.0001, + "loss": 56.752, + "step": 9050 + }, + { + "epoch": 146.130081300813, + "grad_norm": 1.0104299783706665, + "learning_rate": 0.0001, + "loss": 56.2158, + "step": 9060 + }, + { + "epoch": 146.29268292682926, + "grad_norm": 1.4884357452392578, + "learning_rate": 0.0001, + "loss": 57.8631, + "step": 9070 + }, + { + "epoch": 146.45528455284554, + "grad_norm": 0.9095257520675659, + "learning_rate": 0.0001, + "loss": 57.6492, + "step": 9080 + }, + { + "epoch": 146.6178861788618, + "grad_norm": 0.8465835452079773, + "learning_rate": 0.0001, + "loss": 57.78, + "step": 9090 + }, + { + "epoch": 146.78048780487805, + "grad_norm": 1.0254603624343872, + "learning_rate": 0.0001, + "loss": 57.7678, + "step": 9100 + }, + { + "epoch": 146.9430894308943, + "grad_norm": 1.0137311220169067, + "learning_rate": 0.0001, + "loss": 58.2813, + "step": 9110 + }, + { + "epoch": 147.09756097560975, + "grad_norm": 0.9816886186599731, + "learning_rate": 0.0001, + "loss": 54.4789, + "step": 9120 + }, + { + "epoch": 147.260162601626, + "grad_norm": 0.9038514494895935, + "learning_rate": 0.0001, + "loss": 58.5366, + "step": 9130 + }, + { + "epoch": 147.4227642276423, + "grad_norm": 0.7649261355400085, + "learning_rate": 0.0001, + "loss": 57.2898, + "step": 9140 + }, + { + "epoch": 147.58536585365854, + "grad_norm": 0.8841894865036011, + "learning_rate": 0.0001, + "loss": 57.9109, + "step": 9150 + }, + { + "epoch": 147.7479674796748, + "grad_norm": 1.0480542182922363, + "learning_rate": 0.0001, + "loss": 56.9625, + "step": 9160 + }, + { + "epoch": 147.91056910569105, + "grad_norm": 0.8710455894470215, + "learning_rate": 0.0001, + "loss": 57.1721, + "step": 9170 + }, + { + "epoch": 148.0650406504065, + "grad_norm": 1.374118447303772, + "learning_rate": 0.0001, + "loss": 55.3243, + "step": 9180 + }, + { + "epoch": 148.22764227642276, + "grad_norm": 1.0879178047180176, + "learning_rate": 0.0001, + "loss": 57.1631, + "step": 9190 + }, + { + "epoch": 148.390243902439, + "grad_norm": 0.8242596387863159, + "learning_rate": 0.0001, + "loss": 56.9993, + "step": 9200 + }, + { + "epoch": 148.5528455284553, + "grad_norm": 1.02505362033844, + "learning_rate": 0.0001, + "loss": 57.9201, + "step": 9210 + }, + { + "epoch": 148.71544715447155, + "grad_norm": 0.7904821038246155, + "learning_rate": 0.0001, + "loss": 59.2553, + "step": 9220 + }, + { + "epoch": 148.8780487804878, + "grad_norm": 1.4904934167861938, + "learning_rate": 0.0001, + "loss": 59.498, + "step": 9230 + }, + { + "epoch": 149.03252032520325, + "grad_norm": 1.4203144311904907, + "learning_rate": 0.0001, + "loss": 54.6357, + "step": 9240 + }, + { + "epoch": 149.1951219512195, + "grad_norm": 0.8256629705429077, + "learning_rate": 0.0001, + "loss": 57.5996, + "step": 9250 + }, + { + "epoch": 149.35772357723576, + "grad_norm": 0.8736788630485535, + "learning_rate": 0.0001, + "loss": 56.8112, + "step": 9260 + }, + { + "epoch": 149.52032520325204, + "grad_norm": 3.20312762260437, + "learning_rate": 0.0001, + "loss": 55.8763, + "step": 9270 + }, + { + "epoch": 149.6829268292683, + "grad_norm": 0.7431173920631409, + "learning_rate": 0.0001, + "loss": 56.7385, + "step": 9280 + }, + { + "epoch": 149.84552845528455, + "grad_norm": 0.8165793418884277, + "learning_rate": 0.0001, + "loss": 57.8441, + "step": 9290 + }, + { + "epoch": 150.0, + "grad_norm": 1.2387259006500244, + "learning_rate": 0.0001, + "loss": 53.47, + "step": 9300 + }, + { + "epoch": 150.16260162601625, + "grad_norm": 0.8952508568763733, + "learning_rate": 0.0001, + "loss": 57.7642, + "step": 9310 + }, + { + "epoch": 150.3252032520325, + "grad_norm": 0.9837163686752319, + "learning_rate": 0.0001, + "loss": 57.2288, + "step": 9320 + }, + { + "epoch": 150.4878048780488, + "grad_norm": 0.9019481539726257, + "learning_rate": 0.0001, + "loss": 57.4239, + "step": 9330 + }, + { + "epoch": 150.65040650406505, + "grad_norm": 1.0404818058013916, + "learning_rate": 0.0001, + "loss": 57.7678, + "step": 9340 + }, + { + "epoch": 150.8130081300813, + "grad_norm": 1.0074576139450073, + "learning_rate": 0.0001, + "loss": 58.1508, + "step": 9350 + }, + { + "epoch": 150.97560975609755, + "grad_norm": 0.9951609969139099, + "learning_rate": 0.0001, + "loss": 56.1586, + "step": 9360 + }, + { + "epoch": 151.130081300813, + "grad_norm": 1.0799294710159302, + "learning_rate": 0.0001, + "loss": 52.2597, + "step": 9370 + }, + { + "epoch": 151.29268292682926, + "grad_norm": 0.9001291394233704, + "learning_rate": 0.0001, + "loss": 58.1814, + "step": 9380 + }, + { + "epoch": 151.45528455284554, + "grad_norm": 1.0388239622116089, + "learning_rate": 0.0001, + "loss": 57.1812, + "step": 9390 + }, + { + "epoch": 151.6178861788618, + "grad_norm": 0.7662557363510132, + "learning_rate": 0.0001, + "loss": 56.087, + "step": 9400 + }, + { + "epoch": 151.78048780487805, + "grad_norm": 1.0790860652923584, + "learning_rate": 0.0001, + "loss": 55.9247, + "step": 9410 + }, + { + "epoch": 151.9430894308943, + "grad_norm": 0.8173624277114868, + "learning_rate": 0.0001, + "loss": 57.5506, + "step": 9420 + }, + { + "epoch": 152.09756097560975, + "grad_norm": 1.0954891443252563, + "learning_rate": 0.0001, + "loss": 54.6799, + "step": 9430 + }, + { + "epoch": 152.260162601626, + "grad_norm": 1.0103733539581299, + "learning_rate": 0.0001, + "loss": 58.5454, + "step": 9440 + }, + { + "epoch": 152.4227642276423, + "grad_norm": 0.6966420412063599, + "learning_rate": 0.0001, + "loss": 57.561, + "step": 9450 + }, + { + "epoch": 152.58536585365854, + "grad_norm": 0.9085037708282471, + "learning_rate": 0.0001, + "loss": 57.1746, + "step": 9460 + }, + { + "epoch": 152.7479674796748, + "grad_norm": 1.0009453296661377, + "learning_rate": 0.0001, + "loss": 56.8, + "step": 9470 + }, + { + "epoch": 152.91056910569105, + "grad_norm": 0.9139633178710938, + "learning_rate": 0.0001, + "loss": 56.8229, + "step": 9480 + }, + { + "epoch": 153.0650406504065, + "grad_norm": 1.4209275245666504, + "learning_rate": 0.0001, + "loss": 55.3074, + "step": 9490 + }, + { + "epoch": 153.22764227642276, + "grad_norm": 1.3912067413330078, + "learning_rate": 0.0001, + "loss": 58.1401, + "step": 9500 + }, + { + "epoch": 153.390243902439, + "grad_norm": 0.8208648562431335, + "learning_rate": 0.0001, + "loss": 55.2981, + "step": 9510 + }, + { + "epoch": 153.5528455284553, + "grad_norm": 1.1563907861709595, + "learning_rate": 0.0001, + "loss": 59.0262, + "step": 9520 + }, + { + "epoch": 153.71544715447155, + "grad_norm": 1.1142042875289917, + "learning_rate": 0.0001, + "loss": 55.5979, + "step": 9530 + }, + { + "epoch": 153.8780487804878, + "grad_norm": 0.8886147141456604, + "learning_rate": 0.0001, + "loss": 58.5592, + "step": 9540 + }, + { + "epoch": 154.03252032520325, + "grad_norm": 0.9479674696922302, + "learning_rate": 0.0001, + "loss": 53.956, + "step": 9550 + }, + { + "epoch": 154.1951219512195, + "grad_norm": 1.0572675466537476, + "learning_rate": 0.0001, + "loss": 59.5914, + "step": 9560 + }, + { + "epoch": 154.35772357723576, + "grad_norm": 0.8284531831741333, + "learning_rate": 0.0001, + "loss": 56.4313, + "step": 9570 + }, + { + "epoch": 154.52032520325204, + "grad_norm": 0.8882488012313843, + "learning_rate": 0.0001, + "loss": 57.707, + "step": 9580 + }, + { + "epoch": 154.6829268292683, + "grad_norm": 0.9115352630615234, + "learning_rate": 0.0001, + "loss": 58.0544, + "step": 9590 + }, + { + "epoch": 154.84552845528455, + "grad_norm": 1.0006799697875977, + "learning_rate": 0.0001, + "loss": 59.4528, + "step": 9600 + }, + { + "epoch": 155.0, + "grad_norm": 0.9925333261489868, + "learning_rate": 0.0001, + "loss": 52.977, + "step": 9610 + }, + { + "epoch": 155.16260162601625, + "grad_norm": 1.423391580581665, + "learning_rate": 0.0001, + "loss": 57.0083, + "step": 9620 + }, + { + "epoch": 155.3252032520325, + "grad_norm": 1.0822738409042358, + "learning_rate": 0.0001, + "loss": 58.187, + "step": 9630 + }, + { + "epoch": 155.4878048780488, + "grad_norm": 0.932689905166626, + "learning_rate": 0.0001, + "loss": 55.6032, + "step": 9640 + }, + { + "epoch": 155.65040650406505, + "grad_norm": 0.895504355430603, + "learning_rate": 0.0001, + "loss": 57.1324, + "step": 9650 + }, + { + "epoch": 155.8130081300813, + "grad_norm": 1.0883748531341553, + "learning_rate": 0.0001, + "loss": 58.8422, + "step": 9660 + }, + { + "epoch": 155.97560975609755, + "grad_norm": 1.0161819458007812, + "learning_rate": 0.0001, + "loss": 56.8333, + "step": 9670 + }, + { + "epoch": 156.130081300813, + "grad_norm": 0.9483168721199036, + "learning_rate": 0.0001, + "loss": 52.8434, + "step": 9680 + }, + { + "epoch": 156.29268292682926, + "grad_norm": 1.0677295923233032, + "learning_rate": 0.0001, + "loss": 57.749, + "step": 9690 + }, + { + "epoch": 156.45528455284554, + "grad_norm": 0.8123661875724792, + "learning_rate": 0.0001, + "loss": 56.0896, + "step": 9700 + }, + { + "epoch": 156.6178861788618, + "grad_norm": 0.7926080822944641, + "learning_rate": 0.0001, + "loss": 57.8899, + "step": 9710 + }, + { + "epoch": 156.78048780487805, + "grad_norm": 1.0433725118637085, + "learning_rate": 0.0001, + "loss": 59.6728, + "step": 9720 + }, + { + "epoch": 156.9430894308943, + "grad_norm": 0.8284909129142761, + "learning_rate": 0.0001, + "loss": 55.6093, + "step": 9730 + }, + { + "epoch": 157.09756097560975, + "grad_norm": 0.9702232480049133, + "learning_rate": 0.0001, + "loss": 55.3595, + "step": 9740 + }, + { + "epoch": 157.260162601626, + "grad_norm": 0.8909348249435425, + "learning_rate": 0.0001, + "loss": 57.6344, + "step": 9750 + }, + { + "epoch": 157.4227642276423, + "grad_norm": 0.9539676308631897, + "learning_rate": 0.0001, + "loss": 55.8826, + "step": 9760 + }, + { + "epoch": 157.58536585365854, + "grad_norm": 1.1801178455352783, + "learning_rate": 0.0001, + "loss": 55.7141, + "step": 9770 + }, + { + "epoch": 157.7479674796748, + "grad_norm": 0.8674126863479614, + "learning_rate": 0.0001, + "loss": 56.0571, + "step": 9780 + }, + { + "epoch": 157.91056910569105, + "grad_norm": 1.0690264701843262, + "learning_rate": 0.0001, + "loss": 56.1485, + "step": 9790 + }, + { + "epoch": 158.0650406504065, + "grad_norm": 0.8807869553565979, + "learning_rate": 0.0001, + "loss": 56.9874, + "step": 9800 + }, + { + "epoch": 158.22764227642276, + "grad_norm": 0.9477714896202087, + "learning_rate": 0.0001, + "loss": 58.2035, + "step": 9810 + }, + { + "epoch": 158.390243902439, + "grad_norm": 1.4875513315200806, + "learning_rate": 0.0001, + "loss": 59.0302, + "step": 9820 + }, + { + "epoch": 158.5528455284553, + "grad_norm": 0.7979354858398438, + "learning_rate": 0.0001, + "loss": 57.2801, + "step": 9830 + }, + { + "epoch": 158.71544715447155, + "grad_norm": 0.8941402435302734, + "learning_rate": 0.0001, + "loss": 56.3615, + "step": 9840 + }, + { + "epoch": 158.8780487804878, + "grad_norm": 0.8475911021232605, + "learning_rate": 0.0001, + "loss": 57.266, + "step": 9850 + }, + { + "epoch": 159.03252032520325, + "grad_norm": 0.8971277475357056, + "learning_rate": 0.0001, + "loss": 52.792, + "step": 9860 + }, + { + "epoch": 159.1951219512195, + "grad_norm": 0.930777907371521, + "learning_rate": 0.0001, + "loss": 57.4098, + "step": 9870 + }, + { + "epoch": 159.35772357723576, + "grad_norm": 0.9232887625694275, + "learning_rate": 0.0001, + "loss": 56.0267, + "step": 9880 + }, + { + "epoch": 159.52032520325204, + "grad_norm": 0.8904425501823425, + "learning_rate": 0.0001, + "loss": 57.6255, + "step": 9890 + }, + { + "epoch": 159.6829268292683, + "grad_norm": 0.9250755906105042, + "learning_rate": 0.0001, + "loss": 56.5248, + "step": 9900 + }, + { + "epoch": 159.84552845528455, + "grad_norm": 1.3746178150177002, + "learning_rate": 0.0001, + "loss": 56.7843, + "step": 9910 + }, + { + "epoch": 160.0, + "grad_norm": 0.9464873671531677, + "learning_rate": 0.0001, + "loss": 54.1325, + "step": 9920 + }, + { + "epoch": 160.16260162601625, + "grad_norm": 0.9019768238067627, + "learning_rate": 0.0001, + "loss": 58.3641, + "step": 9930 + }, + { + "epoch": 160.3252032520325, + "grad_norm": 0.9282474517822266, + "learning_rate": 0.0001, + "loss": 57.9474, + "step": 9940 + }, + { + "epoch": 160.4878048780488, + "grad_norm": 1.1150450706481934, + "learning_rate": 0.0001, + "loss": 58.2999, + "step": 9950 + }, + { + "epoch": 160.65040650406505, + "grad_norm": 0.9780807495117188, + "learning_rate": 0.0001, + "loss": 55.6409, + "step": 9960 + }, + { + "epoch": 160.8130081300813, + "grad_norm": 0.9708426594734192, + "learning_rate": 0.0001, + "loss": 58.651, + "step": 9970 + }, + { + "epoch": 160.97560975609755, + "grad_norm": 0.9891226887702942, + "learning_rate": 0.0001, + "loss": 57.3798, + "step": 9980 + }, + { + "epoch": 161.130081300813, + "grad_norm": 0.8949639797210693, + "learning_rate": 0.0001, + "loss": 53.4708, + "step": 9990 + }, + { + "epoch": 161.29268292682926, + "grad_norm": 0.831688404083252, + "learning_rate": 0.0001, + "loss": 57.1608, + "step": 10000 + }, + { + "epoch": 161.45528455284554, + "grad_norm": 1.1002815961837769, + "learning_rate": 0.0001, + "loss": 57.5943, + "step": 10010 + }, + { + "epoch": 161.6178861788618, + "grad_norm": 0.823810338973999, + "learning_rate": 0.0001, + "loss": 56.3465, + "step": 10020 + }, + { + "epoch": 161.78048780487805, + "grad_norm": 0.8624166250228882, + "learning_rate": 0.0001, + "loss": 58.2043, + "step": 10030 + }, + { + "epoch": 161.9430894308943, + "grad_norm": 0.9643419981002808, + "learning_rate": 0.0001, + "loss": 58.6286, + "step": 10040 + }, + { + "epoch": 162.09756097560975, + "grad_norm": 0.8477599024772644, + "learning_rate": 0.0001, + "loss": 54.0391, + "step": 10050 + }, + { + "epoch": 162.260162601626, + "grad_norm": 0.932232677936554, + "learning_rate": 0.0001, + "loss": 57.3955, + "step": 10060 + }, + { + "epoch": 162.4227642276423, + "grad_norm": 0.9273334741592407, + "learning_rate": 0.0001, + "loss": 57.3421, + "step": 10070 + }, + { + "epoch": 162.58536585365854, + "grad_norm": 0.746417224407196, + "learning_rate": 0.0001, + "loss": 56.1001, + "step": 10080 + }, + { + "epoch": 162.7479674796748, + "grad_norm": 0.9870802760124207, + "learning_rate": 0.0001, + "loss": 57.1234, + "step": 10090 + }, + { + "epoch": 162.91056910569105, + "grad_norm": 1.2378058433532715, + "learning_rate": 0.0001, + "loss": 58.035, + "step": 10100 + }, + { + "epoch": 163.0650406504065, + "grad_norm": 0.8810584545135498, + "learning_rate": 0.0001, + "loss": 52.1406, + "step": 10110 + }, + { + "epoch": 163.22764227642276, + "grad_norm": 1.1010875701904297, + "learning_rate": 0.0001, + "loss": 57.2534, + "step": 10120 + }, + { + "epoch": 163.390243902439, + "grad_norm": 1.1182267665863037, + "learning_rate": 0.0001, + "loss": 58.0887, + "step": 10130 + }, + { + "epoch": 163.5528455284553, + "grad_norm": 0.8677687048912048, + "learning_rate": 0.0001, + "loss": 56.3502, + "step": 10140 + }, + { + "epoch": 163.71544715447155, + "grad_norm": 1.3890044689178467, + "learning_rate": 0.0001, + "loss": 57.1057, + "step": 10150 + }, + { + "epoch": 163.8780487804878, + "grad_norm": 1.340419054031372, + "learning_rate": 0.0001, + "loss": 56.9196, + "step": 10160 + }, + { + "epoch": 164.03252032520325, + "grad_norm": 0.8187946081161499, + "learning_rate": 0.0001, + "loss": 53.388, + "step": 10170 + }, + { + "epoch": 164.1951219512195, + "grad_norm": 0.7856727838516235, + "learning_rate": 0.0001, + "loss": 57.0134, + "step": 10180 + }, + { + "epoch": 164.35772357723576, + "grad_norm": 0.8677143454551697, + "learning_rate": 0.0001, + "loss": 56.1385, + "step": 10190 + }, + { + "epoch": 164.52032520325204, + "grad_norm": 0.8334061503410339, + "learning_rate": 0.0001, + "loss": 58.4353, + "step": 10200 + }, + { + "epoch": 164.6829268292683, + "grad_norm": 0.9447886943817139, + "learning_rate": 0.0001, + "loss": 56.2007, + "step": 10210 + }, + { + "epoch": 164.84552845528455, + "grad_norm": 0.7787997722625732, + "learning_rate": 0.0001, + "loss": 56.7128, + "step": 10220 + }, + { + "epoch": 165.0, + "grad_norm": 0.864169716835022, + "learning_rate": 0.0001, + "loss": 53.6559, + "step": 10230 + }, + { + "epoch": 165.16260162601625, + "grad_norm": 0.9252000451087952, + "learning_rate": 0.0001, + "loss": 57.1013, + "step": 10240 + }, + { + "epoch": 165.3252032520325, + "grad_norm": 0.9552235007286072, + "learning_rate": 0.0001, + "loss": 58.1834, + "step": 10250 + }, + { + "epoch": 165.4878048780488, + "grad_norm": 1.0003581047058105, + "learning_rate": 0.0001, + "loss": 56.3756, + "step": 10260 + }, + { + "epoch": 165.65040650406505, + "grad_norm": 1.1489440202713013, + "learning_rate": 0.0001, + "loss": 54.8963, + "step": 10270 + }, + { + "epoch": 165.8130081300813, + "grad_norm": 0.9719933867454529, + "learning_rate": 0.0001, + "loss": 59.2853, + "step": 10280 + }, + { + "epoch": 165.97560975609755, + "grad_norm": 0.888495147228241, + "learning_rate": 0.0001, + "loss": 56.7793, + "step": 10290 + }, + { + "epoch": 166.130081300813, + "grad_norm": 0.793580174446106, + "learning_rate": 0.0001, + "loss": 53.6792, + "step": 10300 + }, + { + "epoch": 166.29268292682926, + "grad_norm": 0.928779125213623, + "learning_rate": 0.0001, + "loss": 57.5329, + "step": 10310 + }, + { + "epoch": 166.45528455284554, + "grad_norm": 1.2024686336517334, + "learning_rate": 0.0001, + "loss": 58.08, + "step": 10320 + }, + { + "epoch": 166.6178861788618, + "grad_norm": 1.421026349067688, + "learning_rate": 0.0001, + "loss": 57.5318, + "step": 10330 + }, + { + "epoch": 166.78048780487805, + "grad_norm": 0.9189603328704834, + "learning_rate": 0.0001, + "loss": 57.1355, + "step": 10340 + }, + { + "epoch": 166.9430894308943, + "grad_norm": 1.0614054203033447, + "learning_rate": 0.0001, + "loss": 56.0604, + "step": 10350 + }, + { + "epoch": 167.09756097560975, + "grad_norm": 1.0266008377075195, + "learning_rate": 0.0001, + "loss": 55.0443, + "step": 10360 + }, + { + "epoch": 167.260162601626, + "grad_norm": 0.8291250467300415, + "learning_rate": 0.0001, + "loss": 57.6836, + "step": 10370 + }, + { + "epoch": 167.4227642276423, + "grad_norm": 1.0415455102920532, + "learning_rate": 0.0001, + "loss": 56.1081, + "step": 10380 + }, + { + "epoch": 167.58536585365854, + "grad_norm": 0.8045644164085388, + "learning_rate": 0.0001, + "loss": 54.8943, + "step": 10390 + }, + { + "epoch": 167.7479674796748, + "grad_norm": 0.9025611281394958, + "learning_rate": 0.0001, + "loss": 56.5143, + "step": 10400 + }, + { + "epoch": 167.91056910569105, + "grad_norm": 0.93849116563797, + "learning_rate": 0.0001, + "loss": 57.4787, + "step": 10410 + }, + { + "epoch": 168.0650406504065, + "grad_norm": 0.8948699235916138, + "learning_rate": 0.0001, + "loss": 54.9249, + "step": 10420 + }, + { + "epoch": 168.22764227642276, + "grad_norm": 0.7900148630142212, + "learning_rate": 0.0001, + "loss": 55.9813, + "step": 10430 + }, + { + "epoch": 168.390243902439, + "grad_norm": 1.1853744983673096, + "learning_rate": 0.0001, + "loss": 54.9089, + "step": 10440 + }, + { + "epoch": 168.5528455284553, + "grad_norm": 1.225123643875122, + "learning_rate": 0.0001, + "loss": 58.238, + "step": 10450 + }, + { + "epoch": 168.71544715447155, + "grad_norm": 0.8648725748062134, + "learning_rate": 0.0001, + "loss": 55.4905, + "step": 10460 + }, + { + "epoch": 168.8780487804878, + "grad_norm": 0.9054272770881653, + "learning_rate": 0.0001, + "loss": 58.2697, + "step": 10470 + }, + { + "epoch": 169.03252032520325, + "grad_norm": 0.946158230304718, + "learning_rate": 0.0001, + "loss": 53.3644, + "step": 10480 + }, + { + "epoch": 169.1951219512195, + "grad_norm": 0.8751044273376465, + "learning_rate": 0.0001, + "loss": 54.3469, + "step": 10490 + }, + { + "epoch": 169.35772357723576, + "grad_norm": 0.9814269542694092, + "learning_rate": 0.0001, + "loss": 56.5358, + "step": 10500 + }, + { + "epoch": 169.52032520325204, + "grad_norm": 0.8131198883056641, + "learning_rate": 0.0001, + "loss": 56.2025, + "step": 10510 + }, + { + "epoch": 169.6829268292683, + "grad_norm": 0.7442125678062439, + "learning_rate": 0.0001, + "loss": 54.7679, + "step": 10520 + }, + { + "epoch": 169.84552845528455, + "grad_norm": 0.8504229187965393, + "learning_rate": 0.0001, + "loss": 59.024, + "step": 10530 + }, + { + "epoch": 170.0, + "grad_norm": 0.8483893871307373, + "learning_rate": 0.0001, + "loss": 53.8021, + "step": 10540 + }, + { + "epoch": 170.16260162601625, + "grad_norm": 0.8889706134796143, + "learning_rate": 0.0001, + "loss": 55.1988, + "step": 10550 + }, + { + "epoch": 170.3252032520325, + "grad_norm": 0.9083985686302185, + "learning_rate": 0.0001, + "loss": 57.092, + "step": 10560 + }, + { + "epoch": 170.4878048780488, + "grad_norm": 27.54094123840332, + "learning_rate": 0.0001, + "loss": 56.945, + "step": 10570 + }, + { + "epoch": 170.65040650406505, + "grad_norm": 0.6707919836044312, + "learning_rate": 0.0001, + "loss": 55.8705, + "step": 10580 + }, + { + "epoch": 170.8130081300813, + "grad_norm": 1.0104289054870605, + "learning_rate": 0.0001, + "loss": 56.6519, + "step": 10590 + }, + { + "epoch": 170.97560975609755, + "grad_norm": 1.221200704574585, + "learning_rate": 0.0001, + "loss": 57.0875, + "step": 10600 + }, + { + "epoch": 171.130081300813, + "grad_norm": 1.3986961841583252, + "learning_rate": 0.0001, + "loss": 53.9815, + "step": 10610 + }, + { + "epoch": 171.29268292682926, + "grad_norm": 1.0881519317626953, + "learning_rate": 0.0001, + "loss": 55.8683, + "step": 10620 + }, + { + "epoch": 171.45528455284554, + "grad_norm": 1.172620415687561, + "learning_rate": 0.0001, + "loss": 56.7678, + "step": 10630 + }, + { + "epoch": 171.6178861788618, + "grad_norm": 0.867314875125885, + "learning_rate": 0.0001, + "loss": 55.0647, + "step": 10640 + }, + { + "epoch": 171.78048780487805, + "grad_norm": 1.415421962738037, + "learning_rate": 0.0001, + "loss": 57.5599, + "step": 10650 + }, + { + "epoch": 171.9430894308943, + "grad_norm": 1.0275741815567017, + "learning_rate": 0.0001, + "loss": 55.4741, + "step": 10660 + }, + { + "epoch": 172.09756097560975, + "grad_norm": 0.9037189483642578, + "learning_rate": 0.0001, + "loss": 51.9268, + "step": 10670 + }, + { + "epoch": 172.260162601626, + "grad_norm": 0.9657711386680603, + "learning_rate": 0.0001, + "loss": 57.3183, + "step": 10680 + }, + { + "epoch": 172.4227642276423, + "grad_norm": 0.9031921029090881, + "learning_rate": 0.0001, + "loss": 58.6986, + "step": 10690 + }, + { + "epoch": 172.58536585365854, + "grad_norm": 0.9555232524871826, + "learning_rate": 0.0001, + "loss": 55.7602, + "step": 10700 + }, + { + "epoch": 172.7479674796748, + "grad_norm": 0.985213041305542, + "learning_rate": 0.0001, + "loss": 55.3734, + "step": 10710 + }, + { + "epoch": 172.91056910569105, + "grad_norm": 1.1574981212615967, + "learning_rate": 0.0001, + "loss": 57.8304, + "step": 10720 + }, + { + "epoch": 173.0650406504065, + "grad_norm": 1.0298277139663696, + "learning_rate": 0.0001, + "loss": 54.2185, + "step": 10730 + }, + { + "epoch": 173.22764227642276, + "grad_norm": 1.0708621740341187, + "learning_rate": 0.0001, + "loss": 57.1618, + "step": 10740 + }, + { + "epoch": 173.390243902439, + "grad_norm": 1.0337039232254028, + "learning_rate": 0.0001, + "loss": 56.8093, + "step": 10750 + }, + { + "epoch": 173.5528455284553, + "grad_norm": 1.1115336418151855, + "learning_rate": 0.0001, + "loss": 56.1233, + "step": 10760 + }, + { + "epoch": 173.71544715447155, + "grad_norm": 0.9465245008468628, + "learning_rate": 0.0001, + "loss": 57.79, + "step": 10770 + }, + { + "epoch": 173.8780487804878, + "grad_norm": 1.0282483100891113, + "learning_rate": 0.0001, + "loss": 56.7261, + "step": 10780 + }, + { + "epoch": 174.03252032520325, + "grad_norm": 0.8371811509132385, + "learning_rate": 0.0001, + "loss": 54.3208, + "step": 10790 + }, + { + "epoch": 174.1951219512195, + "grad_norm": 0.75701504945755, + "learning_rate": 0.0001, + "loss": 58.4083, + "step": 10800 + }, + { + "epoch": 174.35772357723576, + "grad_norm": 0.8716368675231934, + "learning_rate": 0.0001, + "loss": 56.3668, + "step": 10810 + }, + { + "epoch": 174.52032520325204, + "grad_norm": 0.8204325437545776, + "learning_rate": 0.0001, + "loss": 57.5088, + "step": 10820 + }, + { + "epoch": 174.6829268292683, + "grad_norm": 0.912240743637085, + "learning_rate": 0.0001, + "loss": 54.392, + "step": 10830 + }, + { + "epoch": 174.84552845528455, + "grad_norm": 1.0001140832901, + "learning_rate": 0.0001, + "loss": 56.2384, + "step": 10840 + }, + { + "epoch": 175.0, + "grad_norm": 1.1657922267913818, + "learning_rate": 0.0001, + "loss": 53.9591, + "step": 10850 + }, + { + "epoch": 175.16260162601625, + "grad_norm": 0.9120311141014099, + "learning_rate": 0.0001, + "loss": 56.7532, + "step": 10860 + }, + { + "epoch": 175.3252032520325, + "grad_norm": 0.821774423122406, + "learning_rate": 0.0001, + "loss": 56.1124, + "step": 10870 + }, + { + "epoch": 175.4878048780488, + "grad_norm": 0.8408815264701843, + "learning_rate": 0.0001, + "loss": 56.8947, + "step": 10880 + }, + { + "epoch": 175.65040650406505, + "grad_norm": 1.2977283000946045, + "learning_rate": 0.0001, + "loss": 55.476, + "step": 10890 + }, + { + "epoch": 175.8130081300813, + "grad_norm": 0.8232496380805969, + "learning_rate": 0.0001, + "loss": 56.0607, + "step": 10900 + }, + { + "epoch": 175.97560975609755, + "grad_norm": 0.8235589265823364, + "learning_rate": 0.0001, + "loss": 56.3365, + "step": 10910 + }, + { + "epoch": 176.130081300813, + "grad_norm": 1.0942591428756714, + "learning_rate": 0.0001, + "loss": 53.065, + "step": 10920 + }, + { + "epoch": 176.29268292682926, + "grad_norm": 0.9026378989219666, + "learning_rate": 0.0001, + "loss": 55.6453, + "step": 10930 + }, + { + "epoch": 176.45528455284554, + "grad_norm": 0.9491869211196899, + "learning_rate": 0.0001, + "loss": 56.4853, + "step": 10940 + }, + { + "epoch": 176.6178861788618, + "grad_norm": 1.072492241859436, + "learning_rate": 0.0001, + "loss": 55.7488, + "step": 10950 + }, + { + "epoch": 176.78048780487805, + "grad_norm": 0.9384275674819946, + "learning_rate": 0.0001, + "loss": 56.8056, + "step": 10960 + }, + { + "epoch": 176.9430894308943, + "grad_norm": 0.9458582997322083, + "learning_rate": 0.0001, + "loss": 57.187, + "step": 10970 + }, + { + "epoch": 177.09756097560975, + "grad_norm": 0.9112198352813721, + "learning_rate": 0.0001, + "loss": 52.9576, + "step": 10980 + }, + { + "epoch": 177.260162601626, + "grad_norm": 1.0807278156280518, + "learning_rate": 0.0001, + "loss": 56.0773, + "step": 10990 + }, + { + "epoch": 177.4227642276423, + "grad_norm": 0.8122154474258423, + "learning_rate": 0.0001, + "loss": 56.0297, + "step": 11000 + }, + { + "epoch": 177.58536585365854, + "grad_norm": 0.7836926579475403, + "learning_rate": 0.0001, + "loss": 56.74, + "step": 11010 + }, + { + "epoch": 177.7479674796748, + "grad_norm": 1.0483779907226562, + "learning_rate": 0.0001, + "loss": 55.1174, + "step": 11020 + }, + { + "epoch": 177.91056910569105, + "grad_norm": 0.8454721570014954, + "learning_rate": 0.0001, + "loss": 55.3519, + "step": 11030 + }, + { + "epoch": 178.0650406504065, + "grad_norm": 1.2100988626480103, + "learning_rate": 0.0001, + "loss": 54.5955, + "step": 11040 + }, + { + "epoch": 178.22764227642276, + "grad_norm": 1.0322265625, + "learning_rate": 0.0001, + "loss": 57.2623, + "step": 11050 + }, + { + "epoch": 178.390243902439, + "grad_norm": 0.8880140781402588, + "learning_rate": 0.0001, + "loss": 56.5935, + "step": 11060 + }, + { + "epoch": 178.5528455284553, + "grad_norm": 0.8864359259605408, + "learning_rate": 0.0001, + "loss": 56.802, + "step": 11070 + }, + { + "epoch": 178.71544715447155, + "grad_norm": 1.1736643314361572, + "learning_rate": 0.0001, + "loss": 56.8208, + "step": 11080 + }, + { + "epoch": 178.8780487804878, + "grad_norm": 0.975029706954956, + "learning_rate": 0.0001, + "loss": 57.5741, + "step": 11090 + }, + { + "epoch": 179.03252032520325, + "grad_norm": 0.852942168712616, + "learning_rate": 0.0001, + "loss": 54.327, + "step": 11100 + }, + { + "epoch": 179.1951219512195, + "grad_norm": 1.1381421089172363, + "learning_rate": 0.0001, + "loss": 56.0875, + "step": 11110 + }, + { + "epoch": 179.35772357723576, + "grad_norm": 1.0923271179199219, + "learning_rate": 0.0001, + "loss": 58.1302, + "step": 11120 + }, + { + "epoch": 179.52032520325204, + "grad_norm": 0.9422796964645386, + "learning_rate": 0.0001, + "loss": 56.2919, + "step": 11130 + }, + { + "epoch": 179.6829268292683, + "grad_norm": 0.9511571526527405, + "learning_rate": 0.0001, + "loss": 55.1329, + "step": 11140 + }, + { + "epoch": 179.84552845528455, + "grad_norm": 0.9221625328063965, + "learning_rate": 0.0001, + "loss": 56.6727, + "step": 11150 + }, + { + "epoch": 180.0, + "grad_norm": 0.8903642296791077, + "learning_rate": 0.0001, + "loss": 53.7472, + "step": 11160 + }, + { + "epoch": 180.16260162601625, + "grad_norm": 0.9752726554870605, + "learning_rate": 0.0001, + "loss": 56.2322, + "step": 11170 + }, + { + "epoch": 180.3252032520325, + "grad_norm": 0.8058391809463501, + "learning_rate": 0.0001, + "loss": 57.7904, + "step": 11180 + }, + { + "epoch": 180.4878048780488, + "grad_norm": 0.9203750491142273, + "learning_rate": 0.0001, + "loss": 55.3608, + "step": 11190 + }, + { + "epoch": 180.65040650406505, + "grad_norm": 1.4366613626480103, + "learning_rate": 0.0001, + "loss": 54.5204, + "step": 11200 + }, + { + "epoch": 180.8130081300813, + "grad_norm": 1.040142297744751, + "learning_rate": 0.0001, + "loss": 56.3904, + "step": 11210 + }, + { + "epoch": 180.97560975609755, + "grad_norm": 1.1629812717437744, + "learning_rate": 0.0001, + "loss": 55.4976, + "step": 11220 + }, + { + "epoch": 181.130081300813, + "grad_norm": 0.847724437713623, + "learning_rate": 0.0001, + "loss": 54.4277, + "step": 11230 + }, + { + "epoch": 181.29268292682926, + "grad_norm": 1.0196763277053833, + "learning_rate": 0.0001, + "loss": 58.1868, + "step": 11240 + }, + { + "epoch": 181.45528455284554, + "grad_norm": 1.2344937324523926, + "learning_rate": 0.0001, + "loss": 56.1104, + "step": 11250 + }, + { + "epoch": 181.6178861788618, + "grad_norm": 0.8804627060890198, + "learning_rate": 0.0001, + "loss": 55.728, + "step": 11260 + }, + { + "epoch": 181.78048780487805, + "grad_norm": 0.8948118686676025, + "learning_rate": 0.0001, + "loss": 56.9878, + "step": 11270 + }, + { + "epoch": 181.9430894308943, + "grad_norm": 0.6648201942443848, + "learning_rate": 0.0001, + "loss": 56.5425, + "step": 11280 + }, + { + "epoch": 182.09756097560975, + "grad_norm": 0.8076626658439636, + "learning_rate": 0.0001, + "loss": 53.4703, + "step": 11290 + }, + { + "epoch": 182.260162601626, + "grad_norm": 0.9063478708267212, + "learning_rate": 0.0001, + "loss": 57.6056, + "step": 11300 + }, + { + "epoch": 182.4227642276423, + "grad_norm": 0.8387530446052551, + "learning_rate": 0.0001, + "loss": 56.0165, + "step": 11310 + }, + { + "epoch": 182.58536585365854, + "grad_norm": 0.8866046667098999, + "learning_rate": 0.0001, + "loss": 57.774, + "step": 11320 + }, + { + "epoch": 182.7479674796748, + "grad_norm": 0.9022682905197144, + "learning_rate": 0.0001, + "loss": 57.5811, + "step": 11330 + }, + { + "epoch": 182.91056910569105, + "grad_norm": 0.8135976195335388, + "learning_rate": 0.0001, + "loss": 56.048, + "step": 11340 + }, + { + "epoch": 183.0650406504065, + "grad_norm": 1.0014373064041138, + "learning_rate": 0.0001, + "loss": 52.5195, + "step": 11350 + }, + { + "epoch": 183.22764227642276, + "grad_norm": 1.0275895595550537, + "learning_rate": 0.0001, + "loss": 57.1421, + "step": 11360 + }, + { + "epoch": 183.390243902439, + "grad_norm": 0.8826107382774353, + "learning_rate": 0.0001, + "loss": 57.5582, + "step": 11370 + }, + { + "epoch": 183.5528455284553, + "grad_norm": 0.9023060202598572, + "learning_rate": 0.0001, + "loss": 56.6695, + "step": 11380 + }, + { + "epoch": 183.71544715447155, + "grad_norm": 0.7782049179077148, + "learning_rate": 0.0001, + "loss": 55.4362, + "step": 11390 + }, + { + "epoch": 183.8780487804878, + "grad_norm": 0.9904818534851074, + "learning_rate": 0.0001, + "loss": 56.7164, + "step": 11400 + }, + { + "epoch": 184.03252032520325, + "grad_norm": 0.9218631982803345, + "learning_rate": 0.0001, + "loss": 53.8536, + "step": 11410 + }, + { + "epoch": 184.1951219512195, + "grad_norm": 0.7807294130325317, + "learning_rate": 0.0001, + "loss": 57.1502, + "step": 11420 + }, + { + "epoch": 184.35772357723576, + "grad_norm": 0.8353742957115173, + "learning_rate": 0.0001, + "loss": 56.8276, + "step": 11430 + }, + { + "epoch": 184.52032520325204, + "grad_norm": 0.7909665107727051, + "learning_rate": 0.0001, + "loss": 56.4164, + "step": 11440 + }, + { + "epoch": 184.6829268292683, + "grad_norm": 1.0313756465911865, + "learning_rate": 0.0001, + "loss": 56.1275, + "step": 11450 + }, + { + "epoch": 184.84552845528455, + "grad_norm": 0.9377022385597229, + "learning_rate": 0.0001, + "loss": 55.9829, + "step": 11460 + }, + { + "epoch": 185.0, + "grad_norm": 1.2855385541915894, + "learning_rate": 0.0001, + "loss": 53.6264, + "step": 11470 + }, + { + "epoch": 185.16260162601625, + "grad_norm": 1.7889217138290405, + "learning_rate": 0.0001, + "loss": 55.1279, + "step": 11480 + }, + { + "epoch": 185.3252032520325, + "grad_norm": 0.8415203094482422, + "learning_rate": 0.0001, + "loss": 57.1529, + "step": 11490 + }, + { + "epoch": 185.4878048780488, + "grad_norm": 1.1245211362838745, + "learning_rate": 0.0001, + "loss": 56.2543, + "step": 11500 + }, + { + "epoch": 185.65040650406505, + "grad_norm": 1.607924222946167, + "learning_rate": 0.0001, + "loss": 55.9007, + "step": 11510 + }, + { + "epoch": 185.8130081300813, + "grad_norm": 0.8068832755088806, + "learning_rate": 0.0001, + "loss": 56.5567, + "step": 11520 + }, + { + "epoch": 185.97560975609755, + "grad_norm": 1.109636902809143, + "learning_rate": 0.0001, + "loss": 55.8676, + "step": 11530 + }, + { + "epoch": 186.130081300813, + "grad_norm": 0.9668437242507935, + "learning_rate": 0.0001, + "loss": 52.9045, + "step": 11540 + }, + { + "epoch": 186.29268292682926, + "grad_norm": 1.1277968883514404, + "learning_rate": 0.0001, + "loss": 55.0794, + "step": 11550 + }, + { + "epoch": 186.45528455284554, + "grad_norm": 0.6413527727127075, + "learning_rate": 0.0001, + "loss": 55.4634, + "step": 11560 + }, + { + "epoch": 186.6178861788618, + "grad_norm": 1.039292812347412, + "learning_rate": 0.0001, + "loss": 56.2108, + "step": 11570 + }, + { + "epoch": 186.78048780487805, + "grad_norm": 0.9112428426742554, + "learning_rate": 0.0001, + "loss": 56.4036, + "step": 11580 + }, + { + "epoch": 186.9430894308943, + "grad_norm": 0.8918914794921875, + "learning_rate": 0.0001, + "loss": 54.9626, + "step": 11590 + }, + { + "epoch": 187.09756097560975, + "grad_norm": 0.9715940952301025, + "learning_rate": 0.0001, + "loss": 54.0208, + "step": 11600 + }, + { + "epoch": 187.260162601626, + "grad_norm": 0.8578653931617737, + "learning_rate": 0.0001, + "loss": 56.4479, + "step": 11610 + }, + { + "epoch": 187.4227642276423, + "grad_norm": 1.7716244459152222, + "learning_rate": 0.0001, + "loss": 55.3543, + "step": 11620 + }, + { + "epoch": 187.58536585365854, + "grad_norm": 0.9152880311012268, + "learning_rate": 0.0001, + "loss": 56.8464, + "step": 11630 + }, + { + "epoch": 187.7479674796748, + "grad_norm": 0.8824559450149536, + "learning_rate": 0.0001, + "loss": 55.9501, + "step": 11640 + }, + { + "epoch": 187.91056910569105, + "grad_norm": 0.780864953994751, + "learning_rate": 0.0001, + "loss": 57.3594, + "step": 11650 + }, + { + "epoch": 188.0650406504065, + "grad_norm": 0.7242524027824402, + "learning_rate": 0.0001, + "loss": 53.9958, + "step": 11660 + }, + { + "epoch": 188.22764227642276, + "grad_norm": 0.9025135636329651, + "learning_rate": 0.0001, + "loss": 54.6309, + "step": 11670 + }, + { + "epoch": 188.390243902439, + "grad_norm": 0.7995520830154419, + "learning_rate": 0.0001, + "loss": 56.7882, + "step": 11680 + }, + { + "epoch": 188.5528455284553, + "grad_norm": 0.8525217175483704, + "learning_rate": 0.0001, + "loss": 54.7559, + "step": 11690 + }, + { + "epoch": 188.71544715447155, + "grad_norm": 0.9954654574394226, + "learning_rate": 0.0001, + "loss": 56.4574, + "step": 11700 + }, + { + "epoch": 188.8780487804878, + "grad_norm": 0.851827085018158, + "learning_rate": 0.0001, + "loss": 55.996, + "step": 11710 + }, + { + "epoch": 189.03252032520325, + "grad_norm": 0.9404662847518921, + "learning_rate": 0.0001, + "loss": 53.8035, + "step": 11720 + }, + { + "epoch": 189.1951219512195, + "grad_norm": 0.8355186581611633, + "learning_rate": 0.0001, + "loss": 54.9002, + "step": 11730 + }, + { + "epoch": 189.35772357723576, + "grad_norm": 1.0156077146530151, + "learning_rate": 0.0001, + "loss": 55.8483, + "step": 11740 + }, + { + "epoch": 189.52032520325204, + "grad_norm": 0.7925308346748352, + "learning_rate": 0.0001, + "loss": 54.1772, + "step": 11750 + }, + { + "epoch": 189.6829268292683, + "grad_norm": 0.8137376308441162, + "learning_rate": 0.0001, + "loss": 53.6278, + "step": 11760 + }, + { + "epoch": 189.84552845528455, + "grad_norm": 0.7871745824813843, + "learning_rate": 0.0001, + "loss": 57.7457, + "step": 11770 + }, + { + "epoch": 190.0, + "grad_norm": 0.7344806790351868, + "learning_rate": 0.0001, + "loss": 55.6082, + "step": 11780 + }, + { + "epoch": 190.16260162601625, + "grad_norm": 0.9094588756561279, + "learning_rate": 0.0001, + "loss": 57.2329, + "step": 11790 + }, + { + "epoch": 190.3252032520325, + "grad_norm": 1.2261676788330078, + "learning_rate": 0.0001, + "loss": 56.2071, + "step": 11800 + }, + { + "epoch": 190.4878048780488, + "grad_norm": 0.9519370794296265, + "learning_rate": 0.0001, + "loss": 56.2488, + "step": 11810 + }, + { + "epoch": 190.65040650406505, + "grad_norm": 0.8949872851371765, + "learning_rate": 0.0001, + "loss": 57.9283, + "step": 11820 + }, + { + "epoch": 190.8130081300813, + "grad_norm": 0.8764474391937256, + "learning_rate": 0.0001, + "loss": 55.2962, + "step": 11830 + }, + { + "epoch": 190.97560975609755, + "grad_norm": 0.9472342729568481, + "learning_rate": 0.0001, + "loss": 56.2262, + "step": 11840 + }, + { + "epoch": 191.130081300813, + "grad_norm": 0.7477512359619141, + "learning_rate": 0.0001, + "loss": 51.2613, + "step": 11850 + }, + { + "epoch": 191.29268292682926, + "grad_norm": 1.0346648693084717, + "learning_rate": 0.0001, + "loss": 56.0406, + "step": 11860 + }, + { + "epoch": 191.45528455284554, + "grad_norm": 1.0473452806472778, + "learning_rate": 0.0001, + "loss": 56.3583, + "step": 11870 + }, + { + "epoch": 191.6178861788618, + "grad_norm": 1.0167906284332275, + "learning_rate": 0.0001, + "loss": 56.3803, + "step": 11880 + }, + { + "epoch": 191.78048780487805, + "grad_norm": 0.8768972754478455, + "learning_rate": 0.0001, + "loss": 57.2783, + "step": 11890 + }, + { + "epoch": 191.9430894308943, + "grad_norm": 0.9370661973953247, + "learning_rate": 0.0001, + "loss": 55.9976, + "step": 11900 + }, + { + "epoch": 192.09756097560975, + "grad_norm": 0.7853850722312927, + "learning_rate": 0.0001, + "loss": 51.9341, + "step": 11910 + }, + { + "epoch": 192.260162601626, + "grad_norm": 1.122916579246521, + "learning_rate": 0.0001, + "loss": 55.832, + "step": 11920 + }, + { + "epoch": 192.4227642276423, + "grad_norm": 0.982596755027771, + "learning_rate": 0.0001, + "loss": 54.2484, + "step": 11930 + }, + { + "epoch": 192.58536585365854, + "grad_norm": 1.0893633365631104, + "learning_rate": 0.0001, + "loss": 54.4806, + "step": 11940 + }, + { + "epoch": 192.7479674796748, + "grad_norm": 0.8936525583267212, + "learning_rate": 0.0001, + "loss": 55.1085, + "step": 11950 + }, + { + "epoch": 192.91056910569105, + "grad_norm": 0.9169464111328125, + "learning_rate": 0.0001, + "loss": 56.8134, + "step": 11960 + }, + { + "epoch": 193.0650406504065, + "grad_norm": 1.2202202081680298, + "learning_rate": 0.0001, + "loss": 53.3595, + "step": 11970 + }, + { + "epoch": 193.22764227642276, + "grad_norm": 0.7786313891410828, + "learning_rate": 0.0001, + "loss": 57.6186, + "step": 11980 + }, + { + "epoch": 193.390243902439, + "grad_norm": 0.9757220149040222, + "learning_rate": 0.0001, + "loss": 54.5559, + "step": 11990 + }, + { + "epoch": 193.5528455284553, + "grad_norm": 0.8599177002906799, + "learning_rate": 0.0001, + "loss": 54.9894, + "step": 12000 + }, + { + "epoch": 193.71544715447155, + "grad_norm": 1.614792823791504, + "learning_rate": 0.0001, + "loss": 56.6223, + "step": 12010 + }, + { + "epoch": 193.8780487804878, + "grad_norm": 0.9011794924736023, + "learning_rate": 0.0001, + "loss": 54.9678, + "step": 12020 + }, + { + "epoch": 194.03252032520325, + "grad_norm": 1.1813637018203735, + "learning_rate": 0.0001, + "loss": 51.7197, + "step": 12030 + }, + { + "epoch": 194.1951219512195, + "grad_norm": 1.420372724533081, + "learning_rate": 0.0001, + "loss": 57.0948, + "step": 12040 + }, + { + "epoch": 194.35772357723576, + "grad_norm": 0.8253830671310425, + "learning_rate": 0.0001, + "loss": 54.8281, + "step": 12050 + }, + { + "epoch": 194.52032520325204, + "grad_norm": 1.055162787437439, + "learning_rate": 0.0001, + "loss": 56.3061, + "step": 12060 + }, + { + "epoch": 194.6829268292683, + "grad_norm": 1.0420011281967163, + "learning_rate": 0.0001, + "loss": 54.7829, + "step": 12070 + }, + { + "epoch": 194.84552845528455, + "grad_norm": 0.7156410813331604, + "learning_rate": 0.0001, + "loss": 55.1366, + "step": 12080 + }, + { + "epoch": 195.0, + "grad_norm": 1.2018579244613647, + "learning_rate": 0.0001, + "loss": 55.2568, + "step": 12090 + }, + { + "epoch": 195.16260162601625, + "grad_norm": 0.9234516620635986, + "learning_rate": 0.0001, + "loss": 55.6524, + "step": 12100 + }, + { + "epoch": 195.3252032520325, + "grad_norm": 0.7889829874038696, + "learning_rate": 0.0001, + "loss": 54.6211, + "step": 12110 + }, + { + "epoch": 195.4878048780488, + "grad_norm": 0.6909085512161255, + "learning_rate": 0.0001, + "loss": 55.161, + "step": 12120 + }, + { + "epoch": 195.65040650406505, + "grad_norm": 1.0910391807556152, + "learning_rate": 0.0001, + "loss": 54.289, + "step": 12130 + }, + { + "epoch": 195.8130081300813, + "grad_norm": 0.8114144206047058, + "learning_rate": 0.0001, + "loss": 57.1045, + "step": 12140 + }, + { + "epoch": 195.97560975609755, + "grad_norm": 0.8750683069229126, + "learning_rate": 0.0001, + "loss": 56.6573, + "step": 12150 + }, + { + "epoch": 196.130081300813, + "grad_norm": 0.9273776412010193, + "learning_rate": 0.0001, + "loss": 52.4588, + "step": 12160 + }, + { + "epoch": 196.29268292682926, + "grad_norm": 0.8890953063964844, + "learning_rate": 0.0001, + "loss": 56.2728, + "step": 12170 + }, + { + "epoch": 196.45528455284554, + "grad_norm": 0.7885172367095947, + "learning_rate": 0.0001, + "loss": 56.9814, + "step": 12180 + }, + { + "epoch": 196.6178861788618, + "grad_norm": 1.0934451818466187, + "learning_rate": 0.0001, + "loss": 54.8799, + "step": 12190 + }, + { + "epoch": 196.78048780487805, + "grad_norm": 0.7452919483184814, + "learning_rate": 0.0001, + "loss": 54.8566, + "step": 12200 + }, + { + "epoch": 196.9430894308943, + "grad_norm": 1.2472970485687256, + "learning_rate": 0.0001, + "loss": 56.9789, + "step": 12210 + }, + { + "epoch": 197.09756097560975, + "grad_norm": 1.0619820356369019, + "learning_rate": 0.0001, + "loss": 52.0264, + "step": 12220 + }, + { + "epoch": 197.260162601626, + "grad_norm": 0.9423636198043823, + "learning_rate": 0.0001, + "loss": 54.6033, + "step": 12230 + }, + { + "epoch": 197.4227642276423, + "grad_norm": 0.6707811951637268, + "learning_rate": 0.0001, + "loss": 54.5683, + "step": 12240 + }, + { + "epoch": 197.58536585365854, + "grad_norm": 1.0682896375656128, + "learning_rate": 0.0001, + "loss": 53.6539, + "step": 12250 + }, + { + "epoch": 197.7479674796748, + "grad_norm": 0.9057245254516602, + "learning_rate": 0.0001, + "loss": 57.1636, + "step": 12260 + }, + { + "epoch": 197.91056910569105, + "grad_norm": 1.0249336957931519, + "learning_rate": 0.0001, + "loss": 54.9638, + "step": 12270 + }, + { + "epoch": 198.0650406504065, + "grad_norm": 0.8086504936218262, + "learning_rate": 0.0001, + "loss": 53.5776, + "step": 12280 + }, + { + "epoch": 198.22764227642276, + "grad_norm": 0.8970116376876831, + "learning_rate": 0.0001, + "loss": 55.9899, + "step": 12290 + }, + { + "epoch": 198.390243902439, + "grad_norm": 0.9402612447738647, + "learning_rate": 0.0001, + "loss": 56.7494, + "step": 12300 + }, + { + "epoch": 198.5528455284553, + "grad_norm": 0.865734338760376, + "learning_rate": 0.0001, + "loss": 56.3205, + "step": 12310 + }, + { + "epoch": 198.71544715447155, + "grad_norm": 0.9791702628135681, + "learning_rate": 0.0001, + "loss": 55.1574, + "step": 12320 + }, + { + "epoch": 198.8780487804878, + "grad_norm": 1.0380010604858398, + "learning_rate": 0.0001, + "loss": 56.5315, + "step": 12330 + }, + { + "epoch": 199.03252032520325, + "grad_norm": 1.14964759349823, + "learning_rate": 0.0001, + "loss": 52.6714, + "step": 12340 + }, + { + "epoch": 199.1951219512195, + "grad_norm": 0.9895046949386597, + "learning_rate": 0.0001, + "loss": 55.0279, + "step": 12350 + }, + { + "epoch": 199.35772357723576, + "grad_norm": 0.8557648062705994, + "learning_rate": 0.0001, + "loss": 54.9455, + "step": 12360 + }, + { + "epoch": 199.52032520325204, + "grad_norm": 0.9442970752716064, + "learning_rate": 0.0001, + "loss": 56.2179, + "step": 12370 + }, + { + "epoch": 199.6829268292683, + "grad_norm": 1.2164298295974731, + "learning_rate": 0.0001, + "loss": 54.7891, + "step": 12380 + }, + { + "epoch": 199.84552845528455, + "grad_norm": 1.1152207851409912, + "learning_rate": 0.0001, + "loss": 53.7524, + "step": 12390 + }, + { + "epoch": 200.0, + "grad_norm": 0.9624357223510742, + "learning_rate": 0.0001, + "loss": 52.4309, + "step": 12400 + }, + { + "epoch": 200.16260162601625, + "grad_norm": 1.1027910709381104, + "learning_rate": 0.0001, + "loss": 55.8449, + "step": 12410 + }, + { + "epoch": 200.3252032520325, + "grad_norm": 0.9644309878349304, + "learning_rate": 0.0001, + "loss": 54.6539, + "step": 12420 + }, + { + "epoch": 200.4878048780488, + "grad_norm": 0.8299253582954407, + "learning_rate": 0.0001, + "loss": 53.8338, + "step": 12430 + }, + { + "epoch": 200.65040650406505, + "grad_norm": 1.0003494024276733, + "learning_rate": 0.0001, + "loss": 56.5585, + "step": 12440 + }, + { + "epoch": 200.8130081300813, + "grad_norm": 0.9802242517471313, + "learning_rate": 0.0001, + "loss": 56.4943, + "step": 12450 + }, + { + "epoch": 200.97560975609755, + "grad_norm": 0.8739794492721558, + "learning_rate": 0.0001, + "loss": 55.1041, + "step": 12460 + }, + { + "epoch": 201.130081300813, + "grad_norm": 1.8339481353759766, + "learning_rate": 0.0001, + "loss": 52.89, + "step": 12470 + }, + { + "epoch": 201.29268292682926, + "grad_norm": 1.3706036806106567, + "learning_rate": 0.0001, + "loss": 55.5897, + "step": 12480 + }, + { + "epoch": 201.45528455284554, + "grad_norm": 1.2508513927459717, + "learning_rate": 0.0001, + "loss": 56.3578, + "step": 12490 + }, + { + "epoch": 201.6178861788618, + "grad_norm": 0.7036287188529968, + "learning_rate": 0.0001, + "loss": 54.2583, + "step": 12500 + }, + { + "epoch": 201.78048780487805, + "grad_norm": 0.9160637259483337, + "learning_rate": 0.0001, + "loss": 56.3668, + "step": 12510 + }, + { + "epoch": 201.9430894308943, + "grad_norm": 0.8451681137084961, + "learning_rate": 0.0001, + "loss": 55.0297, + "step": 12520 + }, + { + "epoch": 202.09756097560975, + "grad_norm": 1.103285789489746, + "learning_rate": 0.0001, + "loss": 52.6796, + "step": 12530 + }, + { + "epoch": 202.260162601626, + "grad_norm": 0.9927880167961121, + "learning_rate": 0.0001, + "loss": 55.9175, + "step": 12540 + }, + { + "epoch": 202.4227642276423, + "grad_norm": 0.6602478623390198, + "learning_rate": 0.0001, + "loss": 56.0773, + "step": 12550 + }, + { + "epoch": 202.58536585365854, + "grad_norm": 0.8949015140533447, + "learning_rate": 0.0001, + "loss": 55.9995, + "step": 12560 + }, + { + "epoch": 202.7479674796748, + "grad_norm": 0.7111349701881409, + "learning_rate": 0.0001, + "loss": 55.4216, + "step": 12570 + }, + { + "epoch": 202.91056910569105, + "grad_norm": 0.8465375900268555, + "learning_rate": 0.0001, + "loss": 57.2378, + "step": 12580 + }, + { + "epoch": 203.0650406504065, + "grad_norm": 0.7122918367385864, + "learning_rate": 0.0001, + "loss": 54.5047, + "step": 12590 + }, + { + "epoch": 203.22764227642276, + "grad_norm": 0.9927072525024414, + "learning_rate": 0.0001, + "loss": 53.9752, + "step": 12600 + }, + { + "epoch": 203.390243902439, + "grad_norm": 0.7084205746650696, + "learning_rate": 0.0001, + "loss": 53.7208, + "step": 12610 + }, + { + "epoch": 203.5528455284553, + "grad_norm": 0.7796747088432312, + "learning_rate": 0.0001, + "loss": 54.5118, + "step": 12620 + }, + { + "epoch": 203.71544715447155, + "grad_norm": 0.7925731539726257, + "learning_rate": 0.0001, + "loss": 55.8361, + "step": 12630 + }, + { + "epoch": 203.8780487804878, + "grad_norm": 0.8884345889091492, + "learning_rate": 0.0001, + "loss": 52.6655, + "step": 12640 + }, + { + "epoch": 204.03252032520325, + "grad_norm": 0.7525721192359924, + "learning_rate": 0.0001, + "loss": 52.0389, + "step": 12650 + }, + { + "epoch": 204.1951219512195, + "grad_norm": 1.071750521659851, + "learning_rate": 0.0001, + "loss": 55.6427, + "step": 12660 + }, + { + "epoch": 204.35772357723576, + "grad_norm": 0.7978437542915344, + "learning_rate": 0.0001, + "loss": 55.1445, + "step": 12670 + }, + { + "epoch": 204.52032520325204, + "grad_norm": 0.6743281483650208, + "learning_rate": 0.0001, + "loss": 56.5749, + "step": 12680 + }, + { + "epoch": 204.6829268292683, + "grad_norm": 0.9658812284469604, + "learning_rate": 0.0001, + "loss": 55.2514, + "step": 12690 + }, + { + "epoch": 204.84552845528455, + "grad_norm": 0.8930355310440063, + "learning_rate": 0.0001, + "loss": 56.5771, + "step": 12700 + }, + { + "epoch": 205.0, + "grad_norm": 2.1135165691375732, + "learning_rate": 0.0001, + "loss": 53.9625, + "step": 12710 + }, + { + "epoch": 205.16260162601625, + "grad_norm": 1.0068354606628418, + "learning_rate": 0.0001, + "loss": 55.2095, + "step": 12720 + }, + { + "epoch": 205.3252032520325, + "grad_norm": 0.8000341653823853, + "learning_rate": 0.0001, + "loss": 56.155, + "step": 12730 + }, + { + "epoch": 205.4878048780488, + "grad_norm": 1.201411485671997, + "learning_rate": 0.0001, + "loss": 56.8318, + "step": 12740 + }, + { + "epoch": 205.65040650406505, + "grad_norm": 1.005187749862671, + "learning_rate": 0.0001, + "loss": 55.8241, + "step": 12750 + }, + { + "epoch": 205.8130081300813, + "grad_norm": 1.0376118421554565, + "learning_rate": 0.0001, + "loss": 55.3489, + "step": 12760 + }, + { + "epoch": 205.97560975609755, + "grad_norm": 0.8955312967300415, + "learning_rate": 0.0001, + "loss": 55.583, + "step": 12770 + }, + { + "epoch": 206.130081300813, + "grad_norm": 0.9010198712348938, + "learning_rate": 0.0001, + "loss": 52.4093, + "step": 12780 + }, + { + "epoch": 206.29268292682926, + "grad_norm": 0.758160412311554, + "learning_rate": 0.0001, + "loss": 56.1256, + "step": 12790 + }, + { + "epoch": 206.45528455284554, + "grad_norm": 0.7211036086082458, + "learning_rate": 0.0001, + "loss": 55.0504, + "step": 12800 + }, + { + "epoch": 206.6178861788618, + "grad_norm": 1.065086841583252, + "learning_rate": 0.0001, + "loss": 55.7009, + "step": 12810 + }, + { + "epoch": 206.78048780487805, + "grad_norm": 0.6722949743270874, + "learning_rate": 0.0001, + "loss": 54.0928, + "step": 12820 + }, + { + "epoch": 206.9430894308943, + "grad_norm": 0.9325671792030334, + "learning_rate": 0.0001, + "loss": 55.6211, + "step": 12830 + }, + { + "epoch": 207.09756097560975, + "grad_norm": 0.9299114942550659, + "learning_rate": 0.0001, + "loss": 51.8944, + "step": 12840 + }, + { + "epoch": 207.260162601626, + "grad_norm": 0.8544942140579224, + "learning_rate": 0.0001, + "loss": 55.1983, + "step": 12850 + }, + { + "epoch": 207.4227642276423, + "grad_norm": 1.0352020263671875, + "learning_rate": 0.0001, + "loss": 53.4288, + "step": 12860 + }, + { + "epoch": 207.58536585365854, + "grad_norm": 1.1637558937072754, + "learning_rate": 0.0001, + "loss": 54.5078, + "step": 12870 + }, + { + "epoch": 207.7479674796748, + "grad_norm": 1.0634640455245972, + "learning_rate": 0.0001, + "loss": 53.8032, + "step": 12880 + }, + { + "epoch": 207.91056910569105, + "grad_norm": 1.0478582382202148, + "learning_rate": 0.0001, + "loss": 54.774, + "step": 12890 + }, + { + "epoch": 208.0650406504065, + "grad_norm": 0.772788405418396, + "learning_rate": 0.0001, + "loss": 52.4807, + "step": 12900 + }, + { + "epoch": 208.22764227642276, + "grad_norm": 1.0519691705703735, + "learning_rate": 0.0001, + "loss": 54.8709, + "step": 12910 + }, + { + "epoch": 208.390243902439, + "grad_norm": 0.9291022419929504, + "learning_rate": 0.0001, + "loss": 55.8279, + "step": 12920 + }, + { + "epoch": 208.5528455284553, + "grad_norm": 0.9870265126228333, + "learning_rate": 0.0001, + "loss": 56.3803, + "step": 12930 + }, + { + "epoch": 208.71544715447155, + "grad_norm": 0.787817656993866, + "learning_rate": 0.0001, + "loss": 55.1475, + "step": 12940 + }, + { + "epoch": 208.8780487804878, + "grad_norm": 1.1020783185958862, + "learning_rate": 0.0001, + "loss": 56.1078, + "step": 12950 + }, + { + "epoch": 209.03252032520325, + "grad_norm": 0.8097794055938721, + "learning_rate": 0.0001, + "loss": 53.1355, + "step": 12960 + }, + { + "epoch": 209.1951219512195, + "grad_norm": 1.1928120851516724, + "learning_rate": 0.0001, + "loss": 53.1578, + "step": 12970 + }, + { + "epoch": 209.35772357723576, + "grad_norm": 1.131969928741455, + "learning_rate": 0.0001, + "loss": 56.2361, + "step": 12980 + }, + { + "epoch": 209.52032520325204, + "grad_norm": 0.7544846534729004, + "learning_rate": 0.0001, + "loss": 55.9766, + "step": 12990 + }, + { + "epoch": 209.6829268292683, + "grad_norm": 1.2758643627166748, + "learning_rate": 0.0001, + "loss": 53.5174, + "step": 13000 + }, + { + "epoch": 209.84552845528455, + "grad_norm": 0.8507160544395447, + "learning_rate": 0.0001, + "loss": 53.9673, + "step": 13010 + }, + { + "epoch": 210.0, + "grad_norm": 1.2480005025863647, + "learning_rate": 0.0001, + "loss": 51.9134, + "step": 13020 + }, + { + "epoch": 210.16260162601625, + "grad_norm": 1.0838913917541504, + "learning_rate": 0.0001, + "loss": 52.4845, + "step": 13030 + }, + { + "epoch": 210.3252032520325, + "grad_norm": 0.8026198744773865, + "learning_rate": 0.0001, + "loss": 54.2063, + "step": 13040 + }, + { + "epoch": 210.4878048780488, + "grad_norm": 0.7196829319000244, + "learning_rate": 0.0001, + "loss": 54.91, + "step": 13050 + }, + { + "epoch": 210.65040650406505, + "grad_norm": 0.8283985257148743, + "learning_rate": 0.0001, + "loss": 56.3195, + "step": 13060 + }, + { + "epoch": 210.8130081300813, + "grad_norm": 0.9254607558250427, + "learning_rate": 0.0001, + "loss": 55.7841, + "step": 13070 + }, + { + "epoch": 210.97560975609755, + "grad_norm": 1.0719525814056396, + "learning_rate": 0.0001, + "loss": 55.4097, + "step": 13080 + }, + { + "epoch": 211.130081300813, + "grad_norm": 0.7990654110908508, + "learning_rate": 0.0001, + "loss": 50.4808, + "step": 13090 + }, + { + "epoch": 211.29268292682926, + "grad_norm": 0.7976981401443481, + "learning_rate": 0.0001, + "loss": 53.8439, + "step": 13100 + }, + { + "epoch": 211.45528455284554, + "grad_norm": 0.8668096661567688, + "learning_rate": 0.0001, + "loss": 55.9312, + "step": 13110 + }, + { + "epoch": 211.6178861788618, + "grad_norm": 0.7176859974861145, + "learning_rate": 0.0001, + "loss": 53.5898, + "step": 13120 + }, + { + "epoch": 211.78048780487805, + "grad_norm": 0.6893277168273926, + "learning_rate": 0.0001, + "loss": 54.099, + "step": 13130 + }, + { + "epoch": 211.9430894308943, + "grad_norm": 0.7272772192955017, + "learning_rate": 0.0001, + "loss": 55.8779, + "step": 13140 + }, + { + "epoch": 212.09756097560975, + "grad_norm": 0.9501060843467712, + "learning_rate": 0.0001, + "loss": 52.049, + "step": 13150 + }, + { + "epoch": 212.260162601626, + "grad_norm": 0.8938133716583252, + "learning_rate": 0.0001, + "loss": 54.9783, + "step": 13160 + }, + { + "epoch": 212.4227642276423, + "grad_norm": 1.3081871271133423, + "learning_rate": 0.0001, + "loss": 54.1436, + "step": 13170 + }, + { + "epoch": 212.58536585365854, + "grad_norm": 1.182673454284668, + "learning_rate": 0.0001, + "loss": 55.5235, + "step": 13180 + }, + { + "epoch": 212.7479674796748, + "grad_norm": 0.7060847878456116, + "learning_rate": 0.0001, + "loss": 54.8775, + "step": 13190 + }, + { + "epoch": 212.91056910569105, + "grad_norm": 0.6047799587249756, + "learning_rate": 0.0001, + "loss": 56.6604, + "step": 13200 + }, + { + "epoch": 213.0650406504065, + "grad_norm": 0.8285370469093323, + "learning_rate": 0.0001, + "loss": 52.9922, + "step": 13210 + }, + { + "epoch": 213.22764227642276, + "grad_norm": 0.8437235951423645, + "learning_rate": 0.0001, + "loss": 54.9914, + "step": 13220 + }, + { + "epoch": 213.390243902439, + "grad_norm": 0.7426273822784424, + "learning_rate": 0.0001, + "loss": 54.7659, + "step": 13230 + }, + { + "epoch": 213.5528455284553, + "grad_norm": 0.8756979703903198, + "learning_rate": 0.0001, + "loss": 53.2097, + "step": 13240 + }, + { + "epoch": 213.71544715447155, + "grad_norm": 0.9625734090805054, + "learning_rate": 0.0001, + "loss": 55.2452, + "step": 13250 + }, + { + "epoch": 213.8780487804878, + "grad_norm": 1.5846962928771973, + "learning_rate": 0.0001, + "loss": 54.1381, + "step": 13260 + }, + { + "epoch": 214.03252032520325, + "grad_norm": 0.6845040321350098, + "learning_rate": 0.0001, + "loss": 50.8792, + "step": 13270 + }, + { + "epoch": 214.1951219512195, + "grad_norm": 1.0069342851638794, + "learning_rate": 0.0001, + "loss": 56.4215, + "step": 13280 + }, + { + "epoch": 214.35772357723576, + "grad_norm": 0.8575270771980286, + "learning_rate": 0.0001, + "loss": 54.7543, + "step": 13290 + }, + { + "epoch": 214.52032520325204, + "grad_norm": 1.1226451396942139, + "learning_rate": 0.0001, + "loss": 53.5045, + "step": 13300 + }, + { + "epoch": 214.6829268292683, + "grad_norm": 0.8359237313270569, + "learning_rate": 0.0001, + "loss": 54.7099, + "step": 13310 + }, + { + "epoch": 214.84552845528455, + "grad_norm": 0.823088526725769, + "learning_rate": 0.0001, + "loss": 55.2504, + "step": 13320 + }, + { + "epoch": 215.0, + "grad_norm": 0.9530013799667358, + "learning_rate": 0.0001, + "loss": 51.2342, + "step": 13330 + }, + { + "epoch": 215.16260162601625, + "grad_norm": 1.024066686630249, + "learning_rate": 0.0001, + "loss": 54.8409, + "step": 13340 + }, + { + "epoch": 215.3252032520325, + "grad_norm": 0.7555123567581177, + "learning_rate": 0.0001, + "loss": 54.6994, + "step": 13350 + }, + { + "epoch": 215.4878048780488, + "grad_norm": 0.8845812678337097, + "learning_rate": 0.0001, + "loss": 55.1865, + "step": 13360 + }, + { + "epoch": 215.65040650406505, + "grad_norm": 0.8159278035163879, + "learning_rate": 0.0001, + "loss": 54.4797, + "step": 13370 + }, + { + "epoch": 215.8130081300813, + "grad_norm": 0.8099329471588135, + "learning_rate": 0.0001, + "loss": 53.4086, + "step": 13380 + }, + { + "epoch": 215.97560975609755, + "grad_norm": 0.7204589247703552, + "learning_rate": 0.0001, + "loss": 54.1444, + "step": 13390 + }, + { + "epoch": 216.130081300813, + "grad_norm": 1.0179848670959473, + "learning_rate": 0.0001, + "loss": 52.6701, + "step": 13400 + }, + { + "epoch": 216.29268292682926, + "grad_norm": 0.7029995322227478, + "learning_rate": 0.0001, + "loss": 55.2125, + "step": 13410 + }, + { + "epoch": 216.45528455284554, + "grad_norm": 0.7679967880249023, + "learning_rate": 0.0001, + "loss": 55.5851, + "step": 13420 + }, + { + "epoch": 216.6178861788618, + "grad_norm": 0.925176739692688, + "learning_rate": 0.0001, + "loss": 54.6698, + "step": 13430 + }, + { + "epoch": 216.78048780487805, + "grad_norm": 1.0775889158248901, + "learning_rate": 0.0001, + "loss": 53.8535, + "step": 13440 + }, + { + "epoch": 216.9430894308943, + "grad_norm": 1.9590994119644165, + "learning_rate": 0.0001, + "loss": 53.3539, + "step": 13450 + }, + { + "epoch": 217.09756097560975, + "grad_norm": 1.1416126489639282, + "learning_rate": 0.0001, + "loss": 53.07, + "step": 13460 + }, + { + "epoch": 217.260162601626, + "grad_norm": 0.6063092947006226, + "learning_rate": 0.0001, + "loss": 55.2222, + "step": 13470 + }, + { + "epoch": 217.4227642276423, + "grad_norm": 1.082871913909912, + "learning_rate": 0.0001, + "loss": 54.9424, + "step": 13480 + }, + { + "epoch": 217.58536585365854, + "grad_norm": 1.0098278522491455, + "learning_rate": 0.0001, + "loss": 55.5704, + "step": 13490 + }, + { + "epoch": 217.7479674796748, + "grad_norm": 0.7821612358093262, + "learning_rate": 0.0001, + "loss": 55.6112, + "step": 13500 + }, + { + "epoch": 217.91056910569105, + "grad_norm": 0.7899888753890991, + "learning_rate": 0.0001, + "loss": 54.2449, + "step": 13510 + }, + { + "epoch": 218.0650406504065, + "grad_norm": 0.8772909045219421, + "learning_rate": 0.0001, + "loss": 51.3149, + "step": 13520 + }, + { + "epoch": 218.22764227642276, + "grad_norm": 0.835057258605957, + "learning_rate": 0.0001, + "loss": 54.9729, + "step": 13530 + }, + { + "epoch": 218.390243902439, + "grad_norm": 0.9645805954933167, + "learning_rate": 0.0001, + "loss": 53.1176, + "step": 13540 + }, + { + "epoch": 218.5528455284553, + "grad_norm": 1.1908318996429443, + "learning_rate": 0.0001, + "loss": 54.3461, + "step": 13550 + }, + { + "epoch": 218.71544715447155, + "grad_norm": 0.8490066528320312, + "learning_rate": 0.0001, + "loss": 56.1804, + "step": 13560 + }, + { + "epoch": 218.8780487804878, + "grad_norm": 0.855278491973877, + "learning_rate": 0.0001, + "loss": 54.6583, + "step": 13570 + }, + { + "epoch": 219.03252032520325, + "grad_norm": 0.8656384944915771, + "learning_rate": 0.0001, + "loss": 51.4255, + "step": 13580 + }, + { + "epoch": 219.1951219512195, + "grad_norm": 0.8291656970977783, + "learning_rate": 0.0001, + "loss": 55.1752, + "step": 13590 + }, + { + "epoch": 219.35772357723576, + "grad_norm": 0.9894065260887146, + "learning_rate": 0.0001, + "loss": 52.9551, + "step": 13600 + }, + { + "epoch": 219.52032520325204, + "grad_norm": 0.7791743278503418, + "learning_rate": 0.0001, + "loss": 53.2005, + "step": 13610 + }, + { + "epoch": 219.6829268292683, + "grad_norm": 0.8251843452453613, + "learning_rate": 0.0001, + "loss": 54.3272, + "step": 13620 + }, + { + "epoch": 219.84552845528455, + "grad_norm": 0.6909898519515991, + "learning_rate": 0.0001, + "loss": 53.4259, + "step": 13630 + }, + { + "epoch": 220.0, + "grad_norm": 0.7680906057357788, + "learning_rate": 0.0001, + "loss": 53.6018, + "step": 13640 + }, + { + "epoch": 220.16260162601625, + "grad_norm": 1.1677294969558716, + "learning_rate": 0.0001, + "loss": 53.7904, + "step": 13650 + }, + { + "epoch": 220.3252032520325, + "grad_norm": 0.8123244047164917, + "learning_rate": 0.0001, + "loss": 55.255, + "step": 13660 + }, + { + "epoch": 220.4878048780488, + "grad_norm": 0.8476759791374207, + "learning_rate": 0.0001, + "loss": 56.5699, + "step": 13670 + }, + { + "epoch": 220.65040650406505, + "grad_norm": 0.9571487903594971, + "learning_rate": 0.0001, + "loss": 54.228, + "step": 13680 + }, + { + "epoch": 220.8130081300813, + "grad_norm": 0.7796895503997803, + "learning_rate": 0.0001, + "loss": 54.8423, + "step": 13690 + }, + { + "epoch": 220.97560975609755, + "grad_norm": 0.9055293798446655, + "learning_rate": 0.0001, + "loss": 53.7139, + "step": 13700 + }, + { + "epoch": 221.130081300813, + "grad_norm": 0.7490646839141846, + "learning_rate": 0.0001, + "loss": 52.4622, + "step": 13710 + }, + { + "epoch": 221.29268292682926, + "grad_norm": 0.7626569271087646, + "learning_rate": 0.0001, + "loss": 55.8454, + "step": 13720 + }, + { + "epoch": 221.45528455284554, + "grad_norm": 0.9692214131355286, + "learning_rate": 0.0001, + "loss": 55.0342, + "step": 13730 + }, + { + "epoch": 221.6178861788618, + "grad_norm": 0.8987938761711121, + "learning_rate": 0.0001, + "loss": 54.8382, + "step": 13740 + }, + { + "epoch": 221.78048780487805, + "grad_norm": 0.7676286101341248, + "learning_rate": 0.0001, + "loss": 55.5017, + "step": 13750 + }, + { + "epoch": 221.9430894308943, + "grad_norm": 0.8445277214050293, + "learning_rate": 0.0001, + "loss": 55.331, + "step": 13760 + }, + { + "epoch": 222.09756097560975, + "grad_norm": 0.7058215737342834, + "learning_rate": 0.0001, + "loss": 52.9581, + "step": 13770 + }, + { + "epoch": 222.260162601626, + "grad_norm": 0.7497864365577698, + "learning_rate": 0.0001, + "loss": 54.9104, + "step": 13780 + }, + { + "epoch": 222.4227642276423, + "grad_norm": 0.9665900468826294, + "learning_rate": 0.0001, + "loss": 55.9701, + "step": 13790 + }, + { + "epoch": 222.58536585365854, + "grad_norm": 0.9786218404769897, + "learning_rate": 0.0001, + "loss": 55.2512, + "step": 13800 + }, + { + "epoch": 222.7479674796748, + "grad_norm": 1.1519272327423096, + "learning_rate": 0.0001, + "loss": 56.0712, + "step": 13810 + }, + { + "epoch": 222.91056910569105, + "grad_norm": 0.8277210593223572, + "learning_rate": 0.0001, + "loss": 56.1046, + "step": 13820 + }, + { + "epoch": 223.0650406504065, + "grad_norm": 0.816068708896637, + "learning_rate": 0.0001, + "loss": 52.8655, + "step": 13830 + }, + { + "epoch": 223.22764227642276, + "grad_norm": 0.6874851584434509, + "learning_rate": 0.0001, + "loss": 54.6701, + "step": 13840 + }, + { + "epoch": 223.390243902439, + "grad_norm": 0.8549659848213196, + "learning_rate": 0.0001, + "loss": 55.4961, + "step": 13850 + }, + { + "epoch": 223.5528455284553, + "grad_norm": 0.9988876581192017, + "learning_rate": 0.0001, + "loss": 54.9155, + "step": 13860 + }, + { + "epoch": 223.71544715447155, + "grad_norm": 0.8321837782859802, + "learning_rate": 0.0001, + "loss": 54.0151, + "step": 13870 + }, + { + "epoch": 223.8780487804878, + "grad_norm": 0.730671763420105, + "learning_rate": 0.0001, + "loss": 53.601, + "step": 13880 + }, + { + "epoch": 224.03252032520325, + "grad_norm": 1.1248708963394165, + "learning_rate": 0.0001, + "loss": 51.4019, + "step": 13890 + }, + { + "epoch": 224.1951219512195, + "grad_norm": 1.034224033355713, + "learning_rate": 0.0001, + "loss": 54.3903, + "step": 13900 + }, + { + "epoch": 224.35772357723576, + "grad_norm": 1.0652328729629517, + "learning_rate": 0.0001, + "loss": 56.968, + "step": 13910 + }, + { + "epoch": 224.52032520325204, + "grad_norm": 0.8998368978500366, + "learning_rate": 0.0001, + "loss": 54.6571, + "step": 13920 + }, + { + "epoch": 224.6829268292683, + "grad_norm": 1.0750027894973755, + "learning_rate": 0.0001, + "loss": 53.7169, + "step": 13930 + }, + { + "epoch": 224.84552845528455, + "grad_norm": 1.1613131761550903, + "learning_rate": 0.0001, + "loss": 55.1695, + "step": 13940 + }, + { + "epoch": 225.0, + "grad_norm": 1.035953402519226, + "learning_rate": 0.0001, + "loss": 53.4469, + "step": 13950 + }, + { + "epoch": 225.16260162601625, + "grad_norm": 0.7781218886375427, + "learning_rate": 0.0001, + "loss": 55.1178, + "step": 13960 + }, + { + "epoch": 225.3252032520325, + "grad_norm": 0.9615271091461182, + "learning_rate": 0.0001, + "loss": 54.1518, + "step": 13970 + }, + { + "epoch": 225.4878048780488, + "grad_norm": 0.8908602595329285, + "learning_rate": 0.0001, + "loss": 56.4974, + "step": 13980 + }, + { + "epoch": 225.65040650406505, + "grad_norm": 1.3588823080062866, + "learning_rate": 0.0001, + "loss": 54.4211, + "step": 13990 + }, + { + "epoch": 225.8130081300813, + "grad_norm": 0.8439602255821228, + "learning_rate": 0.0001, + "loss": 53.8013, + "step": 14000 + }, + { + "epoch": 225.97560975609755, + "grad_norm": 0.8661660552024841, + "learning_rate": 0.0001, + "loss": 53.8126, + "step": 14010 + }, + { + "epoch": 226.130081300813, + "grad_norm": 1.3356671333312988, + "learning_rate": 0.0001, + "loss": 51.0263, + "step": 14020 + }, + { + "epoch": 226.29268292682926, + "grad_norm": 2.5738320350646973, + "learning_rate": 0.0001, + "loss": 55.0098, + "step": 14030 + }, + { + "epoch": 226.45528455284554, + "grad_norm": 0.9103851914405823, + "learning_rate": 0.0001, + "loss": 55.3346, + "step": 14040 + }, + { + "epoch": 226.6178861788618, + "grad_norm": 0.9767026901245117, + "learning_rate": 0.0001, + "loss": 51.9042, + "step": 14050 + }, + { + "epoch": 226.78048780487805, + "grad_norm": 0.7589291334152222, + "learning_rate": 0.0001, + "loss": 53.9771, + "step": 14060 + }, + { + "epoch": 226.9430894308943, + "grad_norm": 0.9664282202720642, + "learning_rate": 0.0001, + "loss": 54.2983, + "step": 14070 + }, + { + "epoch": 227.09756097560975, + "grad_norm": 0.7321500182151794, + "learning_rate": 0.0001, + "loss": 51.867, + "step": 14080 + }, + { + "epoch": 227.260162601626, + "grad_norm": 1.2257362604141235, + "learning_rate": 0.0001, + "loss": 53.7938, + "step": 14090 + }, + { + "epoch": 227.4227642276423, + "grad_norm": 1.1650798320770264, + "learning_rate": 0.0001, + "loss": 55.4182, + "step": 14100 + }, + { + "epoch": 227.58536585365854, + "grad_norm": 0.7246831059455872, + "learning_rate": 0.0001, + "loss": 55.1917, + "step": 14110 + }, + { + "epoch": 227.7479674796748, + "grad_norm": 0.8775220513343811, + "learning_rate": 0.0001, + "loss": 53.7243, + "step": 14120 + }, + { + "epoch": 227.91056910569105, + "grad_norm": 1.4759947061538696, + "learning_rate": 0.0001, + "loss": 54.2947, + "step": 14130 + }, + { + "epoch": 228.0650406504065, + "grad_norm": 1.6673812866210938, + "learning_rate": 0.0001, + "loss": 51.5433, + "step": 14140 + }, + { + "epoch": 228.22764227642276, + "grad_norm": 0.8332659006118774, + "learning_rate": 0.0001, + "loss": 56.2734, + "step": 14150 + }, + { + "epoch": 228.390243902439, + "grad_norm": 0.8206542134284973, + "learning_rate": 0.0001, + "loss": 55.9898, + "step": 14160 + }, + { + "epoch": 228.5528455284553, + "grad_norm": 0.7425916790962219, + "learning_rate": 0.0001, + "loss": 53.8255, + "step": 14170 + }, + { + "epoch": 228.71544715447155, + "grad_norm": 0.7315079569816589, + "learning_rate": 0.0001, + "loss": 54.7144, + "step": 14180 + }, + { + "epoch": 228.8780487804878, + "grad_norm": 0.9928258657455444, + "learning_rate": 0.0001, + "loss": 54.0064, + "step": 14190 + }, + { + "epoch": 229.03252032520325, + "grad_norm": 0.7837854623794556, + "learning_rate": 0.0001, + "loss": 51.6609, + "step": 14200 + }, + { + "epoch": 229.1951219512195, + "grad_norm": 0.90921550989151, + "learning_rate": 0.0001, + "loss": 55.787, + "step": 14210 + }, + { + "epoch": 229.35772357723576, + "grad_norm": 0.871849536895752, + "learning_rate": 0.0001, + "loss": 54.393, + "step": 14220 + }, + { + "epoch": 229.52032520325204, + "grad_norm": 1.0282469987869263, + "learning_rate": 0.0001, + "loss": 53.7822, + "step": 14230 + }, + { + "epoch": 229.6829268292683, + "grad_norm": 0.8177674412727356, + "learning_rate": 0.0001, + "loss": 53.9671, + "step": 14240 + }, + { + "epoch": 229.84552845528455, + "grad_norm": 0.993034303188324, + "learning_rate": 0.0001, + "loss": 54.4043, + "step": 14250 + }, + { + "epoch": 230.0, + "grad_norm": 0.8818158507347107, + "learning_rate": 0.0001, + "loss": 50.8848, + "step": 14260 + }, + { + "epoch": 230.16260162601625, + "grad_norm": 0.7449021935462952, + "learning_rate": 0.0001, + "loss": 54.0188, + "step": 14270 + }, + { + "epoch": 230.3252032520325, + "grad_norm": 0.7258269786834717, + "learning_rate": 0.0001, + "loss": 55.6941, + "step": 14280 + }, + { + "epoch": 230.4878048780488, + "grad_norm": 0.6683617234230042, + "learning_rate": 0.0001, + "loss": 55.1297, + "step": 14290 + }, + { + "epoch": 230.65040650406505, + "grad_norm": 1.3703839778900146, + "learning_rate": 0.0001, + "loss": 53.4986, + "step": 14300 + }, + { + "epoch": 230.8130081300813, + "grad_norm": 0.8369325995445251, + "learning_rate": 0.0001, + "loss": 54.2473, + "step": 14310 + }, + { + "epoch": 230.97560975609755, + "grad_norm": 0.8285283446311951, + "learning_rate": 0.0001, + "loss": 54.8075, + "step": 14320 + }, + { + "epoch": 231.130081300813, + "grad_norm": 0.9424141645431519, + "learning_rate": 0.0001, + "loss": 52.3665, + "step": 14330 + }, + { + "epoch": 231.29268292682926, + "grad_norm": 0.8255526423454285, + "learning_rate": 0.0001, + "loss": 54.6126, + "step": 14340 + }, + { + "epoch": 231.45528455284554, + "grad_norm": 1.006761074066162, + "learning_rate": 0.0001, + "loss": 53.7163, + "step": 14350 + }, + { + "epoch": 231.6178861788618, + "grad_norm": 0.9653576016426086, + "learning_rate": 0.0001, + "loss": 55.3647, + "step": 14360 + }, + { + "epoch": 231.78048780487805, + "grad_norm": 0.8040965795516968, + "learning_rate": 0.0001, + "loss": 55.3375, + "step": 14370 + }, + { + "epoch": 231.9430894308943, + "grad_norm": 0.9425880312919617, + "learning_rate": 0.0001, + "loss": 55.5746, + "step": 14380 + }, + { + "epoch": 232.09756097560975, + "grad_norm": 1.006316900253296, + "learning_rate": 0.0001, + "loss": 50.3978, + "step": 14390 + }, + { + "epoch": 232.260162601626, + "grad_norm": 1.136722207069397, + "learning_rate": 0.0001, + "loss": 54.1219, + "step": 14400 + }, + { + "epoch": 232.4227642276423, + "grad_norm": 0.8423946499824524, + "learning_rate": 0.0001, + "loss": 53.8679, + "step": 14410 + }, + { + "epoch": 232.58536585365854, + "grad_norm": 0.7676213979721069, + "learning_rate": 0.0001, + "loss": 54.5777, + "step": 14420 + }, + { + "epoch": 232.7479674796748, + "grad_norm": 2.3063645362854004, + "learning_rate": 0.0001, + "loss": 53.5352, + "step": 14430 + }, + { + "epoch": 232.91056910569105, + "grad_norm": 0.8056443333625793, + "learning_rate": 0.0001, + "loss": 55.2449, + "step": 14440 + }, + { + "epoch": 233.0650406504065, + "grad_norm": 1.1931108236312866, + "learning_rate": 0.0001, + "loss": 52.0586, + "step": 14450 + }, + { + "epoch": 233.22764227642276, + "grad_norm": 0.820271909236908, + "learning_rate": 0.0001, + "loss": 54.3902, + "step": 14460 + }, + { + "epoch": 233.390243902439, + "grad_norm": 1.0494563579559326, + "learning_rate": 0.0001, + "loss": 54.9187, + "step": 14470 + }, + { + "epoch": 233.5528455284553, + "grad_norm": 1.0478507280349731, + "learning_rate": 0.0001, + "loss": 55.2656, + "step": 14480 + }, + { + "epoch": 233.71544715447155, + "grad_norm": 0.8764968514442444, + "learning_rate": 0.0001, + "loss": 55.4161, + "step": 14490 + }, + { + "epoch": 233.8780487804878, + "grad_norm": 1.4083248376846313, + "learning_rate": 0.0001, + "loss": 55.1284, + "step": 14500 + }, + { + "epoch": 234.03252032520325, + "grad_norm": 0.9571195244789124, + "learning_rate": 0.0001, + "loss": 52.4321, + "step": 14510 + }, + { + "epoch": 234.1951219512195, + "grad_norm": 1.0617722272872925, + "learning_rate": 0.0001, + "loss": 54.0079, + "step": 14520 + }, + { + "epoch": 234.35772357723576, + "grad_norm": 0.8184088468551636, + "learning_rate": 0.0001, + "loss": 56.4753, + "step": 14530 + }, + { + "epoch": 234.52032520325204, + "grad_norm": 1.0400315523147583, + "learning_rate": 0.0001, + "loss": 54.4566, + "step": 14540 + }, + { + "epoch": 234.6829268292683, + "grad_norm": 0.9099937081336975, + "learning_rate": 0.0001, + "loss": 54.6234, + "step": 14550 + }, + { + "epoch": 234.84552845528455, + "grad_norm": 0.8048180341720581, + "learning_rate": 0.0001, + "loss": 56.1784, + "step": 14560 + }, + { + "epoch": 235.0, + "grad_norm": 0.888400673866272, + "learning_rate": 0.0001, + "loss": 50.9432, + "step": 14570 + }, + { + "epoch": 235.16260162601625, + "grad_norm": 0.8743310570716858, + "learning_rate": 0.0001, + "loss": 55.2681, + "step": 14580 + }, + { + "epoch": 235.3252032520325, + "grad_norm": 0.9016884565353394, + "learning_rate": 0.0001, + "loss": 53.566, + "step": 14590 + }, + { + "epoch": 235.4878048780488, + "grad_norm": 0.8748098015785217, + "learning_rate": 0.0001, + "loss": 52.3228, + "step": 14600 + }, + { + "epoch": 235.65040650406505, + "grad_norm": 1.3671938180923462, + "learning_rate": 0.0001, + "loss": 55.7807, + "step": 14610 + }, + { + "epoch": 235.8130081300813, + "grad_norm": 0.7671380043029785, + "learning_rate": 0.0001, + "loss": 53.2217, + "step": 14620 + }, + { + "epoch": 235.97560975609755, + "grad_norm": 1.0221309661865234, + "learning_rate": 0.0001, + "loss": 53.6954, + "step": 14630 + }, + { + "epoch": 236.130081300813, + "grad_norm": 1.9246286153793335, + "learning_rate": 0.0001, + "loss": 52.7675, + "step": 14640 + }, + { + "epoch": 236.29268292682926, + "grad_norm": 0.7186591029167175, + "learning_rate": 0.0001, + "loss": 54.508, + "step": 14650 + }, + { + "epoch": 236.45528455284554, + "grad_norm": 0.8058716654777527, + "learning_rate": 0.0001, + "loss": 54.0912, + "step": 14660 + }, + { + "epoch": 236.6178861788618, + "grad_norm": 0.7054926156997681, + "learning_rate": 0.0001, + "loss": 55.3318, + "step": 14670 + }, + { + "epoch": 236.78048780487805, + "grad_norm": 0.9968011379241943, + "learning_rate": 0.0001, + "loss": 55.3583, + "step": 14680 + }, + { + "epoch": 236.9430894308943, + "grad_norm": 1.5916420221328735, + "learning_rate": 0.0001, + "loss": 54.5985, + "step": 14690 + }, + { + "epoch": 237.09756097560975, + "grad_norm": 1.0716444253921509, + "learning_rate": 0.0001, + "loss": 53.3298, + "step": 14700 + }, + { + "epoch": 237.260162601626, + "grad_norm": 0.7287874221801758, + "learning_rate": 0.0001, + "loss": 54.7082, + "step": 14710 + }, + { + "epoch": 237.4227642276423, + "grad_norm": 0.860691487789154, + "learning_rate": 0.0001, + "loss": 52.5907, + "step": 14720 + }, + { + "epoch": 237.58536585365854, + "grad_norm": 0.8677813410758972, + "learning_rate": 0.0001, + "loss": 55.2442, + "step": 14730 + }, + { + "epoch": 237.7479674796748, + "grad_norm": 1.0112682580947876, + "learning_rate": 0.0001, + "loss": 54.6315, + "step": 14740 + }, + { + "epoch": 237.91056910569105, + "grad_norm": 0.6526017189025879, + "learning_rate": 0.0001, + "loss": 56.9737, + "step": 14750 + }, + { + "epoch": 238.0650406504065, + "grad_norm": 2.012220621109009, + "learning_rate": 0.0001, + "loss": 50.5104, + "step": 14760 + }, + { + "epoch": 238.22764227642276, + "grad_norm": 1.1385548114776611, + "learning_rate": 0.0001, + "loss": 54.953, + "step": 14770 + }, + { + "epoch": 238.390243902439, + "grad_norm": 0.915226936340332, + "learning_rate": 0.0001, + "loss": 53.3019, + "step": 14780 + }, + { + "epoch": 238.5528455284553, + "grad_norm": 1.0534974336624146, + "learning_rate": 0.0001, + "loss": 53.7377, + "step": 14790 + }, + { + "epoch": 238.71544715447155, + "grad_norm": 0.6639984846115112, + "learning_rate": 0.0001, + "loss": 51.6132, + "step": 14800 + }, + { + "epoch": 238.8780487804878, + "grad_norm": 1.1607130765914917, + "learning_rate": 0.0001, + "loss": 51.6656, + "step": 14810 + }, + { + "epoch": 239.03252032520325, + "grad_norm": 0.8796831965446472, + "learning_rate": 0.0001, + "loss": 53.0906, + "step": 14820 + }, + { + "epoch": 239.1951219512195, + "grad_norm": 1.3613159656524658, + "learning_rate": 0.0001, + "loss": 54.371, + "step": 14830 + }, + { + "epoch": 239.35772357723576, + "grad_norm": 1.1560760736465454, + "learning_rate": 0.0001, + "loss": 53.6194, + "step": 14840 + }, + { + "epoch": 239.52032520325204, + "grad_norm": 0.8476263880729675, + "learning_rate": 0.0001, + "loss": 52.5599, + "step": 14850 + }, + { + "epoch": 239.6829268292683, + "grad_norm": 0.7116367816925049, + "learning_rate": 0.0001, + "loss": 54.79, + "step": 14860 + }, + { + "epoch": 239.84552845528455, + "grad_norm": 1.1638411283493042, + "learning_rate": 0.0001, + "loss": 53.5391, + "step": 14870 + }, + { + "epoch": 240.0, + "grad_norm": 0.947271466255188, + "learning_rate": 0.0001, + "loss": 51.3252, + "step": 14880 + }, + { + "epoch": 240.16260162601625, + "grad_norm": 0.960483968257904, + "learning_rate": 0.0001, + "loss": 54.3772, + "step": 14890 + }, + { + "epoch": 240.3252032520325, + "grad_norm": 0.841806948184967, + "learning_rate": 0.0001, + "loss": 52.1828, + "step": 14900 + }, + { + "epoch": 240.4878048780488, + "grad_norm": 0.9069699645042419, + "learning_rate": 0.0001, + "loss": 52.5106, + "step": 14910 + }, + { + "epoch": 240.65040650406505, + "grad_norm": 1.1150813102722168, + "learning_rate": 0.0001, + "loss": 56.0098, + "step": 14920 + }, + { + "epoch": 240.8130081300813, + "grad_norm": 0.7638039588928223, + "learning_rate": 0.0001, + "loss": 53.7415, + "step": 14930 + }, + { + "epoch": 240.97560975609755, + "grad_norm": 1.1885515451431274, + "learning_rate": 0.0001, + "loss": 55.089, + "step": 14940 + }, + { + "epoch": 241.130081300813, + "grad_norm": 1.0159634351730347, + "learning_rate": 0.0001, + "loss": 51.1617, + "step": 14950 + }, + { + "epoch": 241.29268292682926, + "grad_norm": 1.0128728151321411, + "learning_rate": 0.0001, + "loss": 53.957, + "step": 14960 + }, + { + "epoch": 241.45528455284554, + "grad_norm": 0.9399421811103821, + "learning_rate": 0.0001, + "loss": 55.1805, + "step": 14970 + }, + { + "epoch": 241.6178861788618, + "grad_norm": 0.7499178051948547, + "learning_rate": 0.0001, + "loss": 53.4352, + "step": 14980 + }, + { + "epoch": 241.78048780487805, + "grad_norm": 0.6498700380325317, + "learning_rate": 0.0001, + "loss": 54.0107, + "step": 14990 + }, + { + "epoch": 241.9430894308943, + "grad_norm": 0.7191994190216064, + "learning_rate": 0.0001, + "loss": 56.2089, + "step": 15000 + }, + { + "epoch": 242.09756097560975, + "grad_norm": 1.18024742603302, + "learning_rate": 0.0001, + "loss": 51.6447, + "step": 15010 + }, + { + "epoch": 242.260162601626, + "grad_norm": 0.9039168953895569, + "learning_rate": 0.0001, + "loss": 53.4341, + "step": 15020 + }, + { + "epoch": 242.4227642276423, + "grad_norm": 1.0368373394012451, + "learning_rate": 0.0001, + "loss": 53.7973, + "step": 15030 + }, + { + "epoch": 242.58536585365854, + "grad_norm": 0.8648324012756348, + "learning_rate": 0.0001, + "loss": 55.773, + "step": 15040 + }, + { + "epoch": 242.7479674796748, + "grad_norm": 0.7960569262504578, + "learning_rate": 0.0001, + "loss": 53.6785, + "step": 15050 + }, + { + "epoch": 242.91056910569105, + "grad_norm": 0.771918773651123, + "learning_rate": 0.0001, + "loss": 54.7576, + "step": 15060 + }, + { + "epoch": 243.0650406504065, + "grad_norm": 1.105131983757019, + "learning_rate": 0.0001, + "loss": 51.4313, + "step": 15070 + }, + { + "epoch": 243.22764227642276, + "grad_norm": 1.053236722946167, + "learning_rate": 0.0001, + "loss": 54.1687, + "step": 15080 + }, + { + "epoch": 243.390243902439, + "grad_norm": 1.4011870622634888, + "learning_rate": 0.0001, + "loss": 55.7092, + "step": 15090 + }, + { + "epoch": 243.5528455284553, + "grad_norm": 0.8540129065513611, + "learning_rate": 0.0001, + "loss": 54.2341, + "step": 15100 + }, + { + "epoch": 243.71544715447155, + "grad_norm": 1.666799545288086, + "learning_rate": 0.0001, + "loss": 52.3724, + "step": 15110 + }, + { + "epoch": 243.8780487804878, + "grad_norm": 1.0199878215789795, + "learning_rate": 0.0001, + "loss": 54.5811, + "step": 15120 + }, + { + "epoch": 244.03252032520325, + "grad_norm": 0.8971478939056396, + "learning_rate": 0.0001, + "loss": 52.6352, + "step": 15130 + }, + { + "epoch": 244.1951219512195, + "grad_norm": 0.6896243691444397, + "learning_rate": 0.0001, + "loss": 53.9287, + "step": 15140 + }, + { + "epoch": 244.35772357723576, + "grad_norm": 0.6954209208488464, + "learning_rate": 0.0001, + "loss": 53.2609, + "step": 15150 + }, + { + "epoch": 244.52032520325204, + "grad_norm": 0.8604191541671753, + "learning_rate": 0.0001, + "loss": 55.3024, + "step": 15160 + }, + { + "epoch": 244.6829268292683, + "grad_norm": 1.3844572305679321, + "learning_rate": 0.0001, + "loss": 53.6013, + "step": 15170 + }, + { + "epoch": 244.84552845528455, + "grad_norm": 1.3308122158050537, + "learning_rate": 0.0001, + "loss": 53.0818, + "step": 15180 + }, + { + "epoch": 245.0, + "grad_norm": 0.9475764632225037, + "learning_rate": 0.0001, + "loss": 51.509, + "step": 15190 + }, + { + "epoch": 245.16260162601625, + "grad_norm": 0.8317999243736267, + "learning_rate": 0.0001, + "loss": 53.8405, + "step": 15200 + }, + { + "epoch": 245.3252032520325, + "grad_norm": 0.7647470235824585, + "learning_rate": 0.0001, + "loss": 55.3181, + "step": 15210 + }, + { + "epoch": 245.4878048780488, + "grad_norm": 0.8726988434791565, + "learning_rate": 0.0001, + "loss": 55.4537, + "step": 15220 + }, + { + "epoch": 245.65040650406505, + "grad_norm": 0.9479020237922668, + "learning_rate": 0.0001, + "loss": 53.913, + "step": 15230 + }, + { + "epoch": 245.8130081300813, + "grad_norm": 0.7102323770523071, + "learning_rate": 0.0001, + "loss": 54.3977, + "step": 15240 + }, + { + "epoch": 245.97560975609755, + "grad_norm": 0.7271092534065247, + "learning_rate": 0.0001, + "loss": 53.5612, + "step": 15250 + }, + { + "epoch": 246.130081300813, + "grad_norm": 0.9429534673690796, + "learning_rate": 0.0001, + "loss": 52.4, + "step": 15260 + }, + { + "epoch": 246.29268292682926, + "grad_norm": 1.7194772958755493, + "learning_rate": 0.0001, + "loss": 54.2578, + "step": 15270 + }, + { + "epoch": 246.45528455284554, + "grad_norm": 1.1360211372375488, + "learning_rate": 0.0001, + "loss": 52.4441, + "step": 15280 + }, + { + "epoch": 246.6178861788618, + "grad_norm": 0.8528667092323303, + "learning_rate": 0.0001, + "loss": 54.6957, + "step": 15290 + }, + { + "epoch": 246.78048780487805, + "grad_norm": 0.6910762786865234, + "learning_rate": 0.0001, + "loss": 54.7472, + "step": 15300 + }, + { + "epoch": 246.9430894308943, + "grad_norm": 0.6756943464279175, + "learning_rate": 0.0001, + "loss": 53.0433, + "step": 15310 + }, + { + "epoch": 247.09756097560975, + "grad_norm": 0.949343740940094, + "learning_rate": 0.0001, + "loss": 51.568, + "step": 15320 + }, + { + "epoch": 247.260162601626, + "grad_norm": 1.0063872337341309, + "learning_rate": 0.0001, + "loss": 55.162, + "step": 15330 + }, + { + "epoch": 247.4227642276423, + "grad_norm": 0.7869600057601929, + "learning_rate": 0.0001, + "loss": 53.789, + "step": 15340 + }, + { + "epoch": 247.58536585365854, + "grad_norm": 0.9456334114074707, + "learning_rate": 0.0001, + "loss": 54.3977, + "step": 15350 + }, + { + "epoch": 247.7479674796748, + "grad_norm": 0.6960943341255188, + "learning_rate": 0.0001, + "loss": 52.5222, + "step": 15360 + }, + { + "epoch": 247.91056910569105, + "grad_norm": 0.5755976438522339, + "learning_rate": 0.0001, + "loss": 53.8927, + "step": 15370 + }, + { + "epoch": 248.0650406504065, + "grad_norm": 2.368572235107422, + "learning_rate": 0.0001, + "loss": 51.5995, + "step": 15380 + }, + { + "epoch": 248.22764227642276, + "grad_norm": 0.8773524761199951, + "learning_rate": 0.0001, + "loss": 54.296, + "step": 15390 + }, + { + "epoch": 248.390243902439, + "grad_norm": 1.0561209917068481, + "learning_rate": 0.0001, + "loss": 51.7327, + "step": 15400 + }, + { + "epoch": 248.5528455284553, + "grad_norm": 1.0130037069320679, + "learning_rate": 0.0001, + "loss": 54.616, + "step": 15410 + }, + { + "epoch": 248.71544715447155, + "grad_norm": 0.8258432745933533, + "learning_rate": 0.0001, + "loss": 53.8808, + "step": 15420 + }, + { + "epoch": 248.8780487804878, + "grad_norm": 0.7326891422271729, + "learning_rate": 0.0001, + "loss": 53.3624, + "step": 15430 + }, + { + "epoch": 249.03252032520325, + "grad_norm": 1.2207914590835571, + "learning_rate": 0.0001, + "loss": 51.166, + "step": 15440 + }, + { + "epoch": 249.1951219512195, + "grad_norm": 0.9404299855232239, + "learning_rate": 0.0001, + "loss": 54.9002, + "step": 15450 + }, + { + "epoch": 249.35772357723576, + "grad_norm": 0.6494407057762146, + "learning_rate": 0.0001, + "loss": 54.4227, + "step": 15460 + }, + { + "epoch": 249.52032520325204, + "grad_norm": 1.112674355506897, + "learning_rate": 0.0001, + "loss": 52.1128, + "step": 15470 + }, + { + "epoch": 249.6829268292683, + "grad_norm": 0.8266693949699402, + "learning_rate": 0.0001, + "loss": 53.8957, + "step": 15480 + }, + { + "epoch": 249.84552845528455, + "grad_norm": 0.8604085445404053, + "learning_rate": 0.0001, + "loss": 54.8597, + "step": 15490 + }, + { + "epoch": 250.0, + "grad_norm": 0.8825786709785461, + "learning_rate": 0.0001, + "loss": 51.4052, + "step": 15500 + }, + { + "epoch": 250.16260162601625, + "grad_norm": 0.8595220446586609, + "learning_rate": 0.0001, + "loss": 56.6767, + "step": 15510 + }, + { + "epoch": 250.3252032520325, + "grad_norm": 0.9159016013145447, + "learning_rate": 0.0001, + "loss": 53.6622, + "step": 15520 + }, + { + "epoch": 250.4878048780488, + "grad_norm": 0.9752806425094604, + "learning_rate": 0.0001, + "loss": 53.7919, + "step": 15530 + }, + { + "epoch": 250.65040650406505, + "grad_norm": 1.299797773361206, + "learning_rate": 0.0001, + "loss": 52.582, + "step": 15540 + }, + { + "epoch": 250.8130081300813, + "grad_norm": 0.8046541213989258, + "learning_rate": 0.0001, + "loss": 56.1583, + "step": 15550 + }, + { + "epoch": 250.97560975609755, + "grad_norm": 0.8848250508308411, + "learning_rate": 0.0001, + "loss": 55.1164, + "step": 15560 + }, + { + "epoch": 251.130081300813, + "grad_norm": 0.9489118456840515, + "learning_rate": 0.0001, + "loss": 51.5053, + "step": 15570 + }, + { + "epoch": 251.29268292682926, + "grad_norm": 5.069701194763184, + "learning_rate": 0.0001, + "loss": 54.2582, + "step": 15580 + }, + { + "epoch": 251.45528455284554, + "grad_norm": 0.7503376603126526, + "learning_rate": 0.0001, + "loss": 52.8597, + "step": 15590 + }, + { + "epoch": 251.6178861788618, + "grad_norm": 0.7200372815132141, + "learning_rate": 0.0001, + "loss": 52.6846, + "step": 15600 + }, + { + "epoch": 251.78048780487805, + "grad_norm": 1.4221407175064087, + "learning_rate": 0.0001, + "loss": 52.2177, + "step": 15610 + }, + { + "epoch": 251.9430894308943, + "grad_norm": 0.7428510189056396, + "learning_rate": 0.0001, + "loss": 54.7994, + "step": 15620 + }, + { + "epoch": 252.09756097560975, + "grad_norm": 0.9019786715507507, + "learning_rate": 0.0001, + "loss": 51.3638, + "step": 15630 + }, + { + "epoch": 252.260162601626, + "grad_norm": 0.8459905385971069, + "learning_rate": 0.0001, + "loss": 55.4398, + "step": 15640 + }, + { + "epoch": 252.4227642276423, + "grad_norm": 1.040996789932251, + "learning_rate": 0.0001, + "loss": 51.7563, + "step": 15650 + }, + { + "epoch": 252.58536585365854, + "grad_norm": 0.9953641295433044, + "learning_rate": 0.0001, + "loss": 52.351, + "step": 15660 + }, + { + "epoch": 252.7479674796748, + "grad_norm": 1.0336418151855469, + "learning_rate": 0.0001, + "loss": 52.2774, + "step": 15670 + }, + { + "epoch": 252.91056910569105, + "grad_norm": 0.728695809841156, + "learning_rate": 0.0001, + "loss": 53.8307, + "step": 15680 + }, + { + "epoch": 253.0650406504065, + "grad_norm": 0.8020172715187073, + "learning_rate": 0.0001, + "loss": 50.6589, + "step": 15690 + }, + { + "epoch": 253.22764227642276, + "grad_norm": 1.0117321014404297, + "learning_rate": 0.0001, + "loss": 55.3505, + "step": 15700 + }, + { + "epoch": 253.390243902439, + "grad_norm": 0.8227602243423462, + "learning_rate": 0.0001, + "loss": 56.1819, + "step": 15710 + }, + { + "epoch": 253.5528455284553, + "grad_norm": 0.7399566173553467, + "learning_rate": 0.0001, + "loss": 54.7373, + "step": 15720 + }, + { + "epoch": 253.71544715447155, + "grad_norm": 1.1013562679290771, + "learning_rate": 0.0001, + "loss": 54.1074, + "step": 15730 + }, + { + "epoch": 253.8780487804878, + "grad_norm": 1.2620662450790405, + "learning_rate": 0.0001, + "loss": 53.1314, + "step": 15740 + }, + { + "epoch": 254.03252032520325, + "grad_norm": 0.713722825050354, + "learning_rate": 0.0001, + "loss": 50.3889, + "step": 15750 + }, + { + "epoch": 254.1951219512195, + "grad_norm": 1.7682756185531616, + "learning_rate": 0.0001, + "loss": 55.0939, + "step": 15760 + }, + { + "epoch": 254.35772357723576, + "grad_norm": 2.202893018722534, + "learning_rate": 0.0001, + "loss": 53.8187, + "step": 15770 + }, + { + "epoch": 254.52032520325204, + "grad_norm": 1.1321382522583008, + "learning_rate": 0.0001, + "loss": 53.4802, + "step": 15780 + }, + { + "epoch": 254.6829268292683, + "grad_norm": 0.7600924968719482, + "learning_rate": 0.0001, + "loss": 54.0337, + "step": 15790 + }, + { + "epoch": 254.84552845528455, + "grad_norm": 0.7733964323997498, + "learning_rate": 0.0001, + "loss": 55.608, + "step": 15800 + }, + { + "epoch": 255.0, + "grad_norm": 0.8145171403884888, + "learning_rate": 0.0001, + "loss": 50.7658, + "step": 15810 + }, + { + "epoch": 255.16260162601625, + "grad_norm": 0.8330771327018738, + "learning_rate": 0.0001, + "loss": 52.5933, + "step": 15820 + }, + { + "epoch": 255.3252032520325, + "grad_norm": 0.87321537733078, + "learning_rate": 0.0001, + "loss": 55.3489, + "step": 15830 + }, + { + "epoch": 255.4878048780488, + "grad_norm": 0.8081530928611755, + "learning_rate": 0.0001, + "loss": 55.5343, + "step": 15840 + }, + { + "epoch": 255.65040650406505, + "grad_norm": 1.308362603187561, + "learning_rate": 0.0001, + "loss": 54.2978, + "step": 15850 + }, + { + "epoch": 255.8130081300813, + "grad_norm": 0.8281819820404053, + "learning_rate": 0.0001, + "loss": 54.8497, + "step": 15860 + }, + { + "epoch": 255.97560975609755, + "grad_norm": 0.9533831477165222, + "learning_rate": 0.0001, + "loss": 52.4189, + "step": 15870 + }, + { + "epoch": 256.130081300813, + "grad_norm": 1.1174825429916382, + "learning_rate": 0.0001, + "loss": 51.8444, + "step": 15880 + }, + { + "epoch": 256.2926829268293, + "grad_norm": 0.6530044078826904, + "learning_rate": 0.0001, + "loss": 53.4939, + "step": 15890 + }, + { + "epoch": 256.4552845528455, + "grad_norm": 1.254204511642456, + "learning_rate": 0.0001, + "loss": 53.9468, + "step": 15900 + }, + { + "epoch": 256.6178861788618, + "grad_norm": 0.8999534845352173, + "learning_rate": 0.0001, + "loss": 55.0074, + "step": 15910 + }, + { + "epoch": 256.780487804878, + "grad_norm": 0.7126501798629761, + "learning_rate": 0.0001, + "loss": 53.0478, + "step": 15920 + }, + { + "epoch": 256.9430894308943, + "grad_norm": 0.7162296772003174, + "learning_rate": 0.0001, + "loss": 54.1921, + "step": 15930 + }, + { + "epoch": 257.0975609756098, + "grad_norm": 0.7281424403190613, + "learning_rate": 0.0001, + "loss": 50.6751, + "step": 15940 + }, + { + "epoch": 257.260162601626, + "grad_norm": 1.1298840045928955, + "learning_rate": 0.0001, + "loss": 54.8229, + "step": 15950 + }, + { + "epoch": 257.4227642276423, + "grad_norm": 0.6440666317939758, + "learning_rate": 0.0001, + "loss": 53.4585, + "step": 15960 + }, + { + "epoch": 257.5853658536585, + "grad_norm": 0.8525786995887756, + "learning_rate": 0.0001, + "loss": 52.6461, + "step": 15970 + }, + { + "epoch": 257.7479674796748, + "grad_norm": 1.1225342750549316, + "learning_rate": 0.0001, + "loss": 53.8119, + "step": 15980 + }, + { + "epoch": 257.9105691056911, + "grad_norm": 1.1013641357421875, + "learning_rate": 0.0001, + "loss": 52.155, + "step": 15990 + }, + { + "epoch": 258.0650406504065, + "grad_norm": 0.9112017154693604, + "learning_rate": 0.0001, + "loss": 50.1069, + "step": 16000 + }, + { + "epoch": 258.2276422764228, + "grad_norm": 0.725796103477478, + "learning_rate": 0.0001, + "loss": 53.3874, + "step": 16010 + }, + { + "epoch": 258.390243902439, + "grad_norm": 1.0806035995483398, + "learning_rate": 0.0001, + "loss": 53.3612, + "step": 16020 + }, + { + "epoch": 258.5528455284553, + "grad_norm": 0.9333754181861877, + "learning_rate": 0.0001, + "loss": 53.5284, + "step": 16030 + }, + { + "epoch": 258.7154471544715, + "grad_norm": 0.8981278538703918, + "learning_rate": 0.0001, + "loss": 53.5343, + "step": 16040 + }, + { + "epoch": 258.8780487804878, + "grad_norm": 0.9149916768074036, + "learning_rate": 0.0001, + "loss": 54.3175, + "step": 16050 + }, + { + "epoch": 259.0325203252033, + "grad_norm": 0.860251784324646, + "learning_rate": 0.0001, + "loss": 50.5538, + "step": 16060 + }, + { + "epoch": 259.1951219512195, + "grad_norm": 0.673319399356842, + "learning_rate": 0.0001, + "loss": 51.249, + "step": 16070 + }, + { + "epoch": 259.3577235772358, + "grad_norm": 0.7712639570236206, + "learning_rate": 0.0001, + "loss": 53.0692, + "step": 16080 + }, + { + "epoch": 259.520325203252, + "grad_norm": 1.2832105159759521, + "learning_rate": 0.0001, + "loss": 53.4945, + "step": 16090 + }, + { + "epoch": 259.6829268292683, + "grad_norm": 0.8464673161506653, + "learning_rate": 0.0001, + "loss": 52.935, + "step": 16100 + }, + { + "epoch": 259.8455284552846, + "grad_norm": 0.9899376034736633, + "learning_rate": 0.0001, + "loss": 53.0332, + "step": 16110 + }, + { + "epoch": 260.0, + "grad_norm": 0.7303979992866516, + "learning_rate": 0.0001, + "loss": 51.089, + "step": 16120 + }, + { + "epoch": 260.1626016260163, + "grad_norm": 0.9846875667572021, + "learning_rate": 0.0001, + "loss": 53.8519, + "step": 16130 + }, + { + "epoch": 260.3252032520325, + "grad_norm": 1.8243305683135986, + "learning_rate": 0.0001, + "loss": 53.5924, + "step": 16140 + }, + { + "epoch": 260.4878048780488, + "grad_norm": 0.7334924936294556, + "learning_rate": 0.0001, + "loss": 52.7568, + "step": 16150 + }, + { + "epoch": 260.650406504065, + "grad_norm": 0.7590137720108032, + "learning_rate": 0.0001, + "loss": 53.3284, + "step": 16160 + }, + { + "epoch": 260.8130081300813, + "grad_norm": 1.0780458450317383, + "learning_rate": 0.0001, + "loss": 54.5947, + "step": 16170 + }, + { + "epoch": 260.9756097560976, + "grad_norm": 0.7191483974456787, + "learning_rate": 0.0001, + "loss": 53.6848, + "step": 16180 + }, + { + "epoch": 261.130081300813, + "grad_norm": 0.9690728783607483, + "learning_rate": 0.0001, + "loss": 50.6834, + "step": 16190 + }, + { + "epoch": 261.2926829268293, + "grad_norm": 1.1332905292510986, + "learning_rate": 0.0001, + "loss": 51.8668, + "step": 16200 + }, + { + "epoch": 261.4552845528455, + "grad_norm": 1.068014144897461, + "learning_rate": 0.0001, + "loss": 55.1814, + "step": 16210 + }, + { + "epoch": 261.6178861788618, + "grad_norm": 0.9111496806144714, + "learning_rate": 0.0001, + "loss": 54.1879, + "step": 16220 + }, + { + "epoch": 261.780487804878, + "grad_norm": 1.0428881645202637, + "learning_rate": 0.0001, + "loss": 52.5076, + "step": 16230 + }, + { + "epoch": 261.9430894308943, + "grad_norm": 1.2903265953063965, + "learning_rate": 0.0001, + "loss": 55.7951, + "step": 16240 + }, + { + "epoch": 262.0975609756098, + "grad_norm": 0.6565439701080322, + "learning_rate": 0.0001, + "loss": 50.634, + "step": 16250 + }, + { + "epoch": 262.260162601626, + "grad_norm": 0.9201452732086182, + "learning_rate": 0.0001, + "loss": 54.3772, + "step": 16260 + }, + { + "epoch": 262.4227642276423, + "grad_norm": 1.1108834743499756, + "learning_rate": 0.0001, + "loss": 53.7794, + "step": 16270 + }, + { + "epoch": 262.5853658536585, + "grad_norm": 0.8470554351806641, + "learning_rate": 0.0001, + "loss": 52.8131, + "step": 16280 + }, + { + "epoch": 262.7479674796748, + "grad_norm": 0.8877732753753662, + "learning_rate": 0.0001, + "loss": 52.5676, + "step": 16290 + }, + { + "epoch": 262.9105691056911, + "grad_norm": 0.7401541471481323, + "learning_rate": 0.0001, + "loss": 52.3978, + "step": 16300 + }, + { + "epoch": 263.0650406504065, + "grad_norm": 0.7072675228118896, + "learning_rate": 0.0001, + "loss": 50.1861, + "step": 16310 + }, + { + "epoch": 263.2276422764228, + "grad_norm": 0.760439395904541, + "learning_rate": 0.0001, + "loss": 55.0661, + "step": 16320 + }, + { + "epoch": 263.390243902439, + "grad_norm": 0.7817673087120056, + "learning_rate": 0.0001, + "loss": 54.6836, + "step": 16330 + }, + { + "epoch": 263.5528455284553, + "grad_norm": 1.389532446861267, + "learning_rate": 0.0001, + "loss": 54.1218, + "step": 16340 + }, + { + "epoch": 263.7154471544715, + "grad_norm": 0.9161108136177063, + "learning_rate": 0.0001, + "loss": 53.2299, + "step": 16350 + }, + { + "epoch": 263.8780487804878, + "grad_norm": 0.6257120966911316, + "learning_rate": 0.0001, + "loss": 54.9721, + "step": 16360 + }, + { + "epoch": 264.0325203252033, + "grad_norm": 0.9545691013336182, + "learning_rate": 0.0001, + "loss": 51.2657, + "step": 16370 + }, + { + "epoch": 264.1951219512195, + "grad_norm": 0.6417356133460999, + "learning_rate": 0.0001, + "loss": 53.8629, + "step": 16380 + }, + { + "epoch": 264.3577235772358, + "grad_norm": 1.0484164953231812, + "learning_rate": 0.0001, + "loss": 53.6551, + "step": 16390 + }, + { + "epoch": 264.520325203252, + "grad_norm": 0.8819120526313782, + "learning_rate": 0.0001, + "loss": 52.2838, + "step": 16400 + }, + { + "epoch": 264.6829268292683, + "grad_norm": 0.7509406208992004, + "learning_rate": 0.0001, + "loss": 52.0533, + "step": 16410 + }, + { + "epoch": 264.8455284552846, + "grad_norm": 0.8579386472702026, + "learning_rate": 0.0001, + "loss": 53.8813, + "step": 16420 + }, + { + "epoch": 265.0, + "grad_norm": 0.6586844325065613, + "learning_rate": 0.0001, + "loss": 50.9512, + "step": 16430 + }, + { + "epoch": 265.1626016260163, + "grad_norm": 0.956609845161438, + "learning_rate": 0.0001, + "loss": 53.606, + "step": 16440 + }, + { + "epoch": 265.3252032520325, + "grad_norm": 0.7211060523986816, + "learning_rate": 0.0001, + "loss": 52.3784, + "step": 16450 + }, + { + "epoch": 265.4878048780488, + "grad_norm": 1.010276198387146, + "learning_rate": 0.0001, + "loss": 52.9083, + "step": 16460 + }, + { + "epoch": 265.650406504065, + "grad_norm": 0.6324886679649353, + "learning_rate": 0.0001, + "loss": 52.3689, + "step": 16470 + }, + { + "epoch": 265.8130081300813, + "grad_norm": 0.8641326427459717, + "learning_rate": 0.0001, + "loss": 53.8859, + "step": 16480 + }, + { + "epoch": 265.9756097560976, + "grad_norm": 1.0725555419921875, + "learning_rate": 0.0001, + "loss": 55.014, + "step": 16490 + }, + { + "epoch": 266.130081300813, + "grad_norm": 1.107358694076538, + "learning_rate": 0.0001, + "loss": 52.6761, + "step": 16500 + }, + { + "epoch": 266.2926829268293, + "grad_norm": 0.8609797358512878, + "learning_rate": 0.0001, + "loss": 53.7422, + "step": 16510 + }, + { + "epoch": 266.4552845528455, + "grad_norm": 0.9487242698669434, + "learning_rate": 0.0001, + "loss": 53.3573, + "step": 16520 + }, + { + "epoch": 266.6178861788618, + "grad_norm": 0.9563118815422058, + "learning_rate": 0.0001, + "loss": 53.2892, + "step": 16530 + }, + { + "epoch": 266.780487804878, + "grad_norm": 0.8474820256233215, + "learning_rate": 0.0001, + "loss": 52.946, + "step": 16540 + }, + { + "epoch": 266.9430894308943, + "grad_norm": 1.1074727773666382, + "learning_rate": 0.0001, + "loss": 53.2462, + "step": 16550 + }, + { + "epoch": 267.0975609756098, + "grad_norm": 1.1075152158737183, + "learning_rate": 0.0001, + "loss": 51.9224, + "step": 16560 + }, + { + "epoch": 267.260162601626, + "grad_norm": 1.4104292392730713, + "learning_rate": 0.0001, + "loss": 54.2094, + "step": 16570 + }, + { + "epoch": 267.4227642276423, + "grad_norm": 1.3079161643981934, + "learning_rate": 0.0001, + "loss": 52.9606, + "step": 16580 + }, + { + "epoch": 267.5853658536585, + "grad_norm": 0.743679940700531, + "learning_rate": 0.0001, + "loss": 51.7113, + "step": 16590 + }, + { + "epoch": 267.7479674796748, + "grad_norm": 0.8136737942695618, + "learning_rate": 0.0001, + "loss": 52.5865, + "step": 16600 + }, + { + "epoch": 267.9105691056911, + "grad_norm": 0.8470249772071838, + "learning_rate": 0.0001, + "loss": 54.2772, + "step": 16610 + }, + { + "epoch": 268.0650406504065, + "grad_norm": 0.8683418035507202, + "learning_rate": 0.0001, + "loss": 49.6146, + "step": 16620 + }, + { + "epoch": 268.2276422764228, + "grad_norm": 0.8021112084388733, + "learning_rate": 0.0001, + "loss": 54.0896, + "step": 16630 + }, + { + "epoch": 268.390243902439, + "grad_norm": 0.9896494150161743, + "learning_rate": 0.0001, + "loss": 53.8401, + "step": 16640 + }, + { + "epoch": 268.5528455284553, + "grad_norm": 0.9860379099845886, + "learning_rate": 0.0001, + "loss": 53.8633, + "step": 16650 + }, + { + "epoch": 268.7154471544715, + "grad_norm": 0.8762341141700745, + "learning_rate": 0.0001, + "loss": 54.3615, + "step": 16660 + }, + { + "epoch": 268.8780487804878, + "grad_norm": 1.1113921403884888, + "learning_rate": 0.0001, + "loss": 54.6138, + "step": 16670 + }, + { + "epoch": 269.0325203252033, + "grad_norm": 0.8770963549613953, + "learning_rate": 0.0001, + "loss": 51.442, + "step": 16680 + }, + { + "epoch": 269.1951219512195, + "grad_norm": 1.053673505783081, + "learning_rate": 0.0001, + "loss": 52.8322, + "step": 16690 + }, + { + "epoch": 269.3577235772358, + "grad_norm": 1.1643518209457397, + "learning_rate": 0.0001, + "loss": 55.6284, + "step": 16700 + }, + { + "epoch": 269.520325203252, + "grad_norm": 0.8801748752593994, + "learning_rate": 0.0001, + "loss": 52.4478, + "step": 16710 + }, + { + "epoch": 269.6829268292683, + "grad_norm": 0.778647780418396, + "learning_rate": 0.0001, + "loss": 51.5215, + "step": 16720 + }, + { + "epoch": 269.8455284552846, + "grad_norm": 0.8174335956573486, + "learning_rate": 0.0001, + "loss": 53.9705, + "step": 16730 + }, + { + "epoch": 270.0, + "grad_norm": 0.6597013473510742, + "learning_rate": 0.0001, + "loss": 51.8177, + "step": 16740 + }, + { + "epoch": 270.1626016260163, + "grad_norm": 0.7050663828849792, + "learning_rate": 0.0001, + "loss": 53.3996, + "step": 16750 + }, + { + "epoch": 270.3252032520325, + "grad_norm": 1.1230257749557495, + "learning_rate": 0.0001, + "loss": 52.296, + "step": 16760 + }, + { + "epoch": 270.4878048780488, + "grad_norm": 0.9516972303390503, + "learning_rate": 0.0001, + "loss": 52.884, + "step": 16770 + }, + { + "epoch": 270.650406504065, + "grad_norm": 0.6573398113250732, + "learning_rate": 0.0001, + "loss": 52.9429, + "step": 16780 + }, + { + "epoch": 270.8130081300813, + "grad_norm": 0.8624856472015381, + "learning_rate": 0.0001, + "loss": 53.047, + "step": 16790 + }, + { + "epoch": 270.9756097560976, + "grad_norm": 0.6784051656723022, + "learning_rate": 0.0001, + "loss": 52.5851, + "step": 16800 + }, + { + "epoch": 271.130081300813, + "grad_norm": 0.8619254231452942, + "learning_rate": 0.0001, + "loss": 50.1016, + "step": 16810 + }, + { + "epoch": 271.2926829268293, + "grad_norm": 0.7841241359710693, + "learning_rate": 0.0001, + "loss": 52.476, + "step": 16820 + }, + { + "epoch": 271.4552845528455, + "grad_norm": 0.7628340721130371, + "learning_rate": 0.0001, + "loss": 53.4445, + "step": 16830 + }, + { + "epoch": 271.6178861788618, + "grad_norm": 0.730200469493866, + "learning_rate": 0.0001, + "loss": 52.8895, + "step": 16840 + }, + { + "epoch": 271.780487804878, + "grad_norm": 0.9316534399986267, + "learning_rate": 0.0001, + "loss": 53.4486, + "step": 16850 + }, + { + "epoch": 271.9430894308943, + "grad_norm": 0.9244499206542969, + "learning_rate": 0.0001, + "loss": 53.2347, + "step": 16860 + }, + { + "epoch": 272.0975609756098, + "grad_norm": 0.8020139932632446, + "learning_rate": 0.0001, + "loss": 50.1569, + "step": 16870 + }, + { + "epoch": 272.260162601626, + "grad_norm": 0.9394644498825073, + "learning_rate": 0.0001, + "loss": 53.1552, + "step": 16880 + }, + { + "epoch": 272.4227642276423, + "grad_norm": 0.8599722385406494, + "learning_rate": 0.0001, + "loss": 54.9152, + "step": 16890 + }, + { + "epoch": 272.5853658536585, + "grad_norm": 0.7780150175094604, + "learning_rate": 0.0001, + "loss": 53.5196, + "step": 16900 + }, + { + "epoch": 272.7479674796748, + "grad_norm": 0.808452844619751, + "learning_rate": 0.0001, + "loss": 52.8295, + "step": 16910 + }, + { + "epoch": 272.9105691056911, + "grad_norm": 1.1989158391952515, + "learning_rate": 0.0001, + "loss": 53.5478, + "step": 16920 + }, + { + "epoch": 273.0650406504065, + "grad_norm": 0.899551272392273, + "learning_rate": 0.0001, + "loss": 50.1795, + "step": 16930 + }, + { + "epoch": 273.2276422764228, + "grad_norm": 1.118312120437622, + "learning_rate": 0.0001, + "loss": 53.3003, + "step": 16940 + }, + { + "epoch": 273.390243902439, + "grad_norm": 0.8345513939857483, + "learning_rate": 0.0001, + "loss": 54.3808, + "step": 16950 + }, + { + "epoch": 273.5528455284553, + "grad_norm": 0.7402225732803345, + "learning_rate": 0.0001, + "loss": 54.35, + "step": 16960 + }, + { + "epoch": 273.7154471544715, + "grad_norm": 0.6654228568077087, + "learning_rate": 0.0001, + "loss": 53.6087, + "step": 16970 + }, + { + "epoch": 273.8780487804878, + "grad_norm": 1.0572110414505005, + "learning_rate": 0.0001, + "loss": 54.4076, + "step": 16980 + }, + { + "epoch": 274.0325203252033, + "grad_norm": 1.0303733348846436, + "learning_rate": 0.0001, + "loss": 50.8672, + "step": 16990 + }, + { + "epoch": 274.1951219512195, + "grad_norm": 0.8627033233642578, + "learning_rate": 0.0001, + "loss": 54.56, + "step": 17000 + }, + { + "epoch": 274.3577235772358, + "grad_norm": 0.8935522437095642, + "learning_rate": 0.0001, + "loss": 53.1458, + "step": 17010 + }, + { + "epoch": 274.520325203252, + "grad_norm": 0.8734845519065857, + "learning_rate": 0.0001, + "loss": 52.6053, + "step": 17020 + }, + { + "epoch": 274.6829268292683, + "grad_norm": 0.9670326709747314, + "learning_rate": 0.0001, + "loss": 53.5859, + "step": 17030 + }, + { + "epoch": 274.8455284552846, + "grad_norm": 1.0919623374938965, + "learning_rate": 0.0001, + "loss": 53.6295, + "step": 17040 + }, + { + "epoch": 275.0, + "grad_norm": 1.3156044483184814, + "learning_rate": 0.0001, + "loss": 52.3769, + "step": 17050 + }, + { + "epoch": 275.1626016260163, + "grad_norm": 0.6406927108764648, + "learning_rate": 0.0001, + "loss": 53.6763, + "step": 17060 + }, + { + "epoch": 275.3252032520325, + "grad_norm": 0.9203193783760071, + "learning_rate": 0.0001, + "loss": 54.1218, + "step": 17070 + }, + { + "epoch": 275.4878048780488, + "grad_norm": 0.9155068397521973, + "learning_rate": 0.0001, + "loss": 52.3565, + "step": 17080 + }, + { + "epoch": 275.650406504065, + "grad_norm": 0.7740018963813782, + "learning_rate": 0.0001, + "loss": 50.0785, + "step": 17090 + }, + { + "epoch": 275.8130081300813, + "grad_norm": 1.077690601348877, + "learning_rate": 0.0001, + "loss": 53.8974, + "step": 17100 + }, + { + "epoch": 275.9756097560976, + "grad_norm": 2.2603039741516113, + "learning_rate": 0.0001, + "loss": 53.1792, + "step": 17110 + }, + { + "epoch": 276.130081300813, + "grad_norm": 0.9516298174858093, + "learning_rate": 0.0001, + "loss": 50.182, + "step": 17120 + }, + { + "epoch": 276.2926829268293, + "grad_norm": 0.6567034125328064, + "learning_rate": 0.0001, + "loss": 53.2896, + "step": 17130 + }, + { + "epoch": 276.4552845528455, + "grad_norm": 0.9521644115447998, + "learning_rate": 0.0001, + "loss": 55.1808, + "step": 17140 + }, + { + "epoch": 276.6178861788618, + "grad_norm": 0.992795467376709, + "learning_rate": 0.0001, + "loss": 52.4926, + "step": 17150 + }, + { + "epoch": 276.780487804878, + "grad_norm": 0.7574642896652222, + "learning_rate": 0.0001, + "loss": 53.9628, + "step": 17160 + }, + { + "epoch": 276.9430894308943, + "grad_norm": 0.7640664577484131, + "learning_rate": 0.0001, + "loss": 52.5506, + "step": 17170 + }, + { + "epoch": 277.0975609756098, + "grad_norm": 0.7062627673149109, + "learning_rate": 0.0001, + "loss": 50.5452, + "step": 17180 + }, + { + "epoch": 277.260162601626, + "grad_norm": 0.841123104095459, + "learning_rate": 0.0001, + "loss": 53.9044, + "step": 17190 + }, + { + "epoch": 277.4227642276423, + "grad_norm": 0.7484192848205566, + "learning_rate": 0.0001, + "loss": 54.8058, + "step": 17200 + }, + { + "epoch": 277.5853658536585, + "grad_norm": 0.73091059923172, + "learning_rate": 0.0001, + "loss": 54.5563, + "step": 17210 + }, + { + "epoch": 277.7479674796748, + "grad_norm": 0.824150025844574, + "learning_rate": 0.0001, + "loss": 54.471, + "step": 17220 + }, + { + "epoch": 277.9105691056911, + "grad_norm": 0.6294798254966736, + "learning_rate": 0.0001, + "loss": 52.4371, + "step": 17230 + }, + { + "epoch": 278.0650406504065, + "grad_norm": 1.339980125427246, + "learning_rate": 0.0001, + "loss": 50.6154, + "step": 17240 + }, + { + "epoch": 278.2276422764228, + "grad_norm": 0.8717513680458069, + "learning_rate": 0.0001, + "loss": 53.716, + "step": 17250 + }, + { + "epoch": 278.390243902439, + "grad_norm": 0.9326121807098389, + "learning_rate": 0.0001, + "loss": 53.7342, + "step": 17260 + }, + { + "epoch": 278.5528455284553, + "grad_norm": 4.456446170806885, + "learning_rate": 0.0001, + "loss": 53.2821, + "step": 17270 + }, + { + "epoch": 278.7154471544715, + "grad_norm": 0.7791627645492554, + "learning_rate": 0.0001, + "loss": 53.7802, + "step": 17280 + }, + { + "epoch": 278.8780487804878, + "grad_norm": 1.816591501235962, + "learning_rate": 0.0001, + "loss": 53.5946, + "step": 17290 + }, + { + "epoch": 279.0325203252033, + "grad_norm": 0.8470907211303711, + "learning_rate": 0.0001, + "loss": 50.8372, + "step": 17300 + }, + { + "epoch": 279.1951219512195, + "grad_norm": 1.0458787679672241, + "learning_rate": 0.0001, + "loss": 52.5709, + "step": 17310 + }, + { + "epoch": 279.3577235772358, + "grad_norm": 0.8115313053131104, + "learning_rate": 0.0001, + "loss": 52.5381, + "step": 17320 + }, + { + "epoch": 279.520325203252, + "grad_norm": 0.7429511547088623, + "learning_rate": 0.0001, + "loss": 51.3858, + "step": 17330 + }, + { + "epoch": 279.6829268292683, + "grad_norm": 0.7305215001106262, + "learning_rate": 0.0001, + "loss": 54.465, + "step": 17340 + }, + { + "epoch": 279.8455284552846, + "grad_norm": 0.9744503498077393, + "learning_rate": 0.0001, + "loss": 53.6857, + "step": 17350 + }, + { + "epoch": 280.0, + "grad_norm": 0.9988436698913574, + "learning_rate": 0.0001, + "loss": 49.6973, + "step": 17360 + }, + { + "epoch": 280.1626016260163, + "grad_norm": 0.9848176836967468, + "learning_rate": 0.0001, + "loss": 53.8973, + "step": 17370 + }, + { + "epoch": 280.3252032520325, + "grad_norm": 0.8104496002197266, + "learning_rate": 0.0001, + "loss": 52.2282, + "step": 17380 + }, + { + "epoch": 280.4878048780488, + "grad_norm": 0.8746921420097351, + "learning_rate": 0.0001, + "loss": 53.1227, + "step": 17390 + }, + { + "epoch": 280.650406504065, + "grad_norm": 0.6947713494300842, + "learning_rate": 0.0001, + "loss": 53.3872, + "step": 17400 + }, + { + "epoch": 280.8130081300813, + "grad_norm": 0.825151264667511, + "learning_rate": 0.0001, + "loss": 52.9118, + "step": 17410 + }, + { + "epoch": 280.9756097560976, + "grad_norm": 0.896131157875061, + "learning_rate": 0.0001, + "loss": 53.8015, + "step": 17420 + }, + { + "epoch": 281.130081300813, + "grad_norm": 1.0290859937667847, + "learning_rate": 0.0001, + "loss": 51.5315, + "step": 17430 + }, + { + "epoch": 281.2926829268293, + "grad_norm": 0.9260424971580505, + "learning_rate": 0.0001, + "loss": 53.5577, + "step": 17440 + }, + { + "epoch": 281.4552845528455, + "grad_norm": 1.005990982055664, + "learning_rate": 0.0001, + "loss": 52.8236, + "step": 17450 + }, + { + "epoch": 281.6178861788618, + "grad_norm": 0.7965306043624878, + "learning_rate": 0.0001, + "loss": 52.8063, + "step": 17460 + }, + { + "epoch": 281.780487804878, + "grad_norm": 0.9465731382369995, + "learning_rate": 0.0001, + "loss": 53.4346, + "step": 17470 + }, + { + "epoch": 281.9430894308943, + "grad_norm": 0.7939464449882507, + "learning_rate": 0.0001, + "loss": 54.6843, + "step": 17480 + }, + { + "epoch": 282.0975609756098, + "grad_norm": 1.227469563484192, + "learning_rate": 0.0001, + "loss": 50.1346, + "step": 17490 + }, + { + "epoch": 282.260162601626, + "grad_norm": 0.7291248440742493, + "learning_rate": 0.0001, + "loss": 53.5635, + "step": 17500 + }, + { + "epoch": 282.4227642276423, + "grad_norm": 1.0416418313980103, + "learning_rate": 0.0001, + "loss": 55.4623, + "step": 17510 + }, + { + "epoch": 282.5853658536585, + "grad_norm": 1.0839024782180786, + "learning_rate": 0.0001, + "loss": 55.5345, + "step": 17520 + }, + { + "epoch": 282.7479674796748, + "grad_norm": 0.9610557556152344, + "learning_rate": 0.0001, + "loss": 53.6116, + "step": 17530 + }, + { + "epoch": 282.9105691056911, + "grad_norm": 1.0386297702789307, + "learning_rate": 0.0001, + "loss": 54.6996, + "step": 17540 + }, + { + "epoch": 283.0650406504065, + "grad_norm": 1.0302402973175049, + "learning_rate": 0.0001, + "loss": 51.0403, + "step": 17550 + }, + { + "epoch": 283.2276422764228, + "grad_norm": 0.6170961260795593, + "learning_rate": 0.0001, + "loss": 55.0378, + "step": 17560 + }, + { + "epoch": 283.390243902439, + "grad_norm": 0.8771789073944092, + "learning_rate": 0.0001, + "loss": 53.1055, + "step": 17570 + }, + { + "epoch": 283.5528455284553, + "grad_norm": 0.8719056844711304, + "learning_rate": 0.0001, + "loss": 52.6589, + "step": 17580 + }, + { + "epoch": 283.7154471544715, + "grad_norm": 0.7630992531776428, + "learning_rate": 0.0001, + "loss": 52.0284, + "step": 17590 + }, + { + "epoch": 283.8780487804878, + "grad_norm": 1.3767468929290771, + "learning_rate": 0.0001, + "loss": 53.4048, + "step": 17600 + }, + { + "epoch": 284.0325203252033, + "grad_norm": 0.8653029799461365, + "learning_rate": 0.0001, + "loss": 52.3254, + "step": 17610 + }, + { + "epoch": 284.1951219512195, + "grad_norm": 1.428124189376831, + "learning_rate": 0.0001, + "loss": 53.5634, + "step": 17620 + }, + { + "epoch": 284.3577235772358, + "grad_norm": 0.8217633366584778, + "learning_rate": 0.0001, + "loss": 54.1003, + "step": 17630 + }, + { + "epoch": 284.520325203252, + "grad_norm": 0.5823901891708374, + "learning_rate": 0.0001, + "loss": 52.464, + "step": 17640 + }, + { + "epoch": 284.6829268292683, + "grad_norm": 0.9739524126052856, + "learning_rate": 0.0001, + "loss": 51.8379, + "step": 17650 + }, + { + "epoch": 284.8455284552846, + "grad_norm": 0.7030553221702576, + "learning_rate": 0.0001, + "loss": 52.7237, + "step": 17660 + }, + { + "epoch": 285.0, + "grad_norm": 0.8558897376060486, + "learning_rate": 0.0001, + "loss": 50.8923, + "step": 17670 + }, + { + "epoch": 285.1626016260163, + "grad_norm": 0.6686831116676331, + "learning_rate": 0.0001, + "loss": 53.3549, + "step": 17680 + }, + { + "epoch": 285.3252032520325, + "grad_norm": 0.7778508067131042, + "learning_rate": 0.0001, + "loss": 54.5794, + "step": 17690 + }, + { + "epoch": 285.4878048780488, + "grad_norm": 1.1182961463928223, + "learning_rate": 0.0001, + "loss": 52.9605, + "step": 17700 + }, + { + "epoch": 285.650406504065, + "grad_norm": 0.7487860321998596, + "learning_rate": 0.0001, + "loss": 52.3974, + "step": 17710 + }, + { + "epoch": 285.8130081300813, + "grad_norm": 0.9618871808052063, + "learning_rate": 0.0001, + "loss": 52.9826, + "step": 17720 + }, + { + "epoch": 285.9756097560976, + "grad_norm": 1.0091677904129028, + "learning_rate": 0.0001, + "loss": 52.8672, + "step": 17730 + }, + { + "epoch": 286.130081300813, + "grad_norm": 0.7941350936889648, + "learning_rate": 0.0001, + "loss": 48.3773, + "step": 17740 + }, + { + "epoch": 286.2926829268293, + "grad_norm": 0.6843161582946777, + "learning_rate": 0.0001, + "loss": 52.5306, + "step": 17750 + }, + { + "epoch": 286.4552845528455, + "grad_norm": 0.823552131652832, + "learning_rate": 0.0001, + "loss": 54.8838, + "step": 17760 + }, + { + "epoch": 286.6178861788618, + "grad_norm": 1.0429179668426514, + "learning_rate": 0.0001, + "loss": 54.5045, + "step": 17770 + }, + { + "epoch": 286.780487804878, + "grad_norm": 2.2908895015716553, + "learning_rate": 0.0001, + "loss": 52.9332, + "step": 17780 + }, + { + "epoch": 286.9430894308943, + "grad_norm": 0.8092592358589172, + "learning_rate": 0.0001, + "loss": 53.8568, + "step": 17790 + }, + { + "epoch": 287.0975609756098, + "grad_norm": 0.7603244185447693, + "learning_rate": 0.0001, + "loss": 50.3675, + "step": 17800 + }, + { + "epoch": 287.260162601626, + "grad_norm": 0.7366005182266235, + "learning_rate": 0.0001, + "loss": 54.4063, + "step": 17810 + }, + { + "epoch": 287.4227642276423, + "grad_norm": 0.6039050221443176, + "learning_rate": 0.0001, + "loss": 54.0833, + "step": 17820 + }, + { + "epoch": 287.5853658536585, + "grad_norm": 0.8206944465637207, + "learning_rate": 0.0001, + "loss": 52.9736, + "step": 17830 + }, + { + "epoch": 287.7479674796748, + "grad_norm": 0.9840274453163147, + "learning_rate": 0.0001, + "loss": 52.6705, + "step": 17840 + }, + { + "epoch": 287.9105691056911, + "grad_norm": 0.6055956482887268, + "learning_rate": 0.0001, + "loss": 54.7422, + "step": 17850 + }, + { + "epoch": 288.0650406504065, + "grad_norm": 1.0896904468536377, + "learning_rate": 0.0001, + "loss": 50.7414, + "step": 17860 + }, + { + "epoch": 288.2276422764228, + "grad_norm": 0.9435478448867798, + "learning_rate": 0.0001, + "loss": 52.4875, + "step": 17870 + }, + { + "epoch": 288.390243902439, + "grad_norm": 5.281745433807373, + "learning_rate": 0.0001, + "loss": 51.8199, + "step": 17880 + }, + { + "epoch": 288.5528455284553, + "grad_norm": 0.7859227061271667, + "learning_rate": 0.0001, + "loss": 56.0234, + "step": 17890 + }, + { + "epoch": 288.7154471544715, + "grad_norm": 1.0221906900405884, + "learning_rate": 0.0001, + "loss": 53.9615, + "step": 17900 + }, + { + "epoch": 288.8780487804878, + "grad_norm": 0.7079001069068909, + "learning_rate": 0.0001, + "loss": 52.2986, + "step": 17910 + }, + { + "epoch": 289.0325203252033, + "grad_norm": 1.4494272470474243, + "learning_rate": 0.0001, + "loss": 51.589, + "step": 17920 + }, + { + "epoch": 289.1951219512195, + "grad_norm": 1.3255219459533691, + "learning_rate": 0.0001, + "loss": 52.8764, + "step": 17930 + }, + { + "epoch": 289.3577235772358, + "grad_norm": 0.7966728806495667, + "learning_rate": 0.0001, + "loss": 53.5795, + "step": 17940 + }, + { + "epoch": 289.520325203252, + "grad_norm": 0.9541887044906616, + "learning_rate": 0.0001, + "loss": 52.9888, + "step": 17950 + }, + { + "epoch": 289.6829268292683, + "grad_norm": 0.8691675662994385, + "learning_rate": 0.0001, + "loss": 53.2108, + "step": 17960 + }, + { + "epoch": 289.8455284552846, + "grad_norm": 0.8235900402069092, + "learning_rate": 0.0001, + "loss": 54.4012, + "step": 17970 + }, + { + "epoch": 290.0, + "grad_norm": 0.819977879524231, + "learning_rate": 0.0001, + "loss": 49.2828, + "step": 17980 + }, + { + "epoch": 290.1626016260163, + "grad_norm": 0.676188051700592, + "learning_rate": 0.0001, + "loss": 52.0062, + "step": 17990 + }, + { + "epoch": 290.3252032520325, + "grad_norm": 1.1167227029800415, + "learning_rate": 0.0001, + "loss": 51.9235, + "step": 18000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 323, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.1863468605823255e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c7f229ebe1034caaf3e92006c8c7528c21af93d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517586849d044483051c7a59cec7c6ce389951878a7357ef837edb1cbc9983c8 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-18000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/latest new file mode 100644 index 0000000000000000000000000000000000000000..2a79fdc19587e6bc9de060e90633f3a151b04516 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/latest @@ -0,0 +1 @@ +global_step2000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f300291cb2b50c327b2bb1938326d9c66252ea81 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b3bde3828cd2527725111ecadea5455a4dba5ad236b53bbf37b1b450249fd1b +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..3a1b74842f1ec589fb7ae86e4474239cfa7094e9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:819b86e44bece7088259cf0f027fbd568c19a21e59611db49355a9b07fe9673f +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..eaf3db858cce0a5b537e033bd6f674dbee74a09c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c88e6e4e32ad3d77551b89131295930f76f556f426b797fd8b025b456f2a16a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..a3ff9f1dc7a814ae1c48e06b4f55248a9d3746f5 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08d60611c3ca3ae6d84f2ed14463bd552e570909ce569c6798c1a12b3a5da0c5 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..e841c512068f0731785a0ec62167af286afedac8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8a8edbc512111f98e6572d8bedea798e0cb9e4af22675c902434d1e3f4cc035 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..0db269000b782cdcd246efd6b895c9f8646de19d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb34cb08832d27b4f6ffdd7144be48084a8ca49e958214f9a193ab9375636f0a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..478e741669118621a2e320a02a865df42f94a0c4 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8c34a547003747dd5aaa64ab624a7aed6cd17a319102ad101bf9d91da5bf1f1 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..f85fd68fbe73797aa5caa9ed9ce2d66117832240 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08585e6e31118d8d77ab6f67d23a49fb38f752c353e75677f00b80f8dc61122e +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..9f95859087aef2fa1015fd47f94dea3a082c9bdc --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68520073e3ecd735f1ee9617bc1088abf8e1023eca76330d5389c83fe9991362 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..059097d31013bcb853c1f29bd20164bc1deb61d8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6a4e6ea22558ae7a3d868dc31596451ac2fbd9303bffcf03fa9584c5490e3ca +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..61b3123752ec393d4266a00d0f77b5d19fed6c4d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/trainer_state.json @@ -0,0 +1,1434 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 32.260162601626014, + "eval_steps": 2000, + "global_step": 2000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.16260162601626016, + "grad_norm": 23.72986602783203, + "learning_rate": 0.0001, + "loss": 271.6598, + "step": 10 + }, + { + "epoch": 0.3252032520325203, + "grad_norm": 18.99613380432129, + "learning_rate": 0.0001, + "loss": 233.029, + "step": 20 + }, + { + "epoch": 0.4878048780487805, + "grad_norm": 3.9815304279327393, + "learning_rate": 0.0001, + "loss": 231.0776, + "step": 30 + }, + { + "epoch": 0.6504065040650406, + "grad_norm": 4.641753196716309, + "learning_rate": 0.0001, + "loss": 228.6894, + "step": 40 + }, + { + "epoch": 0.8130081300813008, + "grad_norm": 3.000852108001709, + "learning_rate": 0.0001, + "loss": 228.4694, + "step": 50 + }, + { + "epoch": 0.975609756097561, + "grad_norm": 12.213018417358398, + "learning_rate": 0.0001, + "loss": 226.4258, + "step": 60 + }, + { + "epoch": 1.1300813008130082, + "grad_norm": 2.535187005996704, + "learning_rate": 0.0001, + "loss": 209.9772, + "step": 70 + }, + { + "epoch": 1.2926829268292683, + "grad_norm": 2.0984582901000977, + "learning_rate": 0.0001, + "loss": 217.2421, + "step": 80 + }, + { + "epoch": 1.4552845528455285, + "grad_norm": 3.4802305698394775, + "learning_rate": 0.0001, + "loss": 210.9241, + "step": 90 + }, + { + "epoch": 1.6178861788617886, + "grad_norm": 3.813626527786255, + "learning_rate": 0.0001, + "loss": 205.7566, + "step": 100 + }, + { + "epoch": 1.7804878048780488, + "grad_norm": 2.6378512382507324, + "learning_rate": 0.0001, + "loss": 201.8789, + "step": 110 + }, + { + "epoch": 1.943089430894309, + "grad_norm": 2.7502875328063965, + "learning_rate": 0.0001, + "loss": 196.373, + "step": 120 + }, + { + "epoch": 2.097560975609756, + "grad_norm": 2.659212112426758, + "learning_rate": 0.0001, + "loss": 183.1456, + "step": 130 + }, + { + "epoch": 2.2601626016260163, + "grad_norm": 2.513044595718384, + "learning_rate": 0.0001, + "loss": 190.1661, + "step": 140 + }, + { + "epoch": 2.4227642276422765, + "grad_norm": 2.3192145824432373, + "learning_rate": 0.0001, + "loss": 184.9997, + "step": 150 + }, + { + "epoch": 2.5853658536585367, + "grad_norm": 3.056708574295044, + "learning_rate": 0.0001, + "loss": 182.0164, + "step": 160 + }, + { + "epoch": 2.747967479674797, + "grad_norm": 2.9687108993530273, + "learning_rate": 0.0001, + "loss": 179.6539, + "step": 170 + }, + { + "epoch": 2.910569105691057, + "grad_norm": 2.1498093605041504, + "learning_rate": 0.0001, + "loss": 176.7159, + "step": 180 + }, + { + "epoch": 3.065040650406504, + "grad_norm": 3.0103771686553955, + "learning_rate": 0.0001, + "loss": 166.1703, + "step": 190 + }, + { + "epoch": 3.227642276422764, + "grad_norm": 2.992462635040283, + "learning_rate": 0.0001, + "loss": 171.124, + "step": 200 + }, + { + "epoch": 3.3902439024390243, + "grad_norm": 4.8617119789123535, + "learning_rate": 0.0001, + "loss": 168.2646, + "step": 210 + }, + { + "epoch": 3.5528455284552845, + "grad_norm": 3.1047444343566895, + "learning_rate": 0.0001, + "loss": 163.0889, + "step": 220 + }, + { + "epoch": 3.7154471544715446, + "grad_norm": 6.473227500915527, + "learning_rate": 0.0001, + "loss": 151.3667, + "step": 230 + }, + { + "epoch": 3.8780487804878048, + "grad_norm": 6.16747522354126, + "learning_rate": 0.0001, + "loss": 137.5915, + "step": 240 + }, + { + "epoch": 4.032520325203252, + "grad_norm": 7.994515419006348, + "learning_rate": 0.0001, + "loss": 118.8875, + "step": 250 + }, + { + "epoch": 4.195121951219512, + "grad_norm": 5.882101058959961, + "learning_rate": 0.0001, + "loss": 119.4793, + "step": 260 + }, + { + "epoch": 4.357723577235772, + "grad_norm": 12.32968521118164, + "learning_rate": 0.0001, + "loss": 113.1209, + "step": 270 + }, + { + "epoch": 4.520325203252033, + "grad_norm": 5.697225570678711, + "learning_rate": 0.0001, + "loss": 107.51, + "step": 280 + }, + { + "epoch": 4.682926829268292, + "grad_norm": 4.654167175292969, + "learning_rate": 0.0001, + "loss": 107.0172, + "step": 290 + }, + { + "epoch": 4.845528455284553, + "grad_norm": 2.9595484733581543, + "learning_rate": 0.0001, + "loss": 100.9164, + "step": 300 + }, + { + "epoch": 5.0, + "grad_norm": 21.242591857910156, + "learning_rate": 0.0001, + "loss": 94.9884, + "step": 310 + }, + { + "epoch": 5.16260162601626, + "grad_norm": 5.760314464569092, + "learning_rate": 0.0001, + "loss": 97.5901, + "step": 320 + }, + { + "epoch": 5.32520325203252, + "grad_norm": 3.1373329162597656, + "learning_rate": 0.0001, + "loss": 94.8485, + "step": 330 + }, + { + "epoch": 5.487804878048781, + "grad_norm": 2.3777568340301514, + "learning_rate": 0.0001, + "loss": 96.9982, + "step": 340 + }, + { + "epoch": 5.650406504065041, + "grad_norm": 3.0607948303222656, + "learning_rate": 0.0001, + "loss": 96.1586, + "step": 350 + }, + { + "epoch": 5.8130081300813, + "grad_norm": 4.168269634246826, + "learning_rate": 0.0001, + "loss": 94.5743, + "step": 360 + }, + { + "epoch": 5.975609756097561, + "grad_norm": 2.4383699893951416, + "learning_rate": 0.0001, + "loss": 95.5197, + "step": 370 + }, + { + "epoch": 6.130081300813008, + "grad_norm": 2.302259683609009, + "learning_rate": 0.0001, + "loss": 85.6599, + "step": 380 + }, + { + "epoch": 6.2926829268292686, + "grad_norm": 2.552753210067749, + "learning_rate": 0.0001, + "loss": 91.2999, + "step": 390 + }, + { + "epoch": 6.455284552845528, + "grad_norm": 2.2092080116271973, + "learning_rate": 0.0001, + "loss": 90.7041, + "step": 400 + }, + { + "epoch": 6.617886178861789, + "grad_norm": 2.2935690879821777, + "learning_rate": 0.0001, + "loss": 89.9949, + "step": 410 + }, + { + "epoch": 6.780487804878049, + "grad_norm": 2.487510919570923, + "learning_rate": 0.0001, + "loss": 86.9056, + "step": 420 + }, + { + "epoch": 6.943089430894309, + "grad_norm": 2.8630545139312744, + "learning_rate": 0.0001, + "loss": 88.3347, + "step": 430 + }, + { + "epoch": 7.097560975609756, + "grad_norm": 5.87056303024292, + "learning_rate": 0.0001, + "loss": 84.9632, + "step": 440 + }, + { + "epoch": 7.260162601626016, + "grad_norm": 2.587661027908325, + "learning_rate": 0.0001, + "loss": 86.2024, + "step": 450 + }, + { + "epoch": 7.4227642276422765, + "grad_norm": 3.5671255588531494, + "learning_rate": 0.0001, + "loss": 86.218, + "step": 460 + }, + { + "epoch": 7.585365853658536, + "grad_norm": 2.5036680698394775, + "learning_rate": 0.0001, + "loss": 86.5914, + "step": 470 + }, + { + "epoch": 7.747967479674797, + "grad_norm": 3.2656471729278564, + "learning_rate": 0.0001, + "loss": 82.6081, + "step": 480 + }, + { + "epoch": 7.9105691056910565, + "grad_norm": 2.5053915977478027, + "learning_rate": 0.0001, + "loss": 84.6116, + "step": 490 + }, + { + "epoch": 8.065040650406504, + "grad_norm": 2.1669957637786865, + "learning_rate": 0.0001, + "loss": 80.4354, + "step": 500 + }, + { + "epoch": 8.227642276422765, + "grad_norm": 4.2003703117370605, + "learning_rate": 0.0001, + "loss": 85.2286, + "step": 510 + }, + { + "epoch": 8.390243902439025, + "grad_norm": 3.1909608840942383, + "learning_rate": 0.0001, + "loss": 85.0358, + "step": 520 + }, + { + "epoch": 8.552845528455284, + "grad_norm": 1.8420121669769287, + "learning_rate": 0.0001, + "loss": 85.4647, + "step": 530 + }, + { + "epoch": 8.715447154471544, + "grad_norm": 1.9822801351547241, + "learning_rate": 0.0001, + "loss": 84.7134, + "step": 540 + }, + { + "epoch": 8.878048780487806, + "grad_norm": 2.936758518218994, + "learning_rate": 0.0001, + "loss": 83.8068, + "step": 550 + }, + { + "epoch": 9.032520325203253, + "grad_norm": 1.5713881254196167, + "learning_rate": 0.0001, + "loss": 80.0695, + "step": 560 + }, + { + "epoch": 9.195121951219512, + "grad_norm": 2.0584616661071777, + "learning_rate": 0.0001, + "loss": 84.5158, + "step": 570 + }, + { + "epoch": 9.357723577235772, + "grad_norm": 2.101391553878784, + "learning_rate": 0.0001, + "loss": 82.4719, + "step": 580 + }, + { + "epoch": 9.520325203252032, + "grad_norm": 1.9956543445587158, + "learning_rate": 0.0001, + "loss": 81.6715, + "step": 590 + }, + { + "epoch": 9.682926829268293, + "grad_norm": 2.7622063159942627, + "learning_rate": 0.0001, + "loss": 85.4178, + "step": 600 + }, + { + "epoch": 9.845528455284553, + "grad_norm": 2.2084789276123047, + "learning_rate": 0.0001, + "loss": 84.5702, + "step": 610 + }, + { + "epoch": 10.0, + "grad_norm": 1.9635663032531738, + "learning_rate": 0.0001, + "loss": 77.3838, + "step": 620 + }, + { + "epoch": 10.16260162601626, + "grad_norm": 2.5895581245422363, + "learning_rate": 0.0001, + "loss": 81.723, + "step": 630 + }, + { + "epoch": 10.32520325203252, + "grad_norm": 2.743727445602417, + "learning_rate": 0.0001, + "loss": 83.1332, + "step": 640 + }, + { + "epoch": 10.487804878048781, + "grad_norm": 2.024585008621216, + "learning_rate": 0.0001, + "loss": 81.5968, + "step": 650 + }, + { + "epoch": 10.65040650406504, + "grad_norm": 1.8254153728485107, + "learning_rate": 0.0001, + "loss": 78.9589, + "step": 660 + }, + { + "epoch": 10.8130081300813, + "grad_norm": 3.2015740871429443, + "learning_rate": 0.0001, + "loss": 79.1829, + "step": 670 + }, + { + "epoch": 10.975609756097562, + "grad_norm": 1.788453459739685, + "learning_rate": 0.0001, + "loss": 81.527, + "step": 680 + }, + { + "epoch": 11.130081300813009, + "grad_norm": 1.5744423866271973, + "learning_rate": 0.0001, + "loss": 74.9869, + "step": 690 + }, + { + "epoch": 11.292682926829269, + "grad_norm": 2.006002187728882, + "learning_rate": 0.0001, + "loss": 80.2677, + "step": 700 + }, + { + "epoch": 11.455284552845528, + "grad_norm": 1.8847148418426514, + "learning_rate": 0.0001, + "loss": 78.8709, + "step": 710 + }, + { + "epoch": 11.617886178861788, + "grad_norm": 1.8384478092193604, + "learning_rate": 0.0001, + "loss": 79.008, + "step": 720 + }, + { + "epoch": 11.78048780487805, + "grad_norm": 1.9486687183380127, + "learning_rate": 0.0001, + "loss": 76.1525, + "step": 730 + }, + { + "epoch": 11.94308943089431, + "grad_norm": 1.8644076585769653, + "learning_rate": 0.0001, + "loss": 77.5802, + "step": 740 + }, + { + "epoch": 12.097560975609756, + "grad_norm": 1.6035499572753906, + "learning_rate": 0.0001, + "loss": 75.2422, + "step": 750 + }, + { + "epoch": 12.260162601626016, + "grad_norm": 1.6935622692108154, + "learning_rate": 0.0001, + "loss": 77.5838, + "step": 760 + }, + { + "epoch": 12.422764227642276, + "grad_norm": 1.9327987432479858, + "learning_rate": 0.0001, + "loss": 78.3237, + "step": 770 + }, + { + "epoch": 12.585365853658537, + "grad_norm": 1.7903852462768555, + "learning_rate": 0.0001, + "loss": 78.7459, + "step": 780 + }, + { + "epoch": 12.747967479674797, + "grad_norm": 1.4962575435638428, + "learning_rate": 0.0001, + "loss": 78.9193, + "step": 790 + }, + { + "epoch": 12.910569105691057, + "grad_norm": 42.10435104370117, + "learning_rate": 0.0001, + "loss": 80.1429, + "step": 800 + }, + { + "epoch": 13.065040650406504, + "grad_norm": 1.904065728187561, + "learning_rate": 0.0001, + "loss": 73.7798, + "step": 810 + }, + { + "epoch": 13.227642276422765, + "grad_norm": 1.8768481016159058, + "learning_rate": 0.0001, + "loss": 76.8692, + "step": 820 + }, + { + "epoch": 13.390243902439025, + "grad_norm": 16.140565872192383, + "learning_rate": 0.0001, + "loss": 78.122, + "step": 830 + }, + { + "epoch": 13.552845528455284, + "grad_norm": 2.160288095474243, + "learning_rate": 0.0001, + "loss": 76.2169, + "step": 840 + }, + { + "epoch": 13.715447154471544, + "grad_norm": 27.682231903076172, + "learning_rate": 0.0001, + "loss": 76.6266, + "step": 850 + }, + { + "epoch": 13.878048780487806, + "grad_norm": 1.779772162437439, + "learning_rate": 0.0001, + "loss": 80.0434, + "step": 860 + }, + { + "epoch": 14.032520325203253, + "grad_norm": 1.6516485214233398, + "learning_rate": 0.0001, + "loss": 73.3943, + "step": 870 + }, + { + "epoch": 14.195121951219512, + "grad_norm": 1.846498966217041, + "learning_rate": 0.0001, + "loss": 77.5157, + "step": 880 + }, + { + "epoch": 14.357723577235772, + "grad_norm": 1.942946434020996, + "learning_rate": 0.0001, + "loss": 77.6944, + "step": 890 + }, + { + "epoch": 14.520325203252032, + "grad_norm": 2.0929224491119385, + "learning_rate": 0.0001, + "loss": 76.9411, + "step": 900 + }, + { + "epoch": 14.682926829268293, + "grad_norm": 3.4498767852783203, + "learning_rate": 0.0001, + "loss": 74.9234, + "step": 910 + }, + { + "epoch": 14.845528455284553, + "grad_norm": 1.6880728006362915, + "learning_rate": 0.0001, + "loss": 75.3617, + "step": 920 + }, + { + "epoch": 15.0, + "grad_norm": 1.8403468132019043, + "learning_rate": 0.0001, + "loss": 73.2543, + "step": 930 + }, + { + "epoch": 15.16260162601626, + "grad_norm": 1.4368852376937866, + "learning_rate": 0.0001, + "loss": 76.6271, + "step": 940 + }, + { + "epoch": 15.32520325203252, + "grad_norm": 2.9467780590057373, + "learning_rate": 0.0001, + "loss": 75.8744, + "step": 950 + }, + { + "epoch": 15.487804878048781, + "grad_norm": 1.567134141921997, + "learning_rate": 0.0001, + "loss": 76.6629, + "step": 960 + }, + { + "epoch": 15.65040650406504, + "grad_norm": 1.4503871202468872, + "learning_rate": 0.0001, + "loss": 76.3525, + "step": 970 + }, + { + "epoch": 15.8130081300813, + "grad_norm": 1.8871147632598877, + "learning_rate": 0.0001, + "loss": 77.4684, + "step": 980 + }, + { + "epoch": 15.975609756097562, + "grad_norm": 1.6707026958465576, + "learning_rate": 0.0001, + "loss": 76.3063, + "step": 990 + }, + { + "epoch": 16.130081300813007, + "grad_norm": 3.6300716400146484, + "learning_rate": 0.0001, + "loss": 73.0586, + "step": 1000 + }, + { + "epoch": 16.29268292682927, + "grad_norm": 1.6669234037399292, + "learning_rate": 0.0001, + "loss": 75.8027, + "step": 1010 + }, + { + "epoch": 16.45528455284553, + "grad_norm": 1.6920552253723145, + "learning_rate": 0.0001, + "loss": 74.7252, + "step": 1020 + }, + { + "epoch": 16.617886178861788, + "grad_norm": 1.8184431791305542, + "learning_rate": 0.0001, + "loss": 75.3625, + "step": 1030 + }, + { + "epoch": 16.78048780487805, + "grad_norm": 1.5828496217727661, + "learning_rate": 0.0001, + "loss": 78.0302, + "step": 1040 + }, + { + "epoch": 16.943089430894307, + "grad_norm": 1.7372915744781494, + "learning_rate": 0.0001, + "loss": 78.6949, + "step": 1050 + }, + { + "epoch": 17.097560975609756, + "grad_norm": 1.837161898612976, + "learning_rate": 0.0001, + "loss": 74.4726, + "step": 1060 + }, + { + "epoch": 17.260162601626018, + "grad_norm": 3.8733808994293213, + "learning_rate": 0.0001, + "loss": 76.1711, + "step": 1070 + }, + { + "epoch": 17.422764227642276, + "grad_norm": 1.5907851457595825, + "learning_rate": 0.0001, + "loss": 75.8292, + "step": 1080 + }, + { + "epoch": 17.585365853658537, + "grad_norm": 1.8271747827529907, + "learning_rate": 0.0001, + "loss": 74.5316, + "step": 1090 + }, + { + "epoch": 17.747967479674795, + "grad_norm": 1.827980875968933, + "learning_rate": 0.0001, + "loss": 74.7281, + "step": 1100 + }, + { + "epoch": 17.910569105691057, + "grad_norm": 1.4184074401855469, + "learning_rate": 0.0001, + "loss": 75.736, + "step": 1110 + }, + { + "epoch": 18.065040650406505, + "grad_norm": 2.3459699153900146, + "learning_rate": 0.0001, + "loss": 71.9586, + "step": 1120 + }, + { + "epoch": 18.227642276422763, + "grad_norm": 47.673641204833984, + "learning_rate": 0.0001, + "loss": 75.6649, + "step": 1130 + }, + { + "epoch": 18.390243902439025, + "grad_norm": 1.677398443222046, + "learning_rate": 0.0001, + "loss": 76.0574, + "step": 1140 + }, + { + "epoch": 18.552845528455286, + "grad_norm": 1.4180850982666016, + "learning_rate": 0.0001, + "loss": 74.3077, + "step": 1150 + }, + { + "epoch": 18.715447154471544, + "grad_norm": 2.0041465759277344, + "learning_rate": 0.0001, + "loss": 73.5916, + "step": 1160 + }, + { + "epoch": 18.878048780487806, + "grad_norm": 1.9098238945007324, + "learning_rate": 0.0001, + "loss": 74.5002, + "step": 1170 + }, + { + "epoch": 19.03252032520325, + "grad_norm": 1.405555248260498, + "learning_rate": 0.0001, + "loss": 69.8115, + "step": 1180 + }, + { + "epoch": 19.195121951219512, + "grad_norm": 1.424198865890503, + "learning_rate": 0.0001, + "loss": 73.7289, + "step": 1190 + }, + { + "epoch": 19.357723577235774, + "grad_norm": 7.542550563812256, + "learning_rate": 0.0001, + "loss": 73.7696, + "step": 1200 + }, + { + "epoch": 19.520325203252032, + "grad_norm": 1.425329566001892, + "learning_rate": 0.0001, + "loss": 72.1261, + "step": 1210 + }, + { + "epoch": 19.682926829268293, + "grad_norm": 1.525724172592163, + "learning_rate": 0.0001, + "loss": 73.6578, + "step": 1220 + }, + { + "epoch": 19.84552845528455, + "grad_norm": 1.6387193202972412, + "learning_rate": 0.0001, + "loss": 74.3027, + "step": 1230 + }, + { + "epoch": 20.0, + "grad_norm": 1.8429840803146362, + "learning_rate": 0.0001, + "loss": 69.5056, + "step": 1240 + }, + { + "epoch": 20.16260162601626, + "grad_norm": 2.5166006088256836, + "learning_rate": 0.0001, + "loss": 73.5965, + "step": 1250 + }, + { + "epoch": 20.32520325203252, + "grad_norm": 1.6006672382354736, + "learning_rate": 0.0001, + "loss": 72.4581, + "step": 1260 + }, + { + "epoch": 20.48780487804878, + "grad_norm": 1.6722878217697144, + "learning_rate": 0.0001, + "loss": 72.3097, + "step": 1270 + }, + { + "epoch": 20.65040650406504, + "grad_norm": 1.8715728521347046, + "learning_rate": 0.0001, + "loss": 73.4752, + "step": 1280 + }, + { + "epoch": 20.8130081300813, + "grad_norm": 1.3280900716781616, + "learning_rate": 0.0001, + "loss": 73.2145, + "step": 1290 + }, + { + "epoch": 20.975609756097562, + "grad_norm": 1.941451907157898, + "learning_rate": 0.0001, + "loss": 73.2085, + "step": 1300 + }, + { + "epoch": 21.130081300813007, + "grad_norm": 4.277497291564941, + "learning_rate": 0.0001, + "loss": 68.8305, + "step": 1310 + }, + { + "epoch": 21.29268292682927, + "grad_norm": 1.9949846267700195, + "learning_rate": 0.0001, + "loss": 72.6689, + "step": 1320 + }, + { + "epoch": 21.45528455284553, + "grad_norm": 1.524044394493103, + "learning_rate": 0.0001, + "loss": 72.9705, + "step": 1330 + }, + { + "epoch": 21.617886178861788, + "grad_norm": 1.384194016456604, + "learning_rate": 0.0001, + "loss": 72.5465, + "step": 1340 + }, + { + "epoch": 21.78048780487805, + "grad_norm": 1.724225640296936, + "learning_rate": 0.0001, + "loss": 72.9203, + "step": 1350 + }, + { + "epoch": 21.943089430894307, + "grad_norm": 2.0856704711914062, + "learning_rate": 0.0001, + "loss": 72.4034, + "step": 1360 + }, + { + "epoch": 22.097560975609756, + "grad_norm": 5.128044128417969, + "learning_rate": 0.0001, + "loss": 68.7754, + "step": 1370 + }, + { + "epoch": 22.260162601626018, + "grad_norm": 1.7385634183883667, + "learning_rate": 0.0001, + "loss": 73.9349, + "step": 1380 + }, + { + "epoch": 22.422764227642276, + "grad_norm": 1.3946012258529663, + "learning_rate": 0.0001, + "loss": 70.9627, + "step": 1390 + }, + { + "epoch": 22.585365853658537, + "grad_norm": 1.285414457321167, + "learning_rate": 0.0001, + "loss": 72.4578, + "step": 1400 + }, + { + "epoch": 22.747967479674795, + "grad_norm": 1.4197710752487183, + "learning_rate": 0.0001, + "loss": 72.5252, + "step": 1410 + }, + { + "epoch": 22.910569105691057, + "grad_norm": 3.048942804336548, + "learning_rate": 0.0001, + "loss": 71.1107, + "step": 1420 + }, + { + "epoch": 23.065040650406505, + "grad_norm": 1.3114765882492065, + "learning_rate": 0.0001, + "loss": 68.2725, + "step": 1430 + }, + { + "epoch": 23.227642276422763, + "grad_norm": 1.2763880491256714, + "learning_rate": 0.0001, + "loss": 71.5783, + "step": 1440 + }, + { + "epoch": 23.390243902439025, + "grad_norm": 1.3913089036941528, + "learning_rate": 0.0001, + "loss": 72.5986, + "step": 1450 + }, + { + "epoch": 23.552845528455286, + "grad_norm": 1.628751516342163, + "learning_rate": 0.0001, + "loss": 71.8383, + "step": 1460 + }, + { + "epoch": 23.715447154471544, + "grad_norm": 1.4331274032592773, + "learning_rate": 0.0001, + "loss": 70.8051, + "step": 1470 + }, + { + "epoch": 23.878048780487806, + "grad_norm": 1.4154727458953857, + "learning_rate": 0.0001, + "loss": 71.3283, + "step": 1480 + }, + { + "epoch": 24.03252032520325, + "grad_norm": 1.7208784818649292, + "learning_rate": 0.0001, + "loss": 68.4719, + "step": 1490 + }, + { + "epoch": 24.195121951219512, + "grad_norm": 1.466123342514038, + "learning_rate": 0.0001, + "loss": 71.9597, + "step": 1500 + }, + { + "epoch": 24.357723577235774, + "grad_norm": 1.4823886156082153, + "learning_rate": 0.0001, + "loss": 72.4533, + "step": 1510 + }, + { + "epoch": 24.520325203252032, + "grad_norm": 1.8407355546951294, + "learning_rate": 0.0001, + "loss": 71.1468, + "step": 1520 + }, + { + "epoch": 24.682926829268293, + "grad_norm": 1.630565881729126, + "learning_rate": 0.0001, + "loss": 71.5192, + "step": 1530 + }, + { + "epoch": 24.84552845528455, + "grad_norm": 1.5248785018920898, + "learning_rate": 0.0001, + "loss": 68.4804, + "step": 1540 + }, + { + "epoch": 25.0, + "grad_norm": 1.4690219163894653, + "learning_rate": 0.0001, + "loss": 67.5716, + "step": 1550 + }, + { + "epoch": 25.16260162601626, + "grad_norm": 2.0246646404266357, + "learning_rate": 0.0001, + "loss": 71.2554, + "step": 1560 + }, + { + "epoch": 25.32520325203252, + "grad_norm": 1.3611458539962769, + "learning_rate": 0.0001, + "loss": 69.8558, + "step": 1570 + }, + { + "epoch": 25.48780487804878, + "grad_norm": 2.0006039142608643, + "learning_rate": 0.0001, + "loss": 72.5344, + "step": 1580 + }, + { + "epoch": 25.65040650406504, + "grad_norm": 1.7373167276382446, + "learning_rate": 0.0001, + "loss": 70.6253, + "step": 1590 + }, + { + "epoch": 25.8130081300813, + "grad_norm": 1.2878378629684448, + "learning_rate": 0.0001, + "loss": 72.165, + "step": 1600 + }, + { + "epoch": 25.975609756097562, + "grad_norm": 1.5337013006210327, + "learning_rate": 0.0001, + "loss": 70.6989, + "step": 1610 + }, + { + "epoch": 26.130081300813007, + "grad_norm": 1.364615559577942, + "learning_rate": 0.0001, + "loss": 67.3898, + "step": 1620 + }, + { + "epoch": 26.29268292682927, + "grad_norm": 1.3249316215515137, + "learning_rate": 0.0001, + "loss": 72.7102, + "step": 1630 + }, + { + "epoch": 26.45528455284553, + "grad_norm": 4.209898471832275, + "learning_rate": 0.0001, + "loss": 70.3185, + "step": 1640 + }, + { + "epoch": 26.617886178861788, + "grad_norm": 1.5488810539245605, + "learning_rate": 0.0001, + "loss": 72.3678, + "step": 1650 + }, + { + "epoch": 26.78048780487805, + "grad_norm": 1.7743597030639648, + "learning_rate": 0.0001, + "loss": 71.0917, + "step": 1660 + }, + { + "epoch": 26.943089430894307, + "grad_norm": 1.5433533191680908, + "learning_rate": 0.0001, + "loss": 71.4001, + "step": 1670 + }, + { + "epoch": 27.097560975609756, + "grad_norm": 1.6320385932922363, + "learning_rate": 0.0001, + "loss": 66.7441, + "step": 1680 + }, + { + "epoch": 27.260162601626018, + "grad_norm": 1.3137969970703125, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 1690 + }, + { + "epoch": 27.422764227642276, + "grad_norm": 1.9150484800338745, + "learning_rate": 0.0001, + "loss": 69.8956, + "step": 1700 + }, + { + "epoch": 27.585365853658537, + "grad_norm": 1.4442802667617798, + "learning_rate": 0.0001, + "loss": 70.7185, + "step": 1710 + }, + { + "epoch": 27.747967479674795, + "grad_norm": 2.572497606277466, + "learning_rate": 0.0001, + "loss": 70.8934, + "step": 1720 + }, + { + "epoch": 27.910569105691057, + "grad_norm": 1.6669617891311646, + "learning_rate": 0.0001, + "loss": 71.8221, + "step": 1730 + }, + { + "epoch": 28.065040650406505, + "grad_norm": 1.8449876308441162, + "learning_rate": 0.0001, + "loss": 66.3923, + "step": 1740 + }, + { + "epoch": 28.227642276422763, + "grad_norm": 1.6600004434585571, + "learning_rate": 0.0001, + "loss": 69.5301, + "step": 1750 + }, + { + "epoch": 28.390243902439025, + "grad_norm": 1.4655731916427612, + "learning_rate": 0.0001, + "loss": 72.1924, + "step": 1760 + }, + { + "epoch": 28.552845528455286, + "grad_norm": 1.134505033493042, + "learning_rate": 0.0001, + "loss": 69.5855, + "step": 1770 + }, + { + "epoch": 28.715447154471544, + "grad_norm": 1.301984190940857, + "learning_rate": 0.0001, + "loss": 70.7999, + "step": 1780 + }, + { + "epoch": 28.878048780487806, + "grad_norm": 1.5641013383865356, + "learning_rate": 0.0001, + "loss": 69.9091, + "step": 1790 + }, + { + "epoch": 29.03252032520325, + "grad_norm": 1.4850797653198242, + "learning_rate": 0.0001, + "loss": 66.6266, + "step": 1800 + }, + { + "epoch": 29.195121951219512, + "grad_norm": 1.2069118022918701, + "learning_rate": 0.0001, + "loss": 69.3802, + "step": 1810 + }, + { + "epoch": 29.357723577235774, + "grad_norm": 1.2890352010726929, + "learning_rate": 0.0001, + "loss": 68.5044, + "step": 1820 + }, + { + "epoch": 29.520325203252032, + "grad_norm": 1.4150917530059814, + "learning_rate": 0.0001, + "loss": 69.807, + "step": 1830 + }, + { + "epoch": 29.682926829268293, + "grad_norm": 1.514807939529419, + "learning_rate": 0.0001, + "loss": 71.3858, + "step": 1840 + }, + { + "epoch": 29.84552845528455, + "grad_norm": 1.4246330261230469, + "learning_rate": 0.0001, + "loss": 70.494, + "step": 1850 + }, + { + "epoch": 30.0, + "grad_norm": 1.1789597272872925, + "learning_rate": 0.0001, + "loss": 67.1056, + "step": 1860 + }, + { + "epoch": 30.16260162601626, + "grad_norm": 1.23678719997406, + "learning_rate": 0.0001, + "loss": 70.336, + "step": 1870 + }, + { + "epoch": 30.32520325203252, + "grad_norm": 1.1756274700164795, + "learning_rate": 0.0001, + "loss": 67.9909, + "step": 1880 + }, + { + "epoch": 30.48780487804878, + "grad_norm": 1.4175690412521362, + "learning_rate": 0.0001, + "loss": 69.2617, + "step": 1890 + }, + { + "epoch": 30.65040650406504, + "grad_norm": 1.3084465265274048, + "learning_rate": 0.0001, + "loss": 70.2963, + "step": 1900 + }, + { + "epoch": 30.8130081300813, + "grad_norm": 1.486464500427246, + "learning_rate": 0.0001, + "loss": 68.6317, + "step": 1910 + }, + { + "epoch": 30.975609756097562, + "grad_norm": 10.415762901306152, + "learning_rate": 0.0001, + "loss": 69.7767, + "step": 1920 + }, + { + "epoch": 31.130081300813007, + "grad_norm": 1.4484038352966309, + "learning_rate": 0.0001, + "loss": 65.6345, + "step": 1930 + }, + { + "epoch": 31.29268292682927, + "grad_norm": 2.297306537628174, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 1940 + }, + { + "epoch": 31.45528455284553, + "grad_norm": 1.223490595817566, + "learning_rate": 0.0001, + "loss": 69.2139, + "step": 1950 + }, + { + "epoch": 31.617886178861788, + "grad_norm": 1.2144125699996948, + "learning_rate": 0.0001, + "loss": 70.3061, + "step": 1960 + }, + { + "epoch": 31.78048780487805, + "grad_norm": 2.633528470993042, + "learning_rate": 0.0001, + "loss": 68.7096, + "step": 1970 + }, + { + "epoch": 31.943089430894307, + "grad_norm": 1.5048350095748901, + "learning_rate": 0.0001, + "loss": 68.6711, + "step": 1980 + }, + { + "epoch": 32.09756097560975, + "grad_norm": 1.5154467821121216, + "learning_rate": 0.0001, + "loss": 66.7989, + "step": 1990 + }, + { + "epoch": 32.260162601626014, + "grad_norm": 2.721277952194214, + "learning_rate": 0.0001, + "loss": 70.4987, + "step": 2000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 323, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.4294103453190324e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c7f229ebe1034caaf3e92006c8c7528c21af93d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517586849d044483051c7a59cec7c6ce389951878a7357ef837edb1cbc9983c8 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-2000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-20000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-20000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/latest new file mode 100644 index 0000000000000000000000000000000000000000..641c40ce8d5e05e04a0840d69bcc5fac045a3fbd --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/latest @@ -0,0 +1 @@ +global_step4000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2c7787461599f983288988fc316ceb9098d82715 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e51d05e25b39d554231d9e05e3cbf4ebd6b899d1a7817eff42ddff3a43838db8 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..a802e58b1b54493ada961f1dd3f6580292172007 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c2f1b69d28ad54d6fdc9a99baaa9f29dbd6837c1a3b1ad194a5ff6226c35a5f +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..5be182ddd793a4c568418d1053faa820cd074a09 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9305a8f6b871280dbddba64732b278e88b7a55f5a79c809d61884f7cb2a727d +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..3940dc566e8e53b7663be7ce037a49f2e59603a6 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f44e80f60128043840447269f419c84758b2c96eecfe3f560c9faa923000cef +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..58eed9ce36a9f1085c0a356a8e20986794f0a5bd --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5078041e86e9175783a59be1ffee21ef355ec2fc7c902a978d93218b476148f +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..2ce0f3b7bef77d12d487373a02a95fd2ee854833 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d65ef8064fb1c0cb4fd8f471e901906fcdd9990bc547b3645c5d0c2c10f99038 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..6f055b537174215381fe17514f3a44b901d06b91 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ee42624ac5db4123f5246182dcdec2ef012e91f620fea637a45a230118f3130 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..ff776e4b13b4445e2705de1f43c40c9179d9192d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6748199e3d8d6fb95b24ad273beeb30a5d572364d1de80a516ecec47fa958685 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..a7c4c3e1380e3430111a7f9286151019dc663d71 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce0bfb860c65a5a15deb840e63bad7d1558ad587b7baf883a20c27680156c1f8 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7c33095320ec6a7f585e68e4e02ed022aa80652 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e88d5675c3e7af2f5bef04cb9622f8a630e6e25147b9f39eb466dd8ba603a69a +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ab5bbb3ad45b97c49585e8ec4994873120c428ff --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/trainer_state.json @@ -0,0 +1,2834 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 64.52032520325203, + "eval_steps": 2000, + "global_step": 4000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.16260162601626016, + "grad_norm": 23.72986602783203, + "learning_rate": 0.0001, + "loss": 271.6598, + "step": 10 + }, + { + "epoch": 0.3252032520325203, + "grad_norm": 18.99613380432129, + "learning_rate": 0.0001, + "loss": 233.029, + "step": 20 + }, + { + "epoch": 0.4878048780487805, + "grad_norm": 3.9815304279327393, + "learning_rate": 0.0001, + "loss": 231.0776, + "step": 30 + }, + { + "epoch": 0.6504065040650406, + "grad_norm": 4.641753196716309, + "learning_rate": 0.0001, + "loss": 228.6894, + "step": 40 + }, + { + "epoch": 0.8130081300813008, + "grad_norm": 3.000852108001709, + "learning_rate": 0.0001, + "loss": 228.4694, + "step": 50 + }, + { + "epoch": 0.975609756097561, + "grad_norm": 12.213018417358398, + "learning_rate": 0.0001, + "loss": 226.4258, + "step": 60 + }, + { + "epoch": 1.1300813008130082, + "grad_norm": 2.535187005996704, + "learning_rate": 0.0001, + "loss": 209.9772, + "step": 70 + }, + { + "epoch": 1.2926829268292683, + "grad_norm": 2.0984582901000977, + "learning_rate": 0.0001, + "loss": 217.2421, + "step": 80 + }, + { + "epoch": 1.4552845528455285, + "grad_norm": 3.4802305698394775, + "learning_rate": 0.0001, + "loss": 210.9241, + "step": 90 + }, + { + "epoch": 1.6178861788617886, + "grad_norm": 3.813626527786255, + "learning_rate": 0.0001, + "loss": 205.7566, + "step": 100 + }, + { + "epoch": 1.7804878048780488, + "grad_norm": 2.6378512382507324, + "learning_rate": 0.0001, + "loss": 201.8789, + "step": 110 + }, + { + "epoch": 1.943089430894309, + "grad_norm": 2.7502875328063965, + "learning_rate": 0.0001, + "loss": 196.373, + "step": 120 + }, + { + "epoch": 2.097560975609756, + "grad_norm": 2.659212112426758, + "learning_rate": 0.0001, + "loss": 183.1456, + "step": 130 + }, + { + "epoch": 2.2601626016260163, + "grad_norm": 2.513044595718384, + "learning_rate": 0.0001, + "loss": 190.1661, + "step": 140 + }, + { + "epoch": 2.4227642276422765, + "grad_norm": 2.3192145824432373, + "learning_rate": 0.0001, + "loss": 184.9997, + "step": 150 + }, + { + "epoch": 2.5853658536585367, + "grad_norm": 3.056708574295044, + "learning_rate": 0.0001, + "loss": 182.0164, + "step": 160 + }, + { + "epoch": 2.747967479674797, + "grad_norm": 2.9687108993530273, + "learning_rate": 0.0001, + "loss": 179.6539, + "step": 170 + }, + { + "epoch": 2.910569105691057, + "grad_norm": 2.1498093605041504, + "learning_rate": 0.0001, + "loss": 176.7159, + "step": 180 + }, + { + "epoch": 3.065040650406504, + "grad_norm": 3.0103771686553955, + "learning_rate": 0.0001, + "loss": 166.1703, + "step": 190 + }, + { + "epoch": 3.227642276422764, + "grad_norm": 2.992462635040283, + "learning_rate": 0.0001, + "loss": 171.124, + "step": 200 + }, + { + "epoch": 3.3902439024390243, + "grad_norm": 4.8617119789123535, + "learning_rate": 0.0001, + "loss": 168.2646, + "step": 210 + }, + { + "epoch": 3.5528455284552845, + "grad_norm": 3.1047444343566895, + "learning_rate": 0.0001, + "loss": 163.0889, + "step": 220 + }, + { + "epoch": 3.7154471544715446, + "grad_norm": 6.473227500915527, + "learning_rate": 0.0001, + "loss": 151.3667, + "step": 230 + }, + { + "epoch": 3.8780487804878048, + "grad_norm": 6.16747522354126, + "learning_rate": 0.0001, + "loss": 137.5915, + "step": 240 + }, + { + "epoch": 4.032520325203252, + "grad_norm": 7.994515419006348, + "learning_rate": 0.0001, + "loss": 118.8875, + "step": 250 + }, + { + "epoch": 4.195121951219512, + "grad_norm": 5.882101058959961, + "learning_rate": 0.0001, + "loss": 119.4793, + "step": 260 + }, + { + "epoch": 4.357723577235772, + "grad_norm": 12.32968521118164, + "learning_rate": 0.0001, + "loss": 113.1209, + "step": 270 + }, + { + "epoch": 4.520325203252033, + "grad_norm": 5.697225570678711, + "learning_rate": 0.0001, + "loss": 107.51, + "step": 280 + }, + { + "epoch": 4.682926829268292, + "grad_norm": 4.654167175292969, + "learning_rate": 0.0001, + "loss": 107.0172, + "step": 290 + }, + { + "epoch": 4.845528455284553, + "grad_norm": 2.9595484733581543, + "learning_rate": 0.0001, + "loss": 100.9164, + "step": 300 + }, + { + "epoch": 5.0, + "grad_norm": 21.242591857910156, + "learning_rate": 0.0001, + "loss": 94.9884, + "step": 310 + }, + { + "epoch": 5.16260162601626, + "grad_norm": 5.760314464569092, + "learning_rate": 0.0001, + "loss": 97.5901, + "step": 320 + }, + { + "epoch": 5.32520325203252, + "grad_norm": 3.1373329162597656, + "learning_rate": 0.0001, + "loss": 94.8485, + "step": 330 + }, + { + "epoch": 5.487804878048781, + "grad_norm": 2.3777568340301514, + "learning_rate": 0.0001, + "loss": 96.9982, + "step": 340 + }, + { + "epoch": 5.650406504065041, + "grad_norm": 3.0607948303222656, + "learning_rate": 0.0001, + "loss": 96.1586, + "step": 350 + }, + { + "epoch": 5.8130081300813, + "grad_norm": 4.168269634246826, + "learning_rate": 0.0001, + "loss": 94.5743, + "step": 360 + }, + { + "epoch": 5.975609756097561, + "grad_norm": 2.4383699893951416, + "learning_rate": 0.0001, + "loss": 95.5197, + "step": 370 + }, + { + "epoch": 6.130081300813008, + "grad_norm": 2.302259683609009, + "learning_rate": 0.0001, + "loss": 85.6599, + "step": 380 + }, + { + "epoch": 6.2926829268292686, + "grad_norm": 2.552753210067749, + "learning_rate": 0.0001, + "loss": 91.2999, + "step": 390 + }, + { + "epoch": 6.455284552845528, + "grad_norm": 2.2092080116271973, + "learning_rate": 0.0001, + "loss": 90.7041, + "step": 400 + }, + { + "epoch": 6.617886178861789, + "grad_norm": 2.2935690879821777, + "learning_rate": 0.0001, + "loss": 89.9949, + "step": 410 + }, + { + "epoch": 6.780487804878049, + "grad_norm": 2.487510919570923, + "learning_rate": 0.0001, + "loss": 86.9056, + "step": 420 + }, + { + "epoch": 6.943089430894309, + "grad_norm": 2.8630545139312744, + "learning_rate": 0.0001, + "loss": 88.3347, + "step": 430 + }, + { + "epoch": 7.097560975609756, + "grad_norm": 5.87056303024292, + "learning_rate": 0.0001, + "loss": 84.9632, + "step": 440 + }, + { + "epoch": 7.260162601626016, + "grad_norm": 2.587661027908325, + "learning_rate": 0.0001, + "loss": 86.2024, + "step": 450 + }, + { + "epoch": 7.4227642276422765, + "grad_norm": 3.5671255588531494, + "learning_rate": 0.0001, + "loss": 86.218, + "step": 460 + }, + { + "epoch": 7.585365853658536, + "grad_norm": 2.5036680698394775, + "learning_rate": 0.0001, + "loss": 86.5914, + "step": 470 + }, + { + "epoch": 7.747967479674797, + "grad_norm": 3.2656471729278564, + "learning_rate": 0.0001, + "loss": 82.6081, + "step": 480 + }, + { + "epoch": 7.9105691056910565, + "grad_norm": 2.5053915977478027, + "learning_rate": 0.0001, + "loss": 84.6116, + "step": 490 + }, + { + "epoch": 8.065040650406504, + "grad_norm": 2.1669957637786865, + "learning_rate": 0.0001, + "loss": 80.4354, + "step": 500 + }, + { + "epoch": 8.227642276422765, + "grad_norm": 4.2003703117370605, + "learning_rate": 0.0001, + "loss": 85.2286, + "step": 510 + }, + { + "epoch": 8.390243902439025, + "grad_norm": 3.1909608840942383, + "learning_rate": 0.0001, + "loss": 85.0358, + "step": 520 + }, + { + "epoch": 8.552845528455284, + "grad_norm": 1.8420121669769287, + "learning_rate": 0.0001, + "loss": 85.4647, + "step": 530 + }, + { + "epoch": 8.715447154471544, + "grad_norm": 1.9822801351547241, + "learning_rate": 0.0001, + "loss": 84.7134, + "step": 540 + }, + { + "epoch": 8.878048780487806, + "grad_norm": 2.936758518218994, + "learning_rate": 0.0001, + "loss": 83.8068, + "step": 550 + }, + { + "epoch": 9.032520325203253, + "grad_norm": 1.5713881254196167, + "learning_rate": 0.0001, + "loss": 80.0695, + "step": 560 + }, + { + "epoch": 9.195121951219512, + "grad_norm": 2.0584616661071777, + "learning_rate": 0.0001, + "loss": 84.5158, + "step": 570 + }, + { + "epoch": 9.357723577235772, + "grad_norm": 2.101391553878784, + "learning_rate": 0.0001, + "loss": 82.4719, + "step": 580 + }, + { + "epoch": 9.520325203252032, + "grad_norm": 1.9956543445587158, + "learning_rate": 0.0001, + "loss": 81.6715, + "step": 590 + }, + { + "epoch": 9.682926829268293, + "grad_norm": 2.7622063159942627, + "learning_rate": 0.0001, + "loss": 85.4178, + "step": 600 + }, + { + "epoch": 9.845528455284553, + "grad_norm": 2.2084789276123047, + "learning_rate": 0.0001, + "loss": 84.5702, + "step": 610 + }, + { + "epoch": 10.0, + "grad_norm": 1.9635663032531738, + "learning_rate": 0.0001, + "loss": 77.3838, + "step": 620 + }, + { + "epoch": 10.16260162601626, + "grad_norm": 2.5895581245422363, + "learning_rate": 0.0001, + "loss": 81.723, + "step": 630 + }, + { + "epoch": 10.32520325203252, + "grad_norm": 2.743727445602417, + "learning_rate": 0.0001, + "loss": 83.1332, + "step": 640 + }, + { + "epoch": 10.487804878048781, + "grad_norm": 2.024585008621216, + "learning_rate": 0.0001, + "loss": 81.5968, + "step": 650 + }, + { + "epoch": 10.65040650406504, + "grad_norm": 1.8254153728485107, + "learning_rate": 0.0001, + "loss": 78.9589, + "step": 660 + }, + { + "epoch": 10.8130081300813, + "grad_norm": 3.2015740871429443, + "learning_rate": 0.0001, + "loss": 79.1829, + "step": 670 + }, + { + "epoch": 10.975609756097562, + "grad_norm": 1.788453459739685, + "learning_rate": 0.0001, + "loss": 81.527, + "step": 680 + }, + { + "epoch": 11.130081300813009, + "grad_norm": 1.5744423866271973, + "learning_rate": 0.0001, + "loss": 74.9869, + "step": 690 + }, + { + "epoch": 11.292682926829269, + "grad_norm": 2.006002187728882, + "learning_rate": 0.0001, + "loss": 80.2677, + "step": 700 + }, + { + "epoch": 11.455284552845528, + "grad_norm": 1.8847148418426514, + "learning_rate": 0.0001, + "loss": 78.8709, + "step": 710 + }, + { + "epoch": 11.617886178861788, + "grad_norm": 1.8384478092193604, + "learning_rate": 0.0001, + "loss": 79.008, + "step": 720 + }, + { + "epoch": 11.78048780487805, + "grad_norm": 1.9486687183380127, + "learning_rate": 0.0001, + "loss": 76.1525, + "step": 730 + }, + { + "epoch": 11.94308943089431, + "grad_norm": 1.8644076585769653, + "learning_rate": 0.0001, + "loss": 77.5802, + "step": 740 + }, + { + "epoch": 12.097560975609756, + "grad_norm": 1.6035499572753906, + "learning_rate": 0.0001, + "loss": 75.2422, + "step": 750 + }, + { + "epoch": 12.260162601626016, + "grad_norm": 1.6935622692108154, + "learning_rate": 0.0001, + "loss": 77.5838, + "step": 760 + }, + { + "epoch": 12.422764227642276, + "grad_norm": 1.9327987432479858, + "learning_rate": 0.0001, + "loss": 78.3237, + "step": 770 + }, + { + "epoch": 12.585365853658537, + "grad_norm": 1.7903852462768555, + "learning_rate": 0.0001, + "loss": 78.7459, + "step": 780 + }, + { + "epoch": 12.747967479674797, + "grad_norm": 1.4962575435638428, + "learning_rate": 0.0001, + "loss": 78.9193, + "step": 790 + }, + { + "epoch": 12.910569105691057, + "grad_norm": 42.10435104370117, + "learning_rate": 0.0001, + "loss": 80.1429, + "step": 800 + }, + { + "epoch": 13.065040650406504, + "grad_norm": 1.904065728187561, + "learning_rate": 0.0001, + "loss": 73.7798, + "step": 810 + }, + { + "epoch": 13.227642276422765, + "grad_norm": 1.8768481016159058, + "learning_rate": 0.0001, + "loss": 76.8692, + "step": 820 + }, + { + "epoch": 13.390243902439025, + "grad_norm": 16.140565872192383, + "learning_rate": 0.0001, + "loss": 78.122, + "step": 830 + }, + { + "epoch": 13.552845528455284, + "grad_norm": 2.160288095474243, + "learning_rate": 0.0001, + "loss": 76.2169, + "step": 840 + }, + { + "epoch": 13.715447154471544, + "grad_norm": 27.682231903076172, + "learning_rate": 0.0001, + "loss": 76.6266, + "step": 850 + }, + { + "epoch": 13.878048780487806, + "grad_norm": 1.779772162437439, + "learning_rate": 0.0001, + "loss": 80.0434, + "step": 860 + }, + { + "epoch": 14.032520325203253, + "grad_norm": 1.6516485214233398, + "learning_rate": 0.0001, + "loss": 73.3943, + "step": 870 + }, + { + "epoch": 14.195121951219512, + "grad_norm": 1.846498966217041, + "learning_rate": 0.0001, + "loss": 77.5157, + "step": 880 + }, + { + "epoch": 14.357723577235772, + "grad_norm": 1.942946434020996, + "learning_rate": 0.0001, + "loss": 77.6944, + "step": 890 + }, + { + "epoch": 14.520325203252032, + "grad_norm": 2.0929224491119385, + "learning_rate": 0.0001, + "loss": 76.9411, + "step": 900 + }, + { + "epoch": 14.682926829268293, + "grad_norm": 3.4498767852783203, + "learning_rate": 0.0001, + "loss": 74.9234, + "step": 910 + }, + { + "epoch": 14.845528455284553, + "grad_norm": 1.6880728006362915, + "learning_rate": 0.0001, + "loss": 75.3617, + "step": 920 + }, + { + "epoch": 15.0, + "grad_norm": 1.8403468132019043, + "learning_rate": 0.0001, + "loss": 73.2543, + "step": 930 + }, + { + "epoch": 15.16260162601626, + "grad_norm": 1.4368852376937866, + "learning_rate": 0.0001, + "loss": 76.6271, + "step": 940 + }, + { + "epoch": 15.32520325203252, + "grad_norm": 2.9467780590057373, + "learning_rate": 0.0001, + "loss": 75.8744, + "step": 950 + }, + { + "epoch": 15.487804878048781, + "grad_norm": 1.567134141921997, + "learning_rate": 0.0001, + "loss": 76.6629, + "step": 960 + }, + { + "epoch": 15.65040650406504, + "grad_norm": 1.4503871202468872, + "learning_rate": 0.0001, + "loss": 76.3525, + "step": 970 + }, + { + "epoch": 15.8130081300813, + "grad_norm": 1.8871147632598877, + "learning_rate": 0.0001, + "loss": 77.4684, + "step": 980 + }, + { + "epoch": 15.975609756097562, + "grad_norm": 1.6707026958465576, + "learning_rate": 0.0001, + "loss": 76.3063, + "step": 990 + }, + { + "epoch": 16.130081300813007, + "grad_norm": 3.6300716400146484, + "learning_rate": 0.0001, + "loss": 73.0586, + "step": 1000 + }, + { + "epoch": 16.29268292682927, + "grad_norm": 1.6669234037399292, + "learning_rate": 0.0001, + "loss": 75.8027, + "step": 1010 + }, + { + "epoch": 16.45528455284553, + "grad_norm": 1.6920552253723145, + "learning_rate": 0.0001, + "loss": 74.7252, + "step": 1020 + }, + { + "epoch": 16.617886178861788, + "grad_norm": 1.8184431791305542, + "learning_rate": 0.0001, + "loss": 75.3625, + "step": 1030 + }, + { + "epoch": 16.78048780487805, + "grad_norm": 1.5828496217727661, + "learning_rate": 0.0001, + "loss": 78.0302, + "step": 1040 + }, + { + "epoch": 16.943089430894307, + "grad_norm": 1.7372915744781494, + "learning_rate": 0.0001, + "loss": 78.6949, + "step": 1050 + }, + { + "epoch": 17.097560975609756, + "grad_norm": 1.837161898612976, + "learning_rate": 0.0001, + "loss": 74.4726, + "step": 1060 + }, + { + "epoch": 17.260162601626018, + "grad_norm": 3.8733808994293213, + "learning_rate": 0.0001, + "loss": 76.1711, + "step": 1070 + }, + { + "epoch": 17.422764227642276, + "grad_norm": 1.5907851457595825, + "learning_rate": 0.0001, + "loss": 75.8292, + "step": 1080 + }, + { + "epoch": 17.585365853658537, + "grad_norm": 1.8271747827529907, + "learning_rate": 0.0001, + "loss": 74.5316, + "step": 1090 + }, + { + "epoch": 17.747967479674795, + "grad_norm": 1.827980875968933, + "learning_rate": 0.0001, + "loss": 74.7281, + "step": 1100 + }, + { + "epoch": 17.910569105691057, + "grad_norm": 1.4184074401855469, + "learning_rate": 0.0001, + "loss": 75.736, + "step": 1110 + }, + { + "epoch": 18.065040650406505, + "grad_norm": 2.3459699153900146, + "learning_rate": 0.0001, + "loss": 71.9586, + "step": 1120 + }, + { + "epoch": 18.227642276422763, + "grad_norm": 47.673641204833984, + "learning_rate": 0.0001, + "loss": 75.6649, + "step": 1130 + }, + { + "epoch": 18.390243902439025, + "grad_norm": 1.677398443222046, + "learning_rate": 0.0001, + "loss": 76.0574, + "step": 1140 + }, + { + "epoch": 18.552845528455286, + "grad_norm": 1.4180850982666016, + "learning_rate": 0.0001, + "loss": 74.3077, + "step": 1150 + }, + { + "epoch": 18.715447154471544, + "grad_norm": 2.0041465759277344, + "learning_rate": 0.0001, + "loss": 73.5916, + "step": 1160 + }, + { + "epoch": 18.878048780487806, + "grad_norm": 1.9098238945007324, + "learning_rate": 0.0001, + "loss": 74.5002, + "step": 1170 + }, + { + "epoch": 19.03252032520325, + "grad_norm": 1.405555248260498, + "learning_rate": 0.0001, + "loss": 69.8115, + "step": 1180 + }, + { + "epoch": 19.195121951219512, + "grad_norm": 1.424198865890503, + "learning_rate": 0.0001, + "loss": 73.7289, + "step": 1190 + }, + { + "epoch": 19.357723577235774, + "grad_norm": 7.542550563812256, + "learning_rate": 0.0001, + "loss": 73.7696, + "step": 1200 + }, + { + "epoch": 19.520325203252032, + "grad_norm": 1.425329566001892, + "learning_rate": 0.0001, + "loss": 72.1261, + "step": 1210 + }, + { + "epoch": 19.682926829268293, + "grad_norm": 1.525724172592163, + "learning_rate": 0.0001, + "loss": 73.6578, + "step": 1220 + }, + { + "epoch": 19.84552845528455, + "grad_norm": 1.6387193202972412, + "learning_rate": 0.0001, + "loss": 74.3027, + "step": 1230 + }, + { + "epoch": 20.0, + "grad_norm": 1.8429840803146362, + "learning_rate": 0.0001, + "loss": 69.5056, + "step": 1240 + }, + { + "epoch": 20.16260162601626, + "grad_norm": 2.5166006088256836, + "learning_rate": 0.0001, + "loss": 73.5965, + "step": 1250 + }, + { + "epoch": 20.32520325203252, + "grad_norm": 1.6006672382354736, + "learning_rate": 0.0001, + "loss": 72.4581, + "step": 1260 + }, + { + "epoch": 20.48780487804878, + "grad_norm": 1.6722878217697144, + "learning_rate": 0.0001, + "loss": 72.3097, + "step": 1270 + }, + { + "epoch": 20.65040650406504, + "grad_norm": 1.8715728521347046, + "learning_rate": 0.0001, + "loss": 73.4752, + "step": 1280 + }, + { + "epoch": 20.8130081300813, + "grad_norm": 1.3280900716781616, + "learning_rate": 0.0001, + "loss": 73.2145, + "step": 1290 + }, + { + "epoch": 20.975609756097562, + "grad_norm": 1.941451907157898, + "learning_rate": 0.0001, + "loss": 73.2085, + "step": 1300 + }, + { + "epoch": 21.130081300813007, + "grad_norm": 4.277497291564941, + "learning_rate": 0.0001, + "loss": 68.8305, + "step": 1310 + }, + { + "epoch": 21.29268292682927, + "grad_norm": 1.9949846267700195, + "learning_rate": 0.0001, + "loss": 72.6689, + "step": 1320 + }, + { + "epoch": 21.45528455284553, + "grad_norm": 1.524044394493103, + "learning_rate": 0.0001, + "loss": 72.9705, + "step": 1330 + }, + { + "epoch": 21.617886178861788, + "grad_norm": 1.384194016456604, + "learning_rate": 0.0001, + "loss": 72.5465, + "step": 1340 + }, + { + "epoch": 21.78048780487805, + "grad_norm": 1.724225640296936, + "learning_rate": 0.0001, + "loss": 72.9203, + "step": 1350 + }, + { + "epoch": 21.943089430894307, + "grad_norm": 2.0856704711914062, + "learning_rate": 0.0001, + "loss": 72.4034, + "step": 1360 + }, + { + "epoch": 22.097560975609756, + "grad_norm": 5.128044128417969, + "learning_rate": 0.0001, + "loss": 68.7754, + "step": 1370 + }, + { + "epoch": 22.260162601626018, + "grad_norm": 1.7385634183883667, + "learning_rate": 0.0001, + "loss": 73.9349, + "step": 1380 + }, + { + "epoch": 22.422764227642276, + "grad_norm": 1.3946012258529663, + "learning_rate": 0.0001, + "loss": 70.9627, + "step": 1390 + }, + { + "epoch": 22.585365853658537, + "grad_norm": 1.285414457321167, + "learning_rate": 0.0001, + "loss": 72.4578, + "step": 1400 + }, + { + "epoch": 22.747967479674795, + "grad_norm": 1.4197710752487183, + "learning_rate": 0.0001, + "loss": 72.5252, + "step": 1410 + }, + { + "epoch": 22.910569105691057, + "grad_norm": 3.048942804336548, + "learning_rate": 0.0001, + "loss": 71.1107, + "step": 1420 + }, + { + "epoch": 23.065040650406505, + "grad_norm": 1.3114765882492065, + "learning_rate": 0.0001, + "loss": 68.2725, + "step": 1430 + }, + { + "epoch": 23.227642276422763, + "grad_norm": 1.2763880491256714, + "learning_rate": 0.0001, + "loss": 71.5783, + "step": 1440 + }, + { + "epoch": 23.390243902439025, + "grad_norm": 1.3913089036941528, + "learning_rate": 0.0001, + "loss": 72.5986, + "step": 1450 + }, + { + "epoch": 23.552845528455286, + "grad_norm": 1.628751516342163, + "learning_rate": 0.0001, + "loss": 71.8383, + "step": 1460 + }, + { + "epoch": 23.715447154471544, + "grad_norm": 1.4331274032592773, + "learning_rate": 0.0001, + "loss": 70.8051, + "step": 1470 + }, + { + "epoch": 23.878048780487806, + "grad_norm": 1.4154727458953857, + "learning_rate": 0.0001, + "loss": 71.3283, + "step": 1480 + }, + { + "epoch": 24.03252032520325, + "grad_norm": 1.7208784818649292, + "learning_rate": 0.0001, + "loss": 68.4719, + "step": 1490 + }, + { + "epoch": 24.195121951219512, + "grad_norm": 1.466123342514038, + "learning_rate": 0.0001, + "loss": 71.9597, + "step": 1500 + }, + { + "epoch": 24.357723577235774, + "grad_norm": 1.4823886156082153, + "learning_rate": 0.0001, + "loss": 72.4533, + "step": 1510 + }, + { + "epoch": 24.520325203252032, + "grad_norm": 1.8407355546951294, + "learning_rate": 0.0001, + "loss": 71.1468, + "step": 1520 + }, + { + "epoch": 24.682926829268293, + "grad_norm": 1.630565881729126, + "learning_rate": 0.0001, + "loss": 71.5192, + "step": 1530 + }, + { + "epoch": 24.84552845528455, + "grad_norm": 1.5248785018920898, + "learning_rate": 0.0001, + "loss": 68.4804, + "step": 1540 + }, + { + "epoch": 25.0, + "grad_norm": 1.4690219163894653, + "learning_rate": 0.0001, + "loss": 67.5716, + "step": 1550 + }, + { + "epoch": 25.16260162601626, + "grad_norm": 2.0246646404266357, + "learning_rate": 0.0001, + "loss": 71.2554, + "step": 1560 + }, + { + "epoch": 25.32520325203252, + "grad_norm": 1.3611458539962769, + "learning_rate": 0.0001, + "loss": 69.8558, + "step": 1570 + }, + { + "epoch": 25.48780487804878, + "grad_norm": 2.0006039142608643, + "learning_rate": 0.0001, + "loss": 72.5344, + "step": 1580 + }, + { + "epoch": 25.65040650406504, + "grad_norm": 1.7373167276382446, + "learning_rate": 0.0001, + "loss": 70.6253, + "step": 1590 + }, + { + "epoch": 25.8130081300813, + "grad_norm": 1.2878378629684448, + "learning_rate": 0.0001, + "loss": 72.165, + "step": 1600 + }, + { + "epoch": 25.975609756097562, + "grad_norm": 1.5337013006210327, + "learning_rate": 0.0001, + "loss": 70.6989, + "step": 1610 + }, + { + "epoch": 26.130081300813007, + "grad_norm": 1.364615559577942, + "learning_rate": 0.0001, + "loss": 67.3898, + "step": 1620 + }, + { + "epoch": 26.29268292682927, + "grad_norm": 1.3249316215515137, + "learning_rate": 0.0001, + "loss": 72.7102, + "step": 1630 + }, + { + "epoch": 26.45528455284553, + "grad_norm": 4.209898471832275, + "learning_rate": 0.0001, + "loss": 70.3185, + "step": 1640 + }, + { + "epoch": 26.617886178861788, + "grad_norm": 1.5488810539245605, + "learning_rate": 0.0001, + "loss": 72.3678, + "step": 1650 + }, + { + "epoch": 26.78048780487805, + "grad_norm": 1.7743597030639648, + "learning_rate": 0.0001, + "loss": 71.0917, + "step": 1660 + }, + { + "epoch": 26.943089430894307, + "grad_norm": 1.5433533191680908, + "learning_rate": 0.0001, + "loss": 71.4001, + "step": 1670 + }, + { + "epoch": 27.097560975609756, + "grad_norm": 1.6320385932922363, + "learning_rate": 0.0001, + "loss": 66.7441, + "step": 1680 + }, + { + "epoch": 27.260162601626018, + "grad_norm": 1.3137969970703125, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 1690 + }, + { + "epoch": 27.422764227642276, + "grad_norm": 1.9150484800338745, + "learning_rate": 0.0001, + "loss": 69.8956, + "step": 1700 + }, + { + "epoch": 27.585365853658537, + "grad_norm": 1.4442802667617798, + "learning_rate": 0.0001, + "loss": 70.7185, + "step": 1710 + }, + { + "epoch": 27.747967479674795, + "grad_norm": 2.572497606277466, + "learning_rate": 0.0001, + "loss": 70.8934, + "step": 1720 + }, + { + "epoch": 27.910569105691057, + "grad_norm": 1.6669617891311646, + "learning_rate": 0.0001, + "loss": 71.8221, + "step": 1730 + }, + { + "epoch": 28.065040650406505, + "grad_norm": 1.8449876308441162, + "learning_rate": 0.0001, + "loss": 66.3923, + "step": 1740 + }, + { + "epoch": 28.227642276422763, + "grad_norm": 1.6600004434585571, + "learning_rate": 0.0001, + "loss": 69.5301, + "step": 1750 + }, + { + "epoch": 28.390243902439025, + "grad_norm": 1.4655731916427612, + "learning_rate": 0.0001, + "loss": 72.1924, + "step": 1760 + }, + { + "epoch": 28.552845528455286, + "grad_norm": 1.134505033493042, + "learning_rate": 0.0001, + "loss": 69.5855, + "step": 1770 + }, + { + "epoch": 28.715447154471544, + "grad_norm": 1.301984190940857, + "learning_rate": 0.0001, + "loss": 70.7999, + "step": 1780 + }, + { + "epoch": 28.878048780487806, + "grad_norm": 1.5641013383865356, + "learning_rate": 0.0001, + "loss": 69.9091, + "step": 1790 + }, + { + "epoch": 29.03252032520325, + "grad_norm": 1.4850797653198242, + "learning_rate": 0.0001, + "loss": 66.6266, + "step": 1800 + }, + { + "epoch": 29.195121951219512, + "grad_norm": 1.2069118022918701, + "learning_rate": 0.0001, + "loss": 69.3802, + "step": 1810 + }, + { + "epoch": 29.357723577235774, + "grad_norm": 1.2890352010726929, + "learning_rate": 0.0001, + "loss": 68.5044, + "step": 1820 + }, + { + "epoch": 29.520325203252032, + "grad_norm": 1.4150917530059814, + "learning_rate": 0.0001, + "loss": 69.807, + "step": 1830 + }, + { + "epoch": 29.682926829268293, + "grad_norm": 1.514807939529419, + "learning_rate": 0.0001, + "loss": 71.3858, + "step": 1840 + }, + { + "epoch": 29.84552845528455, + "grad_norm": 1.4246330261230469, + "learning_rate": 0.0001, + "loss": 70.494, + "step": 1850 + }, + { + "epoch": 30.0, + "grad_norm": 1.1789597272872925, + "learning_rate": 0.0001, + "loss": 67.1056, + "step": 1860 + }, + { + "epoch": 30.16260162601626, + "grad_norm": 1.23678719997406, + "learning_rate": 0.0001, + "loss": 70.336, + "step": 1870 + }, + { + "epoch": 30.32520325203252, + "grad_norm": 1.1756274700164795, + "learning_rate": 0.0001, + "loss": 67.9909, + "step": 1880 + }, + { + "epoch": 30.48780487804878, + "grad_norm": 1.4175690412521362, + "learning_rate": 0.0001, + "loss": 69.2617, + "step": 1890 + }, + { + "epoch": 30.65040650406504, + "grad_norm": 1.3084465265274048, + "learning_rate": 0.0001, + "loss": 70.2963, + "step": 1900 + }, + { + "epoch": 30.8130081300813, + "grad_norm": 1.486464500427246, + "learning_rate": 0.0001, + "loss": 68.6317, + "step": 1910 + }, + { + "epoch": 30.975609756097562, + "grad_norm": 10.415762901306152, + "learning_rate": 0.0001, + "loss": 69.7767, + "step": 1920 + }, + { + "epoch": 31.130081300813007, + "grad_norm": 1.4484038352966309, + "learning_rate": 0.0001, + "loss": 65.6345, + "step": 1930 + }, + { + "epoch": 31.29268292682927, + "grad_norm": 2.297306537628174, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 1940 + }, + { + "epoch": 31.45528455284553, + "grad_norm": 1.223490595817566, + "learning_rate": 0.0001, + "loss": 69.2139, + "step": 1950 + }, + { + "epoch": 31.617886178861788, + "grad_norm": 1.2144125699996948, + "learning_rate": 0.0001, + "loss": 70.3061, + "step": 1960 + }, + { + "epoch": 31.78048780487805, + "grad_norm": 2.633528470993042, + "learning_rate": 0.0001, + "loss": 68.7096, + "step": 1970 + }, + { + "epoch": 31.943089430894307, + "grad_norm": 1.5048350095748901, + "learning_rate": 0.0001, + "loss": 68.6711, + "step": 1980 + }, + { + "epoch": 32.09756097560975, + "grad_norm": 1.5154467821121216, + "learning_rate": 0.0001, + "loss": 66.7989, + "step": 1990 + }, + { + "epoch": 32.260162601626014, + "grad_norm": 2.721277952194214, + "learning_rate": 0.0001, + "loss": 70.4987, + "step": 2000 + }, + { + "epoch": 32.422764227642276, + "grad_norm": 1.44464910030365, + "learning_rate": 0.0001, + "loss": 69.9712, + "step": 2010 + }, + { + "epoch": 32.58536585365854, + "grad_norm": 1.3462835550308228, + "learning_rate": 0.0001, + "loss": 68.9381, + "step": 2020 + }, + { + "epoch": 32.7479674796748, + "grad_norm": 1.3283029794692993, + "learning_rate": 0.0001, + "loss": 68.5301, + "step": 2030 + }, + { + "epoch": 32.91056910569106, + "grad_norm": 1.5172960758209229, + "learning_rate": 0.0001, + "loss": 70.1894, + "step": 2040 + }, + { + "epoch": 33.0650406504065, + "grad_norm": 1.1872576475143433, + "learning_rate": 0.0001, + "loss": 65.0158, + "step": 2050 + }, + { + "epoch": 33.22764227642276, + "grad_norm": 1.5414419174194336, + "learning_rate": 0.0001, + "loss": 69.1537, + "step": 2060 + }, + { + "epoch": 33.390243902439025, + "grad_norm": 1.3143430948257446, + "learning_rate": 0.0001, + "loss": 68.6792, + "step": 2070 + }, + { + "epoch": 33.552845528455286, + "grad_norm": 1.3863723278045654, + "learning_rate": 0.0001, + "loss": 70.3867, + "step": 2080 + }, + { + "epoch": 33.71544715447155, + "grad_norm": 1.4060479402542114, + "learning_rate": 0.0001, + "loss": 68.4981, + "step": 2090 + }, + { + "epoch": 33.8780487804878, + "grad_norm": 1.34132981300354, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 2100 + }, + { + "epoch": 34.03252032520325, + "grad_norm": 1.8303673267364502, + "learning_rate": 0.0001, + "loss": 65.819, + "step": 2110 + }, + { + "epoch": 34.19512195121951, + "grad_norm": 1.387339472770691, + "learning_rate": 0.0001, + "loss": 69.9825, + "step": 2120 + }, + { + "epoch": 34.357723577235774, + "grad_norm": 1.4125195741653442, + "learning_rate": 0.0001, + "loss": 67.5149, + "step": 2130 + }, + { + "epoch": 34.520325203252035, + "grad_norm": 1.2909412384033203, + "learning_rate": 0.0001, + "loss": 69.7848, + "step": 2140 + }, + { + "epoch": 34.68292682926829, + "grad_norm": 1.8860795497894287, + "learning_rate": 0.0001, + "loss": 68.6145, + "step": 2150 + }, + { + "epoch": 34.84552845528455, + "grad_norm": 1.5111079216003418, + "learning_rate": 0.0001, + "loss": 68.2612, + "step": 2160 + }, + { + "epoch": 35.0, + "grad_norm": 1.3275264501571655, + "learning_rate": 0.0001, + "loss": 64.0298, + "step": 2170 + }, + { + "epoch": 35.16260162601626, + "grad_norm": 1.174864411354065, + "learning_rate": 0.0001, + "loss": 66.6153, + "step": 2180 + }, + { + "epoch": 35.32520325203252, + "grad_norm": 1.6873915195465088, + "learning_rate": 0.0001, + "loss": 69.6278, + "step": 2190 + }, + { + "epoch": 35.48780487804878, + "grad_norm": 2.4457507133483887, + "learning_rate": 0.0001, + "loss": 68.3611, + "step": 2200 + }, + { + "epoch": 35.65040650406504, + "grad_norm": 1.4779958724975586, + "learning_rate": 0.0001, + "loss": 69.2609, + "step": 2210 + }, + { + "epoch": 35.8130081300813, + "grad_norm": 2.273709774017334, + "learning_rate": 0.0001, + "loss": 69.1312, + "step": 2220 + }, + { + "epoch": 35.97560975609756, + "grad_norm": 1.5741862058639526, + "learning_rate": 0.0001, + "loss": 69.1191, + "step": 2230 + }, + { + "epoch": 36.13008130081301, + "grad_norm": 1.2730036973953247, + "learning_rate": 0.0001, + "loss": 65.692, + "step": 2240 + }, + { + "epoch": 36.292682926829265, + "grad_norm": 1.2936429977416992, + "learning_rate": 0.0001, + "loss": 68.2962, + "step": 2250 + }, + { + "epoch": 36.45528455284553, + "grad_norm": 1.354279637336731, + "learning_rate": 0.0001, + "loss": 70.5507, + "step": 2260 + }, + { + "epoch": 36.61788617886179, + "grad_norm": 1.4165853261947632, + "learning_rate": 0.0001, + "loss": 68.0048, + "step": 2270 + }, + { + "epoch": 36.78048780487805, + "grad_norm": 1.2992445230484009, + "learning_rate": 0.0001, + "loss": 67.7072, + "step": 2280 + }, + { + "epoch": 36.94308943089431, + "grad_norm": 1.5053461790084839, + "learning_rate": 0.0001, + "loss": 66.7587, + "step": 2290 + }, + { + "epoch": 37.09756097560975, + "grad_norm": 1.337390661239624, + "learning_rate": 0.0001, + "loss": 63.786, + "step": 2300 + }, + { + "epoch": 37.260162601626014, + "grad_norm": 1.0407774448394775, + "learning_rate": 0.0001, + "loss": 66.6253, + "step": 2310 + }, + { + "epoch": 37.422764227642276, + "grad_norm": 1.564152479171753, + "learning_rate": 0.0001, + "loss": 68.3606, + "step": 2320 + }, + { + "epoch": 37.58536585365854, + "grad_norm": 1.2028454542160034, + "learning_rate": 0.0001, + "loss": 66.2841, + "step": 2330 + }, + { + "epoch": 37.7479674796748, + "grad_norm": 1.4528603553771973, + "learning_rate": 0.0001, + "loss": 67.4277, + "step": 2340 + }, + { + "epoch": 37.91056910569106, + "grad_norm": 1.4954335689544678, + "learning_rate": 0.0001, + "loss": 68.9954, + "step": 2350 + }, + { + "epoch": 38.0650406504065, + "grad_norm": 1.2820416688919067, + "learning_rate": 0.0001, + "loss": 65.2449, + "step": 2360 + }, + { + "epoch": 38.22764227642276, + "grad_norm": 1.2392579317092896, + "learning_rate": 0.0001, + "loss": 67.5006, + "step": 2370 + }, + { + "epoch": 38.390243902439025, + "grad_norm": 1.2135238647460938, + "learning_rate": 0.0001, + "loss": 69.2214, + "step": 2380 + }, + { + "epoch": 38.552845528455286, + "grad_norm": 1.5964744091033936, + "learning_rate": 0.0001, + "loss": 67.934, + "step": 2390 + }, + { + "epoch": 38.71544715447155, + "grad_norm": 1.2841702699661255, + "learning_rate": 0.0001, + "loss": 67.0848, + "step": 2400 + }, + { + "epoch": 38.8780487804878, + "grad_norm": 1.8026081323623657, + "learning_rate": 0.0001, + "loss": 67.5358, + "step": 2410 + }, + { + "epoch": 39.03252032520325, + "grad_norm": 1.4178807735443115, + "learning_rate": 0.0001, + "loss": 64.771, + "step": 2420 + }, + { + "epoch": 39.19512195121951, + "grad_norm": 1.3671854734420776, + "learning_rate": 0.0001, + "loss": 68.0531, + "step": 2430 + }, + { + "epoch": 39.357723577235774, + "grad_norm": 1.43596613407135, + "learning_rate": 0.0001, + "loss": 67.7735, + "step": 2440 + }, + { + "epoch": 39.520325203252035, + "grad_norm": 1.464250922203064, + "learning_rate": 0.0001, + "loss": 67.8331, + "step": 2450 + }, + { + "epoch": 39.68292682926829, + "grad_norm": 3.101461410522461, + "learning_rate": 0.0001, + "loss": 65.8956, + "step": 2460 + }, + { + "epoch": 39.84552845528455, + "grad_norm": 1.8332289457321167, + "learning_rate": 0.0001, + "loss": 66.8218, + "step": 2470 + }, + { + "epoch": 40.0, + "grad_norm": 1.2660984992980957, + "learning_rate": 0.0001, + "loss": 63.9969, + "step": 2480 + }, + { + "epoch": 40.16260162601626, + "grad_norm": 2.0309898853302, + "learning_rate": 0.0001, + "loss": 67.3538, + "step": 2490 + }, + { + "epoch": 40.32520325203252, + "grad_norm": 1.207000494003296, + "learning_rate": 0.0001, + "loss": 68.0898, + "step": 2500 + }, + { + "epoch": 40.48780487804878, + "grad_norm": 1.3609018325805664, + "learning_rate": 0.0001, + "loss": 67.0381, + "step": 2510 + }, + { + "epoch": 40.65040650406504, + "grad_norm": 1.310826301574707, + "learning_rate": 0.0001, + "loss": 69.0267, + "step": 2520 + }, + { + "epoch": 40.8130081300813, + "grad_norm": 1.4150960445404053, + "learning_rate": 0.0001, + "loss": 69.1934, + "step": 2530 + }, + { + "epoch": 40.97560975609756, + "grad_norm": 1.5414319038391113, + "learning_rate": 0.0001, + "loss": 68.0604, + "step": 2540 + }, + { + "epoch": 41.13008130081301, + "grad_norm": 2.3246140480041504, + "learning_rate": 0.0001, + "loss": 61.5698, + "step": 2550 + }, + { + "epoch": 41.292682926829265, + "grad_norm": 1.206296443939209, + "learning_rate": 0.0001, + "loss": 64.9898, + "step": 2560 + }, + { + "epoch": 41.45528455284553, + "grad_norm": 1.6080626249313354, + "learning_rate": 0.0001, + "loss": 68.2685, + "step": 2570 + }, + { + "epoch": 41.61788617886179, + "grad_norm": 1.2404112815856934, + "learning_rate": 0.0001, + "loss": 67.8457, + "step": 2580 + }, + { + "epoch": 41.78048780487805, + "grad_norm": 1.2304514646530151, + "learning_rate": 0.0001, + "loss": 67.9052, + "step": 2590 + }, + { + "epoch": 41.94308943089431, + "grad_norm": 1.5806187391281128, + "learning_rate": 0.0001, + "loss": 66.0663, + "step": 2600 + }, + { + "epoch": 42.09756097560975, + "grad_norm": 1.2564151287078857, + "learning_rate": 0.0001, + "loss": 65.7763, + "step": 2610 + }, + { + "epoch": 42.260162601626014, + "grad_norm": 1.4972114562988281, + "learning_rate": 0.0001, + "loss": 66.4269, + "step": 2620 + }, + { + "epoch": 42.422764227642276, + "grad_norm": 1.2913843393325806, + "learning_rate": 0.0001, + "loss": 67.7554, + "step": 2630 + }, + { + "epoch": 42.58536585365854, + "grad_norm": 1.3743863105773926, + "learning_rate": 0.0001, + "loss": 67.2627, + "step": 2640 + }, + { + "epoch": 42.7479674796748, + "grad_norm": 1.1267883777618408, + "learning_rate": 0.0001, + "loss": 67.4597, + "step": 2650 + }, + { + "epoch": 42.91056910569106, + "grad_norm": 2.0194661617279053, + "learning_rate": 0.0001, + "loss": 66.9295, + "step": 2660 + }, + { + "epoch": 43.0650406504065, + "grad_norm": 1.273779273033142, + "learning_rate": 0.0001, + "loss": 60.7906, + "step": 2670 + }, + { + "epoch": 43.22764227642276, + "grad_norm": 1.1498609781265259, + "learning_rate": 0.0001, + "loss": 68.0405, + "step": 2680 + }, + { + "epoch": 43.390243902439025, + "grad_norm": 1.4689130783081055, + "learning_rate": 0.0001, + "loss": 66.2358, + "step": 2690 + }, + { + "epoch": 43.552845528455286, + "grad_norm": 1.4495564699172974, + "learning_rate": 0.0001, + "loss": 67.4267, + "step": 2700 + }, + { + "epoch": 43.71544715447155, + "grad_norm": 2.5769612789154053, + "learning_rate": 0.0001, + "loss": 66.953, + "step": 2710 + }, + { + "epoch": 43.8780487804878, + "grad_norm": 2.1684205532073975, + "learning_rate": 0.0001, + "loss": 65.6393, + "step": 2720 + }, + { + "epoch": 44.03252032520325, + "grad_norm": 1.4968472719192505, + "learning_rate": 0.0001, + "loss": 60.8256, + "step": 2730 + }, + { + "epoch": 44.19512195121951, + "grad_norm": 1.1874353885650635, + "learning_rate": 0.0001, + "loss": 66.818, + "step": 2740 + }, + { + "epoch": 44.357723577235774, + "grad_norm": 2.271235227584839, + "learning_rate": 0.0001, + "loss": 66.5048, + "step": 2750 + }, + { + "epoch": 44.520325203252035, + "grad_norm": 1.20489501953125, + "learning_rate": 0.0001, + "loss": 67.0487, + "step": 2760 + }, + { + "epoch": 44.68292682926829, + "grad_norm": 1.6273691654205322, + "learning_rate": 0.0001, + "loss": 66.2228, + "step": 2770 + }, + { + "epoch": 44.84552845528455, + "grad_norm": 1.4935433864593506, + "learning_rate": 0.0001, + "loss": 67.2039, + "step": 2780 + }, + { + "epoch": 45.0, + "grad_norm": 1.2668659687042236, + "learning_rate": 0.0001, + "loss": 62.7855, + "step": 2790 + }, + { + "epoch": 45.16260162601626, + "grad_norm": 1.6439573764801025, + "learning_rate": 0.0001, + "loss": 67.919, + "step": 2800 + }, + { + "epoch": 45.32520325203252, + "grad_norm": 1.311854600906372, + "learning_rate": 0.0001, + "loss": 65.1797, + "step": 2810 + }, + { + "epoch": 45.48780487804878, + "grad_norm": 1.3596776723861694, + "learning_rate": 0.0001, + "loss": 67.469, + "step": 2820 + }, + { + "epoch": 45.65040650406504, + "grad_norm": 1.5618443489074707, + "learning_rate": 0.0001, + "loss": 66.1808, + "step": 2830 + }, + { + "epoch": 45.8130081300813, + "grad_norm": 1.7551205158233643, + "learning_rate": 0.0001, + "loss": 65.3803, + "step": 2840 + }, + { + "epoch": 45.97560975609756, + "grad_norm": 1.433794379234314, + "learning_rate": 0.0001, + "loss": 67.8314, + "step": 2850 + }, + { + "epoch": 46.13008130081301, + "grad_norm": 1.4634851217269897, + "learning_rate": 0.0001, + "loss": 63.4219, + "step": 2860 + }, + { + "epoch": 46.292682926829265, + "grad_norm": 1.083052396774292, + "learning_rate": 0.0001, + "loss": 68.0719, + "step": 2870 + }, + { + "epoch": 46.45528455284553, + "grad_norm": 1.1784703731536865, + "learning_rate": 0.0001, + "loss": 66.1391, + "step": 2880 + }, + { + "epoch": 46.61788617886179, + "grad_norm": 1.2552465200424194, + "learning_rate": 0.0001, + "loss": 65.6006, + "step": 2890 + }, + { + "epoch": 46.78048780487805, + "grad_norm": 1.4684522151947021, + "learning_rate": 0.0001, + "loss": 66.8314, + "step": 2900 + }, + { + "epoch": 46.94308943089431, + "grad_norm": 1.741514801979065, + "learning_rate": 0.0001, + "loss": 67.5056, + "step": 2910 + }, + { + "epoch": 47.09756097560975, + "grad_norm": 5.696534156799316, + "learning_rate": 0.0001, + "loss": 62.8016, + "step": 2920 + }, + { + "epoch": 47.260162601626014, + "grad_norm": 1.5000969171524048, + "learning_rate": 0.0001, + "loss": 68.0272, + "step": 2930 + }, + { + "epoch": 47.422764227642276, + "grad_norm": 1.1044933795928955, + "learning_rate": 0.0001, + "loss": 67.0905, + "step": 2940 + }, + { + "epoch": 47.58536585365854, + "grad_norm": 0.9933589100837708, + "learning_rate": 0.0001, + "loss": 67.6077, + "step": 2950 + }, + { + "epoch": 47.7479674796748, + "grad_norm": 1.2908599376678467, + "learning_rate": 0.0001, + "loss": 65.869, + "step": 2960 + }, + { + "epoch": 47.91056910569106, + "grad_norm": 1.0470529794692993, + "learning_rate": 0.0001, + "loss": 66.5353, + "step": 2970 + }, + { + "epoch": 48.0650406504065, + "grad_norm": 1.0927976369857788, + "learning_rate": 0.0001, + "loss": 62.7541, + "step": 2980 + }, + { + "epoch": 48.22764227642276, + "grad_norm": 1.0870513916015625, + "learning_rate": 0.0001, + "loss": 64.172, + "step": 2990 + }, + { + "epoch": 48.390243902439025, + "grad_norm": 1.3492090702056885, + "learning_rate": 0.0001, + "loss": 67.6642, + "step": 3000 + }, + { + "epoch": 48.552845528455286, + "grad_norm": 1.4024657011032104, + "learning_rate": 0.0001, + "loss": 65.2722, + "step": 3010 + }, + { + "epoch": 48.71544715447155, + "grad_norm": 1.1165252923965454, + "learning_rate": 0.0001, + "loss": 66.7233, + "step": 3020 + }, + { + "epoch": 48.8780487804878, + "grad_norm": 1.2147928476333618, + "learning_rate": 0.0001, + "loss": 65.7277, + "step": 3030 + }, + { + "epoch": 49.03252032520325, + "grad_norm": 0.925310492515564, + "learning_rate": 0.0001, + "loss": 62.9256, + "step": 3040 + }, + { + "epoch": 49.19512195121951, + "grad_norm": 1.2353442907333374, + "learning_rate": 0.0001, + "loss": 66.1218, + "step": 3050 + }, + { + "epoch": 49.357723577235774, + "grad_norm": 1.2177367210388184, + "learning_rate": 0.0001, + "loss": 66.7978, + "step": 3060 + }, + { + "epoch": 49.520325203252035, + "grad_norm": 1.248044729232788, + "learning_rate": 0.0001, + "loss": 66.2132, + "step": 3070 + }, + { + "epoch": 49.68292682926829, + "grad_norm": 1.366050124168396, + "learning_rate": 0.0001, + "loss": 67.0118, + "step": 3080 + }, + { + "epoch": 49.84552845528455, + "grad_norm": 1.3701151609420776, + "learning_rate": 0.0001, + "loss": 67.6099, + "step": 3090 + }, + { + "epoch": 50.0, + "grad_norm": 1.0420141220092773, + "learning_rate": 0.0001, + "loss": 61.5289, + "step": 3100 + }, + { + "epoch": 50.16260162601626, + "grad_norm": 1.0957746505737305, + "learning_rate": 0.0001, + "loss": 66.7348, + "step": 3110 + }, + { + "epoch": 50.32520325203252, + "grad_norm": 1.108628749847412, + "learning_rate": 0.0001, + "loss": 66.6199, + "step": 3120 + }, + { + "epoch": 50.48780487804878, + "grad_norm": 1.3680156469345093, + "learning_rate": 0.0001, + "loss": 67.0825, + "step": 3130 + }, + { + "epoch": 50.65040650406504, + "grad_norm": 1.169851303100586, + "learning_rate": 0.0001, + "loss": 65.4901, + "step": 3140 + }, + { + "epoch": 50.8130081300813, + "grad_norm": 2.1822028160095215, + "learning_rate": 0.0001, + "loss": 65.6841, + "step": 3150 + }, + { + "epoch": 50.97560975609756, + "grad_norm": 1.053402304649353, + "learning_rate": 0.0001, + "loss": 66.3119, + "step": 3160 + }, + { + "epoch": 51.13008130081301, + "grad_norm": 1.0627789497375488, + "learning_rate": 0.0001, + "loss": 61.3642, + "step": 3170 + }, + { + "epoch": 51.292682926829265, + "grad_norm": 1.926291823387146, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 3180 + }, + { + "epoch": 51.45528455284553, + "grad_norm": 1.5791983604431152, + "learning_rate": 0.0001, + "loss": 63.913, + "step": 3190 + }, + { + "epoch": 51.61788617886179, + "grad_norm": 1.210265040397644, + "learning_rate": 0.0001, + "loss": 64.5261, + "step": 3200 + }, + { + "epoch": 51.78048780487805, + "grad_norm": 1.0720808506011963, + "learning_rate": 0.0001, + "loss": 67.0194, + "step": 3210 + }, + { + "epoch": 51.94308943089431, + "grad_norm": 1.94563889503479, + "learning_rate": 0.0001, + "loss": 68.4662, + "step": 3220 + }, + { + "epoch": 52.09756097560975, + "grad_norm": 1.598980188369751, + "learning_rate": 0.0001, + "loss": 62.4946, + "step": 3230 + }, + { + "epoch": 52.260162601626014, + "grad_norm": 1.0564576387405396, + "learning_rate": 0.0001, + "loss": 65.7912, + "step": 3240 + }, + { + "epoch": 52.422764227642276, + "grad_norm": 1.2189645767211914, + "learning_rate": 0.0001, + "loss": 65.5772, + "step": 3250 + }, + { + "epoch": 52.58536585365854, + "grad_norm": 1.1357049942016602, + "learning_rate": 0.0001, + "loss": 65.1559, + "step": 3260 + }, + { + "epoch": 52.7479674796748, + "grad_norm": 1.316867709159851, + "learning_rate": 0.0001, + "loss": 64.7897, + "step": 3270 + }, + { + "epoch": 52.91056910569106, + "grad_norm": 1.3870197534561157, + "learning_rate": 0.0001, + "loss": 65.5195, + "step": 3280 + }, + { + "epoch": 53.0650406504065, + "grad_norm": 1.2467927932739258, + "learning_rate": 0.0001, + "loss": 61.3392, + "step": 3290 + }, + { + "epoch": 53.22764227642276, + "grad_norm": 1.1661109924316406, + "learning_rate": 0.0001, + "loss": 66.0117, + "step": 3300 + }, + { + "epoch": 53.390243902439025, + "grad_norm": 1.3509628772735596, + "learning_rate": 0.0001, + "loss": 64.7228, + "step": 3310 + }, + { + "epoch": 53.552845528455286, + "grad_norm": 1.1093642711639404, + "learning_rate": 0.0001, + "loss": 65.2534, + "step": 3320 + }, + { + "epoch": 53.71544715447155, + "grad_norm": 1.1801207065582275, + "learning_rate": 0.0001, + "loss": 65.7097, + "step": 3330 + }, + { + "epoch": 53.8780487804878, + "grad_norm": 1.175884485244751, + "learning_rate": 0.0001, + "loss": 64.4368, + "step": 3340 + }, + { + "epoch": 54.03252032520325, + "grad_norm": 1.1727889776229858, + "learning_rate": 0.0001, + "loss": 62.9373, + "step": 3350 + }, + { + "epoch": 54.19512195121951, + "grad_norm": 1.5330945253372192, + "learning_rate": 0.0001, + "loss": 66.7763, + "step": 3360 + }, + { + "epoch": 54.357723577235774, + "grad_norm": 1.2992504835128784, + "learning_rate": 0.0001, + "loss": 65.7854, + "step": 3370 + }, + { + "epoch": 54.520325203252035, + "grad_norm": 1.0185420513153076, + "learning_rate": 0.0001, + "loss": 65.7046, + "step": 3380 + }, + { + "epoch": 54.68292682926829, + "grad_norm": 1.426229476928711, + "learning_rate": 0.0001, + "loss": 64.0286, + "step": 3390 + }, + { + "epoch": 54.84552845528455, + "grad_norm": 1.2600868940353394, + "learning_rate": 0.0001, + "loss": 66.1738, + "step": 3400 + }, + { + "epoch": 55.0, + "grad_norm": 1.917455792427063, + "learning_rate": 0.0001, + "loss": 61.9381, + "step": 3410 + }, + { + "epoch": 55.16260162601626, + "grad_norm": 1.2000917196273804, + "learning_rate": 0.0001, + "loss": 65.5626, + "step": 3420 + }, + { + "epoch": 55.32520325203252, + "grad_norm": 1.362176775932312, + "learning_rate": 0.0001, + "loss": 66.8012, + "step": 3430 + }, + { + "epoch": 55.48780487804878, + "grad_norm": 1.1611402034759521, + "learning_rate": 0.0001, + "loss": 64.5227, + "step": 3440 + }, + { + "epoch": 55.65040650406504, + "grad_norm": 1.2066621780395508, + "learning_rate": 0.0001, + "loss": 63.4351, + "step": 3450 + }, + { + "epoch": 55.8130081300813, + "grad_norm": 1.005340814590454, + "learning_rate": 0.0001, + "loss": 65.9261, + "step": 3460 + }, + { + "epoch": 55.97560975609756, + "grad_norm": 1.0906637907028198, + "learning_rate": 0.0001, + "loss": 64.6635, + "step": 3470 + }, + { + "epoch": 56.13008130081301, + "grad_norm": 1.0608748197555542, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 3480 + }, + { + "epoch": 56.292682926829265, + "grad_norm": 1.025319218635559, + "learning_rate": 0.0001, + "loss": 64.7783, + "step": 3490 + }, + { + "epoch": 56.45528455284553, + "grad_norm": 1.1596784591674805, + "learning_rate": 0.0001, + "loss": 66.4629, + "step": 3500 + }, + { + "epoch": 56.61788617886179, + "grad_norm": 0.9318640232086182, + "learning_rate": 0.0001, + "loss": 65.5533, + "step": 3510 + }, + { + "epoch": 56.78048780487805, + "grad_norm": 1.0977418422698975, + "learning_rate": 0.0001, + "loss": 63.647, + "step": 3520 + }, + { + "epoch": 56.94308943089431, + "grad_norm": 1.3912436962127686, + "learning_rate": 0.0001, + "loss": 62.8273, + "step": 3530 + }, + { + "epoch": 57.09756097560975, + "grad_norm": 1.0921591520309448, + "learning_rate": 0.0001, + "loss": 61.5968, + "step": 3540 + }, + { + "epoch": 57.260162601626014, + "grad_norm": 1.17819344997406, + "learning_rate": 0.0001, + "loss": 64.9381, + "step": 3550 + }, + { + "epoch": 57.422764227642276, + "grad_norm": 1.188829779624939, + "learning_rate": 0.0001, + "loss": 64.6498, + "step": 3560 + }, + { + "epoch": 57.58536585365854, + "grad_norm": 3.664839506149292, + "learning_rate": 0.0001, + "loss": 65.3834, + "step": 3570 + }, + { + "epoch": 57.7479674796748, + "grad_norm": 1.57958984375, + "learning_rate": 0.0001, + "loss": 63.8808, + "step": 3580 + }, + { + "epoch": 57.91056910569106, + "grad_norm": 1.1837552785873413, + "learning_rate": 0.0001, + "loss": 64.1545, + "step": 3590 + }, + { + "epoch": 58.0650406504065, + "grad_norm": 1.5986013412475586, + "learning_rate": 0.0001, + "loss": 60.5742, + "step": 3600 + }, + { + "epoch": 58.22764227642276, + "grad_norm": 1.1551810503005981, + "learning_rate": 0.0001, + "loss": 66.2378, + "step": 3610 + }, + { + "epoch": 58.390243902439025, + "grad_norm": 1.0869863033294678, + "learning_rate": 0.0001, + "loss": 65.3034, + "step": 3620 + }, + { + "epoch": 58.552845528455286, + "grad_norm": 1.2263212203979492, + "learning_rate": 0.0001, + "loss": 63.976, + "step": 3630 + }, + { + "epoch": 58.71544715447155, + "grad_norm": 1.4410817623138428, + "learning_rate": 0.0001, + "loss": 63.4778, + "step": 3640 + }, + { + "epoch": 58.8780487804878, + "grad_norm": 1.9633759260177612, + "learning_rate": 0.0001, + "loss": 65.5946, + "step": 3650 + }, + { + "epoch": 59.03252032520325, + "grad_norm": 1.1729766130447388, + "learning_rate": 0.0001, + "loss": 62.2171, + "step": 3660 + }, + { + "epoch": 59.19512195121951, + "grad_norm": 1.0049556493759155, + "learning_rate": 0.0001, + "loss": 65.1854, + "step": 3670 + }, + { + "epoch": 59.357723577235774, + "grad_norm": 1.1727055311203003, + "learning_rate": 0.0001, + "loss": 65.4839, + "step": 3680 + }, + { + "epoch": 59.520325203252035, + "grad_norm": 1.5394458770751953, + "learning_rate": 0.0001, + "loss": 64.9341, + "step": 3690 + }, + { + "epoch": 59.68292682926829, + "grad_norm": 1.1583774089813232, + "learning_rate": 0.0001, + "loss": 63.9197, + "step": 3700 + }, + { + "epoch": 59.84552845528455, + "grad_norm": 1.0835607051849365, + "learning_rate": 0.0001, + "loss": 64.0494, + "step": 3710 + }, + { + "epoch": 60.0, + "grad_norm": 1.7504246234893799, + "learning_rate": 0.0001, + "loss": 59.4134, + "step": 3720 + }, + { + "epoch": 60.16260162601626, + "grad_norm": 1.1870343685150146, + "learning_rate": 0.0001, + "loss": 63.2158, + "step": 3730 + }, + { + "epoch": 60.32520325203252, + "grad_norm": 1.1983181238174438, + "learning_rate": 0.0001, + "loss": 64.7144, + "step": 3740 + }, + { + "epoch": 60.48780487804878, + "grad_norm": 1.3014581203460693, + "learning_rate": 0.0001, + "loss": 64.6219, + "step": 3750 + }, + { + "epoch": 60.65040650406504, + "grad_norm": 1.4954087734222412, + "learning_rate": 0.0001, + "loss": 63.3785, + "step": 3760 + }, + { + "epoch": 60.8130081300813, + "grad_norm": 1.2236649990081787, + "learning_rate": 0.0001, + "loss": 64.6933, + "step": 3770 + }, + { + "epoch": 60.97560975609756, + "grad_norm": 1.0796610116958618, + "learning_rate": 0.0001, + "loss": 63.2704, + "step": 3780 + }, + { + "epoch": 61.13008130081301, + "grad_norm": 0.9969968795776367, + "learning_rate": 0.0001, + "loss": 62.1838, + "step": 3790 + }, + { + "epoch": 61.292682926829265, + "grad_norm": 1.1624361276626587, + "learning_rate": 0.0001, + "loss": 63.6249, + "step": 3800 + }, + { + "epoch": 61.45528455284553, + "grad_norm": 1.3849223852157593, + "learning_rate": 0.0001, + "loss": 64.5321, + "step": 3810 + }, + { + "epoch": 61.61788617886179, + "grad_norm": 1.6607458591461182, + "learning_rate": 0.0001, + "loss": 64.651, + "step": 3820 + }, + { + "epoch": 61.78048780487805, + "grad_norm": 1.09684419631958, + "learning_rate": 0.0001, + "loss": 64.1129, + "step": 3830 + }, + { + "epoch": 61.94308943089431, + "grad_norm": 1.406246304512024, + "learning_rate": 0.0001, + "loss": 64.3642, + "step": 3840 + }, + { + "epoch": 62.09756097560975, + "grad_norm": 0.9850211143493652, + "learning_rate": 0.0001, + "loss": 60.8703, + "step": 3850 + }, + { + "epoch": 62.260162601626014, + "grad_norm": 1.001344084739685, + "learning_rate": 0.0001, + "loss": 67.5523, + "step": 3860 + }, + { + "epoch": 62.422764227642276, + "grad_norm": 2.435969591140747, + "learning_rate": 0.0001, + "loss": 62.7363, + "step": 3870 + }, + { + "epoch": 62.58536585365854, + "grad_norm": 1.0675824880599976, + "learning_rate": 0.0001, + "loss": 64.8513, + "step": 3880 + }, + { + "epoch": 62.7479674796748, + "grad_norm": 0.9793754816055298, + "learning_rate": 0.0001, + "loss": 64.9904, + "step": 3890 + }, + { + "epoch": 62.91056910569106, + "grad_norm": 1.2187775373458862, + "learning_rate": 0.0001, + "loss": 63.2821, + "step": 3900 + }, + { + "epoch": 63.0650406504065, + "grad_norm": 1.130772590637207, + "learning_rate": 0.0001, + "loss": 61.5957, + "step": 3910 + }, + { + "epoch": 63.22764227642276, + "grad_norm": 1.0573055744171143, + "learning_rate": 0.0001, + "loss": 63.6337, + "step": 3920 + }, + { + "epoch": 63.390243902439025, + "grad_norm": 2.055490255355835, + "learning_rate": 0.0001, + "loss": 63.9598, + "step": 3930 + }, + { + "epoch": 63.552845528455286, + "grad_norm": 1.0780516862869263, + "learning_rate": 0.0001, + "loss": 62.9303, + "step": 3940 + }, + { + "epoch": 63.71544715447155, + "grad_norm": 1.8136590719223022, + "learning_rate": 0.0001, + "loss": 64.4588, + "step": 3950 + }, + { + "epoch": 63.8780487804878, + "grad_norm": 1.0525016784667969, + "learning_rate": 0.0001, + "loss": 64.192, + "step": 3960 + }, + { + "epoch": 64.03252032520325, + "grad_norm": 1.12627375125885, + "learning_rate": 0.0001, + "loss": 59.1823, + "step": 3970 + }, + { + "epoch": 64.1951219512195, + "grad_norm": 1.2558730840682983, + "learning_rate": 0.0001, + "loss": 65.126, + "step": 3980 + }, + { + "epoch": 64.35772357723577, + "grad_norm": 1.2304726839065552, + "learning_rate": 0.0001, + "loss": 62.0868, + "step": 3990 + }, + { + "epoch": 64.52032520325203, + "grad_norm": 1.4179201126098633, + "learning_rate": 0.0001, + "loss": 62.3114, + "step": 4000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 323, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.858820690638065e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c7f229ebe1034caaf3e92006c8c7528c21af93d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517586849d044483051c7a59cec7c6ce389951878a7357ef837edb1cbc9983c8 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-4000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/latest new file mode 100644 index 0000000000000000000000000000000000000000..3ccb5110452b4a1eef79a1c432a18944dc25a985 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/latest @@ -0,0 +1 @@ +global_step6000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2200e5c3e16658b0404dd85e40cca731658d9782 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4afa525c08ac1a9df23591dbb5881d1eee452291c5498cc34d10373d420f5f8c +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..6f0a9b40812f521aab5a948b050c37c480675bfc --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bd15f4ed1af58ac6432b191eb41cab30a02cf0978ad2f17627b85b43e1f9961 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..8ca276769e1f52a116d39a76f8848553a6bbf691 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e08f154873bb3a0ada858ea229eacbf9f701708d0f26ec38d9b3b4bbbcff6b8d +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..90b0042f8a006fc5f9a4150ebcbc207ec69fb3f6 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a461d925b14193589a64ef548cec03efc9c52be0306fefff3fc1b7c371f861e +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..49b11c9aa6bd121a885b24dd8822d21f565ea461 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5fe528b66da9b68b41cd83535401e6d8fa32a110fc30198b4f749e85941bc2d +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..2c13f924bc816526a158fa9443ef032ef8f57ca4 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c41a836388b830d7bb994af20d7ec970b6c2c7c79e6cd6e3c7a34c727e3f6c30 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..ddc00b49fc94e2a88457064ff83ee97fbce6b965 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45d34596d3cac0e2dd648adb6160326ed71188e339deff6acf95788a72e31ddf +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..6d96fb6fac9be9c113cf4b795bb2b7a1f7787bd5 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8180fd49ce894e19e3ebd80c024158f4918ebb10bde93cb1234cdbff2d861f7b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..83dcd031eaf5939c1777cca60b9bd0bf7633f7fc --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a762c77f85ba54a6b329a4c01a665a30492602c520557956a9341a9adf65b124 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..a76cd8e8b9ba139f39ea2186e82134fa532ba2e8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e7682127f80cd25a98dfdc8b12ba353464983fd8dca3440f4fbf60a3e51c0a5 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..efcc4c083cfaf930731c9b0ae69b7d99807d9584 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/trainer_state.json @@ -0,0 +1,4234 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 96.78048780487805, + "eval_steps": 2000, + "global_step": 6000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.16260162601626016, + "grad_norm": 23.72986602783203, + "learning_rate": 0.0001, + "loss": 271.6598, + "step": 10 + }, + { + "epoch": 0.3252032520325203, + "grad_norm": 18.99613380432129, + "learning_rate": 0.0001, + "loss": 233.029, + "step": 20 + }, + { + "epoch": 0.4878048780487805, + "grad_norm": 3.9815304279327393, + "learning_rate": 0.0001, + "loss": 231.0776, + "step": 30 + }, + { + "epoch": 0.6504065040650406, + "grad_norm": 4.641753196716309, + "learning_rate": 0.0001, + "loss": 228.6894, + "step": 40 + }, + { + "epoch": 0.8130081300813008, + "grad_norm": 3.000852108001709, + "learning_rate": 0.0001, + "loss": 228.4694, + "step": 50 + }, + { + "epoch": 0.975609756097561, + "grad_norm": 12.213018417358398, + "learning_rate": 0.0001, + "loss": 226.4258, + "step": 60 + }, + { + "epoch": 1.1300813008130082, + "grad_norm": 2.535187005996704, + "learning_rate": 0.0001, + "loss": 209.9772, + "step": 70 + }, + { + "epoch": 1.2926829268292683, + "grad_norm": 2.0984582901000977, + "learning_rate": 0.0001, + "loss": 217.2421, + "step": 80 + }, + { + "epoch": 1.4552845528455285, + "grad_norm": 3.4802305698394775, + "learning_rate": 0.0001, + "loss": 210.9241, + "step": 90 + }, + { + "epoch": 1.6178861788617886, + "grad_norm": 3.813626527786255, + "learning_rate": 0.0001, + "loss": 205.7566, + "step": 100 + }, + { + "epoch": 1.7804878048780488, + "grad_norm": 2.6378512382507324, + "learning_rate": 0.0001, + "loss": 201.8789, + "step": 110 + }, + { + "epoch": 1.943089430894309, + "grad_norm": 2.7502875328063965, + "learning_rate": 0.0001, + "loss": 196.373, + "step": 120 + }, + { + "epoch": 2.097560975609756, + "grad_norm": 2.659212112426758, + "learning_rate": 0.0001, + "loss": 183.1456, + "step": 130 + }, + { + "epoch": 2.2601626016260163, + "grad_norm": 2.513044595718384, + "learning_rate": 0.0001, + "loss": 190.1661, + "step": 140 + }, + { + "epoch": 2.4227642276422765, + "grad_norm": 2.3192145824432373, + "learning_rate": 0.0001, + "loss": 184.9997, + "step": 150 + }, + { + "epoch": 2.5853658536585367, + "grad_norm": 3.056708574295044, + "learning_rate": 0.0001, + "loss": 182.0164, + "step": 160 + }, + { + "epoch": 2.747967479674797, + "grad_norm": 2.9687108993530273, + "learning_rate": 0.0001, + "loss": 179.6539, + "step": 170 + }, + { + "epoch": 2.910569105691057, + "grad_norm": 2.1498093605041504, + "learning_rate": 0.0001, + "loss": 176.7159, + "step": 180 + }, + { + "epoch": 3.065040650406504, + "grad_norm": 3.0103771686553955, + "learning_rate": 0.0001, + "loss": 166.1703, + "step": 190 + }, + { + "epoch": 3.227642276422764, + "grad_norm": 2.992462635040283, + "learning_rate": 0.0001, + "loss": 171.124, + "step": 200 + }, + { + "epoch": 3.3902439024390243, + "grad_norm": 4.8617119789123535, + "learning_rate": 0.0001, + "loss": 168.2646, + "step": 210 + }, + { + "epoch": 3.5528455284552845, + "grad_norm": 3.1047444343566895, + "learning_rate": 0.0001, + "loss": 163.0889, + "step": 220 + }, + { + "epoch": 3.7154471544715446, + "grad_norm": 6.473227500915527, + "learning_rate": 0.0001, + "loss": 151.3667, + "step": 230 + }, + { + "epoch": 3.8780487804878048, + "grad_norm": 6.16747522354126, + "learning_rate": 0.0001, + "loss": 137.5915, + "step": 240 + }, + { + "epoch": 4.032520325203252, + "grad_norm": 7.994515419006348, + "learning_rate": 0.0001, + "loss": 118.8875, + "step": 250 + }, + { + "epoch": 4.195121951219512, + "grad_norm": 5.882101058959961, + "learning_rate": 0.0001, + "loss": 119.4793, + "step": 260 + }, + { + "epoch": 4.357723577235772, + "grad_norm": 12.32968521118164, + "learning_rate": 0.0001, + "loss": 113.1209, + "step": 270 + }, + { + "epoch": 4.520325203252033, + "grad_norm": 5.697225570678711, + "learning_rate": 0.0001, + "loss": 107.51, + "step": 280 + }, + { + "epoch": 4.682926829268292, + "grad_norm": 4.654167175292969, + "learning_rate": 0.0001, + "loss": 107.0172, + "step": 290 + }, + { + "epoch": 4.845528455284553, + "grad_norm": 2.9595484733581543, + "learning_rate": 0.0001, + "loss": 100.9164, + "step": 300 + }, + { + "epoch": 5.0, + "grad_norm": 21.242591857910156, + "learning_rate": 0.0001, + "loss": 94.9884, + "step": 310 + }, + { + "epoch": 5.16260162601626, + "grad_norm": 5.760314464569092, + "learning_rate": 0.0001, + "loss": 97.5901, + "step": 320 + }, + { + "epoch": 5.32520325203252, + "grad_norm": 3.1373329162597656, + "learning_rate": 0.0001, + "loss": 94.8485, + "step": 330 + }, + { + "epoch": 5.487804878048781, + "grad_norm": 2.3777568340301514, + "learning_rate": 0.0001, + "loss": 96.9982, + "step": 340 + }, + { + "epoch": 5.650406504065041, + "grad_norm": 3.0607948303222656, + "learning_rate": 0.0001, + "loss": 96.1586, + "step": 350 + }, + { + "epoch": 5.8130081300813, + "grad_norm": 4.168269634246826, + "learning_rate": 0.0001, + "loss": 94.5743, + "step": 360 + }, + { + "epoch": 5.975609756097561, + "grad_norm": 2.4383699893951416, + "learning_rate": 0.0001, + "loss": 95.5197, + "step": 370 + }, + { + "epoch": 6.130081300813008, + "grad_norm": 2.302259683609009, + "learning_rate": 0.0001, + "loss": 85.6599, + "step": 380 + }, + { + "epoch": 6.2926829268292686, + "grad_norm": 2.552753210067749, + "learning_rate": 0.0001, + "loss": 91.2999, + "step": 390 + }, + { + "epoch": 6.455284552845528, + "grad_norm": 2.2092080116271973, + "learning_rate": 0.0001, + "loss": 90.7041, + "step": 400 + }, + { + "epoch": 6.617886178861789, + "grad_norm": 2.2935690879821777, + "learning_rate": 0.0001, + "loss": 89.9949, + "step": 410 + }, + { + "epoch": 6.780487804878049, + "grad_norm": 2.487510919570923, + "learning_rate": 0.0001, + "loss": 86.9056, + "step": 420 + }, + { + "epoch": 6.943089430894309, + "grad_norm": 2.8630545139312744, + "learning_rate": 0.0001, + "loss": 88.3347, + "step": 430 + }, + { + "epoch": 7.097560975609756, + "grad_norm": 5.87056303024292, + "learning_rate": 0.0001, + "loss": 84.9632, + "step": 440 + }, + { + "epoch": 7.260162601626016, + "grad_norm": 2.587661027908325, + "learning_rate": 0.0001, + "loss": 86.2024, + "step": 450 + }, + { + "epoch": 7.4227642276422765, + "grad_norm": 3.5671255588531494, + "learning_rate": 0.0001, + "loss": 86.218, + "step": 460 + }, + { + "epoch": 7.585365853658536, + "grad_norm": 2.5036680698394775, + "learning_rate": 0.0001, + "loss": 86.5914, + "step": 470 + }, + { + "epoch": 7.747967479674797, + "grad_norm": 3.2656471729278564, + "learning_rate": 0.0001, + "loss": 82.6081, + "step": 480 + }, + { + "epoch": 7.9105691056910565, + "grad_norm": 2.5053915977478027, + "learning_rate": 0.0001, + "loss": 84.6116, + "step": 490 + }, + { + "epoch": 8.065040650406504, + "grad_norm": 2.1669957637786865, + "learning_rate": 0.0001, + "loss": 80.4354, + "step": 500 + }, + { + "epoch": 8.227642276422765, + "grad_norm": 4.2003703117370605, + "learning_rate": 0.0001, + "loss": 85.2286, + "step": 510 + }, + { + "epoch": 8.390243902439025, + "grad_norm": 3.1909608840942383, + "learning_rate": 0.0001, + "loss": 85.0358, + "step": 520 + }, + { + "epoch": 8.552845528455284, + "grad_norm": 1.8420121669769287, + "learning_rate": 0.0001, + "loss": 85.4647, + "step": 530 + }, + { + "epoch": 8.715447154471544, + "grad_norm": 1.9822801351547241, + "learning_rate": 0.0001, + "loss": 84.7134, + "step": 540 + }, + { + "epoch": 8.878048780487806, + "grad_norm": 2.936758518218994, + "learning_rate": 0.0001, + "loss": 83.8068, + "step": 550 + }, + { + "epoch": 9.032520325203253, + "grad_norm": 1.5713881254196167, + "learning_rate": 0.0001, + "loss": 80.0695, + "step": 560 + }, + { + "epoch": 9.195121951219512, + "grad_norm": 2.0584616661071777, + "learning_rate": 0.0001, + "loss": 84.5158, + "step": 570 + }, + { + "epoch": 9.357723577235772, + "grad_norm": 2.101391553878784, + "learning_rate": 0.0001, + "loss": 82.4719, + "step": 580 + }, + { + "epoch": 9.520325203252032, + "grad_norm": 1.9956543445587158, + "learning_rate": 0.0001, + "loss": 81.6715, + "step": 590 + }, + { + "epoch": 9.682926829268293, + "grad_norm": 2.7622063159942627, + "learning_rate": 0.0001, + "loss": 85.4178, + "step": 600 + }, + { + "epoch": 9.845528455284553, + "grad_norm": 2.2084789276123047, + "learning_rate": 0.0001, + "loss": 84.5702, + "step": 610 + }, + { + "epoch": 10.0, + "grad_norm": 1.9635663032531738, + "learning_rate": 0.0001, + "loss": 77.3838, + "step": 620 + }, + { + "epoch": 10.16260162601626, + "grad_norm": 2.5895581245422363, + "learning_rate": 0.0001, + "loss": 81.723, + "step": 630 + }, + { + "epoch": 10.32520325203252, + "grad_norm": 2.743727445602417, + "learning_rate": 0.0001, + "loss": 83.1332, + "step": 640 + }, + { + "epoch": 10.487804878048781, + "grad_norm": 2.024585008621216, + "learning_rate": 0.0001, + "loss": 81.5968, + "step": 650 + }, + { + "epoch": 10.65040650406504, + "grad_norm": 1.8254153728485107, + "learning_rate": 0.0001, + "loss": 78.9589, + "step": 660 + }, + { + "epoch": 10.8130081300813, + "grad_norm": 3.2015740871429443, + "learning_rate": 0.0001, + "loss": 79.1829, + "step": 670 + }, + { + "epoch": 10.975609756097562, + "grad_norm": 1.788453459739685, + "learning_rate": 0.0001, + "loss": 81.527, + "step": 680 + }, + { + "epoch": 11.130081300813009, + "grad_norm": 1.5744423866271973, + "learning_rate": 0.0001, + "loss": 74.9869, + "step": 690 + }, + { + "epoch": 11.292682926829269, + "grad_norm": 2.006002187728882, + "learning_rate": 0.0001, + "loss": 80.2677, + "step": 700 + }, + { + "epoch": 11.455284552845528, + "grad_norm": 1.8847148418426514, + "learning_rate": 0.0001, + "loss": 78.8709, + "step": 710 + }, + { + "epoch": 11.617886178861788, + "grad_norm": 1.8384478092193604, + "learning_rate": 0.0001, + "loss": 79.008, + "step": 720 + }, + { + "epoch": 11.78048780487805, + "grad_norm": 1.9486687183380127, + "learning_rate": 0.0001, + "loss": 76.1525, + "step": 730 + }, + { + "epoch": 11.94308943089431, + "grad_norm": 1.8644076585769653, + "learning_rate": 0.0001, + "loss": 77.5802, + "step": 740 + }, + { + "epoch": 12.097560975609756, + "grad_norm": 1.6035499572753906, + "learning_rate": 0.0001, + "loss": 75.2422, + "step": 750 + }, + { + "epoch": 12.260162601626016, + "grad_norm": 1.6935622692108154, + "learning_rate": 0.0001, + "loss": 77.5838, + "step": 760 + }, + { + "epoch": 12.422764227642276, + "grad_norm": 1.9327987432479858, + "learning_rate": 0.0001, + "loss": 78.3237, + "step": 770 + }, + { + "epoch": 12.585365853658537, + "grad_norm": 1.7903852462768555, + "learning_rate": 0.0001, + "loss": 78.7459, + "step": 780 + }, + { + "epoch": 12.747967479674797, + "grad_norm": 1.4962575435638428, + "learning_rate": 0.0001, + "loss": 78.9193, + "step": 790 + }, + { + "epoch": 12.910569105691057, + "grad_norm": 42.10435104370117, + "learning_rate": 0.0001, + "loss": 80.1429, + "step": 800 + }, + { + "epoch": 13.065040650406504, + "grad_norm": 1.904065728187561, + "learning_rate": 0.0001, + "loss": 73.7798, + "step": 810 + }, + { + "epoch": 13.227642276422765, + "grad_norm": 1.8768481016159058, + "learning_rate": 0.0001, + "loss": 76.8692, + "step": 820 + }, + { + "epoch": 13.390243902439025, + "grad_norm": 16.140565872192383, + "learning_rate": 0.0001, + "loss": 78.122, + "step": 830 + }, + { + "epoch": 13.552845528455284, + "grad_norm": 2.160288095474243, + "learning_rate": 0.0001, + "loss": 76.2169, + "step": 840 + }, + { + "epoch": 13.715447154471544, + "grad_norm": 27.682231903076172, + "learning_rate": 0.0001, + "loss": 76.6266, + "step": 850 + }, + { + "epoch": 13.878048780487806, + "grad_norm": 1.779772162437439, + "learning_rate": 0.0001, + "loss": 80.0434, + "step": 860 + }, + { + "epoch": 14.032520325203253, + "grad_norm": 1.6516485214233398, + "learning_rate": 0.0001, + "loss": 73.3943, + "step": 870 + }, + { + "epoch": 14.195121951219512, + "grad_norm": 1.846498966217041, + "learning_rate": 0.0001, + "loss": 77.5157, + "step": 880 + }, + { + "epoch": 14.357723577235772, + "grad_norm": 1.942946434020996, + "learning_rate": 0.0001, + "loss": 77.6944, + "step": 890 + }, + { + "epoch": 14.520325203252032, + "grad_norm": 2.0929224491119385, + "learning_rate": 0.0001, + "loss": 76.9411, + "step": 900 + }, + { + "epoch": 14.682926829268293, + "grad_norm": 3.4498767852783203, + "learning_rate": 0.0001, + "loss": 74.9234, + "step": 910 + }, + { + "epoch": 14.845528455284553, + "grad_norm": 1.6880728006362915, + "learning_rate": 0.0001, + "loss": 75.3617, + "step": 920 + }, + { + "epoch": 15.0, + "grad_norm": 1.8403468132019043, + "learning_rate": 0.0001, + "loss": 73.2543, + "step": 930 + }, + { + "epoch": 15.16260162601626, + "grad_norm": 1.4368852376937866, + "learning_rate": 0.0001, + "loss": 76.6271, + "step": 940 + }, + { + "epoch": 15.32520325203252, + "grad_norm": 2.9467780590057373, + "learning_rate": 0.0001, + "loss": 75.8744, + "step": 950 + }, + { + "epoch": 15.487804878048781, + "grad_norm": 1.567134141921997, + "learning_rate": 0.0001, + "loss": 76.6629, + "step": 960 + }, + { + "epoch": 15.65040650406504, + "grad_norm": 1.4503871202468872, + "learning_rate": 0.0001, + "loss": 76.3525, + "step": 970 + }, + { + "epoch": 15.8130081300813, + "grad_norm": 1.8871147632598877, + "learning_rate": 0.0001, + "loss": 77.4684, + "step": 980 + }, + { + "epoch": 15.975609756097562, + "grad_norm": 1.6707026958465576, + "learning_rate": 0.0001, + "loss": 76.3063, + "step": 990 + }, + { + "epoch": 16.130081300813007, + "grad_norm": 3.6300716400146484, + "learning_rate": 0.0001, + "loss": 73.0586, + "step": 1000 + }, + { + "epoch": 16.29268292682927, + "grad_norm": 1.6669234037399292, + "learning_rate": 0.0001, + "loss": 75.8027, + "step": 1010 + }, + { + "epoch": 16.45528455284553, + "grad_norm": 1.6920552253723145, + "learning_rate": 0.0001, + "loss": 74.7252, + "step": 1020 + }, + { + "epoch": 16.617886178861788, + "grad_norm": 1.8184431791305542, + "learning_rate": 0.0001, + "loss": 75.3625, + "step": 1030 + }, + { + "epoch": 16.78048780487805, + "grad_norm": 1.5828496217727661, + "learning_rate": 0.0001, + "loss": 78.0302, + "step": 1040 + }, + { + "epoch": 16.943089430894307, + "grad_norm": 1.7372915744781494, + "learning_rate": 0.0001, + "loss": 78.6949, + "step": 1050 + }, + { + "epoch": 17.097560975609756, + "grad_norm": 1.837161898612976, + "learning_rate": 0.0001, + "loss": 74.4726, + "step": 1060 + }, + { + "epoch": 17.260162601626018, + "grad_norm": 3.8733808994293213, + "learning_rate": 0.0001, + "loss": 76.1711, + "step": 1070 + }, + { + "epoch": 17.422764227642276, + "grad_norm": 1.5907851457595825, + "learning_rate": 0.0001, + "loss": 75.8292, + "step": 1080 + }, + { + "epoch": 17.585365853658537, + "grad_norm": 1.8271747827529907, + "learning_rate": 0.0001, + "loss": 74.5316, + "step": 1090 + }, + { + "epoch": 17.747967479674795, + "grad_norm": 1.827980875968933, + "learning_rate": 0.0001, + "loss": 74.7281, + "step": 1100 + }, + { + "epoch": 17.910569105691057, + "grad_norm": 1.4184074401855469, + "learning_rate": 0.0001, + "loss": 75.736, + "step": 1110 + }, + { + "epoch": 18.065040650406505, + "grad_norm": 2.3459699153900146, + "learning_rate": 0.0001, + "loss": 71.9586, + "step": 1120 + }, + { + "epoch": 18.227642276422763, + "grad_norm": 47.673641204833984, + "learning_rate": 0.0001, + "loss": 75.6649, + "step": 1130 + }, + { + "epoch": 18.390243902439025, + "grad_norm": 1.677398443222046, + "learning_rate": 0.0001, + "loss": 76.0574, + "step": 1140 + }, + { + "epoch": 18.552845528455286, + "grad_norm": 1.4180850982666016, + "learning_rate": 0.0001, + "loss": 74.3077, + "step": 1150 + }, + { + "epoch": 18.715447154471544, + "grad_norm": 2.0041465759277344, + "learning_rate": 0.0001, + "loss": 73.5916, + "step": 1160 + }, + { + "epoch": 18.878048780487806, + "grad_norm": 1.9098238945007324, + "learning_rate": 0.0001, + "loss": 74.5002, + "step": 1170 + }, + { + "epoch": 19.03252032520325, + "grad_norm": 1.405555248260498, + "learning_rate": 0.0001, + "loss": 69.8115, + "step": 1180 + }, + { + "epoch": 19.195121951219512, + "grad_norm": 1.424198865890503, + "learning_rate": 0.0001, + "loss": 73.7289, + "step": 1190 + }, + { + "epoch": 19.357723577235774, + "grad_norm": 7.542550563812256, + "learning_rate": 0.0001, + "loss": 73.7696, + "step": 1200 + }, + { + "epoch": 19.520325203252032, + "grad_norm": 1.425329566001892, + "learning_rate": 0.0001, + "loss": 72.1261, + "step": 1210 + }, + { + "epoch": 19.682926829268293, + "grad_norm": 1.525724172592163, + "learning_rate": 0.0001, + "loss": 73.6578, + "step": 1220 + }, + { + "epoch": 19.84552845528455, + "grad_norm": 1.6387193202972412, + "learning_rate": 0.0001, + "loss": 74.3027, + "step": 1230 + }, + { + "epoch": 20.0, + "grad_norm": 1.8429840803146362, + "learning_rate": 0.0001, + "loss": 69.5056, + "step": 1240 + }, + { + "epoch": 20.16260162601626, + "grad_norm": 2.5166006088256836, + "learning_rate": 0.0001, + "loss": 73.5965, + "step": 1250 + }, + { + "epoch": 20.32520325203252, + "grad_norm": 1.6006672382354736, + "learning_rate": 0.0001, + "loss": 72.4581, + "step": 1260 + }, + { + "epoch": 20.48780487804878, + "grad_norm": 1.6722878217697144, + "learning_rate": 0.0001, + "loss": 72.3097, + "step": 1270 + }, + { + "epoch": 20.65040650406504, + "grad_norm": 1.8715728521347046, + "learning_rate": 0.0001, + "loss": 73.4752, + "step": 1280 + }, + { + "epoch": 20.8130081300813, + "grad_norm": 1.3280900716781616, + "learning_rate": 0.0001, + "loss": 73.2145, + "step": 1290 + }, + { + "epoch": 20.975609756097562, + "grad_norm": 1.941451907157898, + "learning_rate": 0.0001, + "loss": 73.2085, + "step": 1300 + }, + { + "epoch": 21.130081300813007, + "grad_norm": 4.277497291564941, + "learning_rate": 0.0001, + "loss": 68.8305, + "step": 1310 + }, + { + "epoch": 21.29268292682927, + "grad_norm": 1.9949846267700195, + "learning_rate": 0.0001, + "loss": 72.6689, + "step": 1320 + }, + { + "epoch": 21.45528455284553, + "grad_norm": 1.524044394493103, + "learning_rate": 0.0001, + "loss": 72.9705, + "step": 1330 + }, + { + "epoch": 21.617886178861788, + "grad_norm": 1.384194016456604, + "learning_rate": 0.0001, + "loss": 72.5465, + "step": 1340 + }, + { + "epoch": 21.78048780487805, + "grad_norm": 1.724225640296936, + "learning_rate": 0.0001, + "loss": 72.9203, + "step": 1350 + }, + { + "epoch": 21.943089430894307, + "grad_norm": 2.0856704711914062, + "learning_rate": 0.0001, + "loss": 72.4034, + "step": 1360 + }, + { + "epoch": 22.097560975609756, + "grad_norm": 5.128044128417969, + "learning_rate": 0.0001, + "loss": 68.7754, + "step": 1370 + }, + { + "epoch": 22.260162601626018, + "grad_norm": 1.7385634183883667, + "learning_rate": 0.0001, + "loss": 73.9349, + "step": 1380 + }, + { + "epoch": 22.422764227642276, + "grad_norm": 1.3946012258529663, + "learning_rate": 0.0001, + "loss": 70.9627, + "step": 1390 + }, + { + "epoch": 22.585365853658537, + "grad_norm": 1.285414457321167, + "learning_rate": 0.0001, + "loss": 72.4578, + "step": 1400 + }, + { + "epoch": 22.747967479674795, + "grad_norm": 1.4197710752487183, + "learning_rate": 0.0001, + "loss": 72.5252, + "step": 1410 + }, + { + "epoch": 22.910569105691057, + "grad_norm": 3.048942804336548, + "learning_rate": 0.0001, + "loss": 71.1107, + "step": 1420 + }, + { + "epoch": 23.065040650406505, + "grad_norm": 1.3114765882492065, + "learning_rate": 0.0001, + "loss": 68.2725, + "step": 1430 + }, + { + "epoch": 23.227642276422763, + "grad_norm": 1.2763880491256714, + "learning_rate": 0.0001, + "loss": 71.5783, + "step": 1440 + }, + { + "epoch": 23.390243902439025, + "grad_norm": 1.3913089036941528, + "learning_rate": 0.0001, + "loss": 72.5986, + "step": 1450 + }, + { + "epoch": 23.552845528455286, + "grad_norm": 1.628751516342163, + "learning_rate": 0.0001, + "loss": 71.8383, + "step": 1460 + }, + { + "epoch": 23.715447154471544, + "grad_norm": 1.4331274032592773, + "learning_rate": 0.0001, + "loss": 70.8051, + "step": 1470 + }, + { + "epoch": 23.878048780487806, + "grad_norm": 1.4154727458953857, + "learning_rate": 0.0001, + "loss": 71.3283, + "step": 1480 + }, + { + "epoch": 24.03252032520325, + "grad_norm": 1.7208784818649292, + "learning_rate": 0.0001, + "loss": 68.4719, + "step": 1490 + }, + { + "epoch": 24.195121951219512, + "grad_norm": 1.466123342514038, + "learning_rate": 0.0001, + "loss": 71.9597, + "step": 1500 + }, + { + "epoch": 24.357723577235774, + "grad_norm": 1.4823886156082153, + "learning_rate": 0.0001, + "loss": 72.4533, + "step": 1510 + }, + { + "epoch": 24.520325203252032, + "grad_norm": 1.8407355546951294, + "learning_rate": 0.0001, + "loss": 71.1468, + "step": 1520 + }, + { + "epoch": 24.682926829268293, + "grad_norm": 1.630565881729126, + "learning_rate": 0.0001, + "loss": 71.5192, + "step": 1530 + }, + { + "epoch": 24.84552845528455, + "grad_norm": 1.5248785018920898, + "learning_rate": 0.0001, + "loss": 68.4804, + "step": 1540 + }, + { + "epoch": 25.0, + "grad_norm": 1.4690219163894653, + "learning_rate": 0.0001, + "loss": 67.5716, + "step": 1550 + }, + { + "epoch": 25.16260162601626, + "grad_norm": 2.0246646404266357, + "learning_rate": 0.0001, + "loss": 71.2554, + "step": 1560 + }, + { + "epoch": 25.32520325203252, + "grad_norm": 1.3611458539962769, + "learning_rate": 0.0001, + "loss": 69.8558, + "step": 1570 + }, + { + "epoch": 25.48780487804878, + "grad_norm": 2.0006039142608643, + "learning_rate": 0.0001, + "loss": 72.5344, + "step": 1580 + }, + { + "epoch": 25.65040650406504, + "grad_norm": 1.7373167276382446, + "learning_rate": 0.0001, + "loss": 70.6253, + "step": 1590 + }, + { + "epoch": 25.8130081300813, + "grad_norm": 1.2878378629684448, + "learning_rate": 0.0001, + "loss": 72.165, + "step": 1600 + }, + { + "epoch": 25.975609756097562, + "grad_norm": 1.5337013006210327, + "learning_rate": 0.0001, + "loss": 70.6989, + "step": 1610 + }, + { + "epoch": 26.130081300813007, + "grad_norm": 1.364615559577942, + "learning_rate": 0.0001, + "loss": 67.3898, + "step": 1620 + }, + { + "epoch": 26.29268292682927, + "grad_norm": 1.3249316215515137, + "learning_rate": 0.0001, + "loss": 72.7102, + "step": 1630 + }, + { + "epoch": 26.45528455284553, + "grad_norm": 4.209898471832275, + "learning_rate": 0.0001, + "loss": 70.3185, + "step": 1640 + }, + { + "epoch": 26.617886178861788, + "grad_norm": 1.5488810539245605, + "learning_rate": 0.0001, + "loss": 72.3678, + "step": 1650 + }, + { + "epoch": 26.78048780487805, + "grad_norm": 1.7743597030639648, + "learning_rate": 0.0001, + "loss": 71.0917, + "step": 1660 + }, + { + "epoch": 26.943089430894307, + "grad_norm": 1.5433533191680908, + "learning_rate": 0.0001, + "loss": 71.4001, + "step": 1670 + }, + { + "epoch": 27.097560975609756, + "grad_norm": 1.6320385932922363, + "learning_rate": 0.0001, + "loss": 66.7441, + "step": 1680 + }, + { + "epoch": 27.260162601626018, + "grad_norm": 1.3137969970703125, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 1690 + }, + { + "epoch": 27.422764227642276, + "grad_norm": 1.9150484800338745, + "learning_rate": 0.0001, + "loss": 69.8956, + "step": 1700 + }, + { + "epoch": 27.585365853658537, + "grad_norm": 1.4442802667617798, + "learning_rate": 0.0001, + "loss": 70.7185, + "step": 1710 + }, + { + "epoch": 27.747967479674795, + "grad_norm": 2.572497606277466, + "learning_rate": 0.0001, + "loss": 70.8934, + "step": 1720 + }, + { + "epoch": 27.910569105691057, + "grad_norm": 1.6669617891311646, + "learning_rate": 0.0001, + "loss": 71.8221, + "step": 1730 + }, + { + "epoch": 28.065040650406505, + "grad_norm": 1.8449876308441162, + "learning_rate": 0.0001, + "loss": 66.3923, + "step": 1740 + }, + { + "epoch": 28.227642276422763, + "grad_norm": 1.6600004434585571, + "learning_rate": 0.0001, + "loss": 69.5301, + "step": 1750 + }, + { + "epoch": 28.390243902439025, + "grad_norm": 1.4655731916427612, + "learning_rate": 0.0001, + "loss": 72.1924, + "step": 1760 + }, + { + "epoch": 28.552845528455286, + "grad_norm": 1.134505033493042, + "learning_rate": 0.0001, + "loss": 69.5855, + "step": 1770 + }, + { + "epoch": 28.715447154471544, + "grad_norm": 1.301984190940857, + "learning_rate": 0.0001, + "loss": 70.7999, + "step": 1780 + }, + { + "epoch": 28.878048780487806, + "grad_norm": 1.5641013383865356, + "learning_rate": 0.0001, + "loss": 69.9091, + "step": 1790 + }, + { + "epoch": 29.03252032520325, + "grad_norm": 1.4850797653198242, + "learning_rate": 0.0001, + "loss": 66.6266, + "step": 1800 + }, + { + "epoch": 29.195121951219512, + "grad_norm": 1.2069118022918701, + "learning_rate": 0.0001, + "loss": 69.3802, + "step": 1810 + }, + { + "epoch": 29.357723577235774, + "grad_norm": 1.2890352010726929, + "learning_rate": 0.0001, + "loss": 68.5044, + "step": 1820 + }, + { + "epoch": 29.520325203252032, + "grad_norm": 1.4150917530059814, + "learning_rate": 0.0001, + "loss": 69.807, + "step": 1830 + }, + { + "epoch": 29.682926829268293, + "grad_norm": 1.514807939529419, + "learning_rate": 0.0001, + "loss": 71.3858, + "step": 1840 + }, + { + "epoch": 29.84552845528455, + "grad_norm": 1.4246330261230469, + "learning_rate": 0.0001, + "loss": 70.494, + "step": 1850 + }, + { + "epoch": 30.0, + "grad_norm": 1.1789597272872925, + "learning_rate": 0.0001, + "loss": 67.1056, + "step": 1860 + }, + { + "epoch": 30.16260162601626, + "grad_norm": 1.23678719997406, + "learning_rate": 0.0001, + "loss": 70.336, + "step": 1870 + }, + { + "epoch": 30.32520325203252, + "grad_norm": 1.1756274700164795, + "learning_rate": 0.0001, + "loss": 67.9909, + "step": 1880 + }, + { + "epoch": 30.48780487804878, + "grad_norm": 1.4175690412521362, + "learning_rate": 0.0001, + "loss": 69.2617, + "step": 1890 + }, + { + "epoch": 30.65040650406504, + "grad_norm": 1.3084465265274048, + "learning_rate": 0.0001, + "loss": 70.2963, + "step": 1900 + }, + { + "epoch": 30.8130081300813, + "grad_norm": 1.486464500427246, + "learning_rate": 0.0001, + "loss": 68.6317, + "step": 1910 + }, + { + "epoch": 30.975609756097562, + "grad_norm": 10.415762901306152, + "learning_rate": 0.0001, + "loss": 69.7767, + "step": 1920 + }, + { + "epoch": 31.130081300813007, + "grad_norm": 1.4484038352966309, + "learning_rate": 0.0001, + "loss": 65.6345, + "step": 1930 + }, + { + "epoch": 31.29268292682927, + "grad_norm": 2.297306537628174, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 1940 + }, + { + "epoch": 31.45528455284553, + "grad_norm": 1.223490595817566, + "learning_rate": 0.0001, + "loss": 69.2139, + "step": 1950 + }, + { + "epoch": 31.617886178861788, + "grad_norm": 1.2144125699996948, + "learning_rate": 0.0001, + "loss": 70.3061, + "step": 1960 + }, + { + "epoch": 31.78048780487805, + "grad_norm": 2.633528470993042, + "learning_rate": 0.0001, + "loss": 68.7096, + "step": 1970 + }, + { + "epoch": 31.943089430894307, + "grad_norm": 1.5048350095748901, + "learning_rate": 0.0001, + "loss": 68.6711, + "step": 1980 + }, + { + "epoch": 32.09756097560975, + "grad_norm": 1.5154467821121216, + "learning_rate": 0.0001, + "loss": 66.7989, + "step": 1990 + }, + { + "epoch": 32.260162601626014, + "grad_norm": 2.721277952194214, + "learning_rate": 0.0001, + "loss": 70.4987, + "step": 2000 + }, + { + "epoch": 32.422764227642276, + "grad_norm": 1.44464910030365, + "learning_rate": 0.0001, + "loss": 69.9712, + "step": 2010 + }, + { + "epoch": 32.58536585365854, + "grad_norm": 1.3462835550308228, + "learning_rate": 0.0001, + "loss": 68.9381, + "step": 2020 + }, + { + "epoch": 32.7479674796748, + "grad_norm": 1.3283029794692993, + "learning_rate": 0.0001, + "loss": 68.5301, + "step": 2030 + }, + { + "epoch": 32.91056910569106, + "grad_norm": 1.5172960758209229, + "learning_rate": 0.0001, + "loss": 70.1894, + "step": 2040 + }, + { + "epoch": 33.0650406504065, + "grad_norm": 1.1872576475143433, + "learning_rate": 0.0001, + "loss": 65.0158, + "step": 2050 + }, + { + "epoch": 33.22764227642276, + "grad_norm": 1.5414419174194336, + "learning_rate": 0.0001, + "loss": 69.1537, + "step": 2060 + }, + { + "epoch": 33.390243902439025, + "grad_norm": 1.3143430948257446, + "learning_rate": 0.0001, + "loss": 68.6792, + "step": 2070 + }, + { + "epoch": 33.552845528455286, + "grad_norm": 1.3863723278045654, + "learning_rate": 0.0001, + "loss": 70.3867, + "step": 2080 + }, + { + "epoch": 33.71544715447155, + "grad_norm": 1.4060479402542114, + "learning_rate": 0.0001, + "loss": 68.4981, + "step": 2090 + }, + { + "epoch": 33.8780487804878, + "grad_norm": 1.34132981300354, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 2100 + }, + { + "epoch": 34.03252032520325, + "grad_norm": 1.8303673267364502, + "learning_rate": 0.0001, + "loss": 65.819, + "step": 2110 + }, + { + "epoch": 34.19512195121951, + "grad_norm": 1.387339472770691, + "learning_rate": 0.0001, + "loss": 69.9825, + "step": 2120 + }, + { + "epoch": 34.357723577235774, + "grad_norm": 1.4125195741653442, + "learning_rate": 0.0001, + "loss": 67.5149, + "step": 2130 + }, + { + "epoch": 34.520325203252035, + "grad_norm": 1.2909412384033203, + "learning_rate": 0.0001, + "loss": 69.7848, + "step": 2140 + }, + { + "epoch": 34.68292682926829, + "grad_norm": 1.8860795497894287, + "learning_rate": 0.0001, + "loss": 68.6145, + "step": 2150 + }, + { + "epoch": 34.84552845528455, + "grad_norm": 1.5111079216003418, + "learning_rate": 0.0001, + "loss": 68.2612, + "step": 2160 + }, + { + "epoch": 35.0, + "grad_norm": 1.3275264501571655, + "learning_rate": 0.0001, + "loss": 64.0298, + "step": 2170 + }, + { + "epoch": 35.16260162601626, + "grad_norm": 1.174864411354065, + "learning_rate": 0.0001, + "loss": 66.6153, + "step": 2180 + }, + { + "epoch": 35.32520325203252, + "grad_norm": 1.6873915195465088, + "learning_rate": 0.0001, + "loss": 69.6278, + "step": 2190 + }, + { + "epoch": 35.48780487804878, + "grad_norm": 2.4457507133483887, + "learning_rate": 0.0001, + "loss": 68.3611, + "step": 2200 + }, + { + "epoch": 35.65040650406504, + "grad_norm": 1.4779958724975586, + "learning_rate": 0.0001, + "loss": 69.2609, + "step": 2210 + }, + { + "epoch": 35.8130081300813, + "grad_norm": 2.273709774017334, + "learning_rate": 0.0001, + "loss": 69.1312, + "step": 2220 + }, + { + "epoch": 35.97560975609756, + "grad_norm": 1.5741862058639526, + "learning_rate": 0.0001, + "loss": 69.1191, + "step": 2230 + }, + { + "epoch": 36.13008130081301, + "grad_norm": 1.2730036973953247, + "learning_rate": 0.0001, + "loss": 65.692, + "step": 2240 + }, + { + "epoch": 36.292682926829265, + "grad_norm": 1.2936429977416992, + "learning_rate": 0.0001, + "loss": 68.2962, + "step": 2250 + }, + { + "epoch": 36.45528455284553, + "grad_norm": 1.354279637336731, + "learning_rate": 0.0001, + "loss": 70.5507, + "step": 2260 + }, + { + "epoch": 36.61788617886179, + "grad_norm": 1.4165853261947632, + "learning_rate": 0.0001, + "loss": 68.0048, + "step": 2270 + }, + { + "epoch": 36.78048780487805, + "grad_norm": 1.2992445230484009, + "learning_rate": 0.0001, + "loss": 67.7072, + "step": 2280 + }, + { + "epoch": 36.94308943089431, + "grad_norm": 1.5053461790084839, + "learning_rate": 0.0001, + "loss": 66.7587, + "step": 2290 + }, + { + "epoch": 37.09756097560975, + "grad_norm": 1.337390661239624, + "learning_rate": 0.0001, + "loss": 63.786, + "step": 2300 + }, + { + "epoch": 37.260162601626014, + "grad_norm": 1.0407774448394775, + "learning_rate": 0.0001, + "loss": 66.6253, + "step": 2310 + }, + { + "epoch": 37.422764227642276, + "grad_norm": 1.564152479171753, + "learning_rate": 0.0001, + "loss": 68.3606, + "step": 2320 + }, + { + "epoch": 37.58536585365854, + "grad_norm": 1.2028454542160034, + "learning_rate": 0.0001, + "loss": 66.2841, + "step": 2330 + }, + { + "epoch": 37.7479674796748, + "grad_norm": 1.4528603553771973, + "learning_rate": 0.0001, + "loss": 67.4277, + "step": 2340 + }, + { + "epoch": 37.91056910569106, + "grad_norm": 1.4954335689544678, + "learning_rate": 0.0001, + "loss": 68.9954, + "step": 2350 + }, + { + "epoch": 38.0650406504065, + "grad_norm": 1.2820416688919067, + "learning_rate": 0.0001, + "loss": 65.2449, + "step": 2360 + }, + { + "epoch": 38.22764227642276, + "grad_norm": 1.2392579317092896, + "learning_rate": 0.0001, + "loss": 67.5006, + "step": 2370 + }, + { + "epoch": 38.390243902439025, + "grad_norm": 1.2135238647460938, + "learning_rate": 0.0001, + "loss": 69.2214, + "step": 2380 + }, + { + "epoch": 38.552845528455286, + "grad_norm": 1.5964744091033936, + "learning_rate": 0.0001, + "loss": 67.934, + "step": 2390 + }, + { + "epoch": 38.71544715447155, + "grad_norm": 1.2841702699661255, + "learning_rate": 0.0001, + "loss": 67.0848, + "step": 2400 + }, + { + "epoch": 38.8780487804878, + "grad_norm": 1.8026081323623657, + "learning_rate": 0.0001, + "loss": 67.5358, + "step": 2410 + }, + { + "epoch": 39.03252032520325, + "grad_norm": 1.4178807735443115, + "learning_rate": 0.0001, + "loss": 64.771, + "step": 2420 + }, + { + "epoch": 39.19512195121951, + "grad_norm": 1.3671854734420776, + "learning_rate": 0.0001, + "loss": 68.0531, + "step": 2430 + }, + { + "epoch": 39.357723577235774, + "grad_norm": 1.43596613407135, + "learning_rate": 0.0001, + "loss": 67.7735, + "step": 2440 + }, + { + "epoch": 39.520325203252035, + "grad_norm": 1.464250922203064, + "learning_rate": 0.0001, + "loss": 67.8331, + "step": 2450 + }, + { + "epoch": 39.68292682926829, + "grad_norm": 3.101461410522461, + "learning_rate": 0.0001, + "loss": 65.8956, + "step": 2460 + }, + { + "epoch": 39.84552845528455, + "grad_norm": 1.8332289457321167, + "learning_rate": 0.0001, + "loss": 66.8218, + "step": 2470 + }, + { + "epoch": 40.0, + "grad_norm": 1.2660984992980957, + "learning_rate": 0.0001, + "loss": 63.9969, + "step": 2480 + }, + { + "epoch": 40.16260162601626, + "grad_norm": 2.0309898853302, + "learning_rate": 0.0001, + "loss": 67.3538, + "step": 2490 + }, + { + "epoch": 40.32520325203252, + "grad_norm": 1.207000494003296, + "learning_rate": 0.0001, + "loss": 68.0898, + "step": 2500 + }, + { + "epoch": 40.48780487804878, + "grad_norm": 1.3609018325805664, + "learning_rate": 0.0001, + "loss": 67.0381, + "step": 2510 + }, + { + "epoch": 40.65040650406504, + "grad_norm": 1.310826301574707, + "learning_rate": 0.0001, + "loss": 69.0267, + "step": 2520 + }, + { + "epoch": 40.8130081300813, + "grad_norm": 1.4150960445404053, + "learning_rate": 0.0001, + "loss": 69.1934, + "step": 2530 + }, + { + "epoch": 40.97560975609756, + "grad_norm": 1.5414319038391113, + "learning_rate": 0.0001, + "loss": 68.0604, + "step": 2540 + }, + { + "epoch": 41.13008130081301, + "grad_norm": 2.3246140480041504, + "learning_rate": 0.0001, + "loss": 61.5698, + "step": 2550 + }, + { + "epoch": 41.292682926829265, + "grad_norm": 1.206296443939209, + "learning_rate": 0.0001, + "loss": 64.9898, + "step": 2560 + }, + { + "epoch": 41.45528455284553, + "grad_norm": 1.6080626249313354, + "learning_rate": 0.0001, + "loss": 68.2685, + "step": 2570 + }, + { + "epoch": 41.61788617886179, + "grad_norm": 1.2404112815856934, + "learning_rate": 0.0001, + "loss": 67.8457, + "step": 2580 + }, + { + "epoch": 41.78048780487805, + "grad_norm": 1.2304514646530151, + "learning_rate": 0.0001, + "loss": 67.9052, + "step": 2590 + }, + { + "epoch": 41.94308943089431, + "grad_norm": 1.5806187391281128, + "learning_rate": 0.0001, + "loss": 66.0663, + "step": 2600 + }, + { + "epoch": 42.09756097560975, + "grad_norm": 1.2564151287078857, + "learning_rate": 0.0001, + "loss": 65.7763, + "step": 2610 + }, + { + "epoch": 42.260162601626014, + "grad_norm": 1.4972114562988281, + "learning_rate": 0.0001, + "loss": 66.4269, + "step": 2620 + }, + { + "epoch": 42.422764227642276, + "grad_norm": 1.2913843393325806, + "learning_rate": 0.0001, + "loss": 67.7554, + "step": 2630 + }, + { + "epoch": 42.58536585365854, + "grad_norm": 1.3743863105773926, + "learning_rate": 0.0001, + "loss": 67.2627, + "step": 2640 + }, + { + "epoch": 42.7479674796748, + "grad_norm": 1.1267883777618408, + "learning_rate": 0.0001, + "loss": 67.4597, + "step": 2650 + }, + { + "epoch": 42.91056910569106, + "grad_norm": 2.0194661617279053, + "learning_rate": 0.0001, + "loss": 66.9295, + "step": 2660 + }, + { + "epoch": 43.0650406504065, + "grad_norm": 1.273779273033142, + "learning_rate": 0.0001, + "loss": 60.7906, + "step": 2670 + }, + { + "epoch": 43.22764227642276, + "grad_norm": 1.1498609781265259, + "learning_rate": 0.0001, + "loss": 68.0405, + "step": 2680 + }, + { + "epoch": 43.390243902439025, + "grad_norm": 1.4689130783081055, + "learning_rate": 0.0001, + "loss": 66.2358, + "step": 2690 + }, + { + "epoch": 43.552845528455286, + "grad_norm": 1.4495564699172974, + "learning_rate": 0.0001, + "loss": 67.4267, + "step": 2700 + }, + { + "epoch": 43.71544715447155, + "grad_norm": 2.5769612789154053, + "learning_rate": 0.0001, + "loss": 66.953, + "step": 2710 + }, + { + "epoch": 43.8780487804878, + "grad_norm": 2.1684205532073975, + "learning_rate": 0.0001, + "loss": 65.6393, + "step": 2720 + }, + { + "epoch": 44.03252032520325, + "grad_norm": 1.4968472719192505, + "learning_rate": 0.0001, + "loss": 60.8256, + "step": 2730 + }, + { + "epoch": 44.19512195121951, + "grad_norm": 1.1874353885650635, + "learning_rate": 0.0001, + "loss": 66.818, + "step": 2740 + }, + { + "epoch": 44.357723577235774, + "grad_norm": 2.271235227584839, + "learning_rate": 0.0001, + "loss": 66.5048, + "step": 2750 + }, + { + "epoch": 44.520325203252035, + "grad_norm": 1.20489501953125, + "learning_rate": 0.0001, + "loss": 67.0487, + "step": 2760 + }, + { + "epoch": 44.68292682926829, + "grad_norm": 1.6273691654205322, + "learning_rate": 0.0001, + "loss": 66.2228, + "step": 2770 + }, + { + "epoch": 44.84552845528455, + "grad_norm": 1.4935433864593506, + "learning_rate": 0.0001, + "loss": 67.2039, + "step": 2780 + }, + { + "epoch": 45.0, + "grad_norm": 1.2668659687042236, + "learning_rate": 0.0001, + "loss": 62.7855, + "step": 2790 + }, + { + "epoch": 45.16260162601626, + "grad_norm": 1.6439573764801025, + "learning_rate": 0.0001, + "loss": 67.919, + "step": 2800 + }, + { + "epoch": 45.32520325203252, + "grad_norm": 1.311854600906372, + "learning_rate": 0.0001, + "loss": 65.1797, + "step": 2810 + }, + { + "epoch": 45.48780487804878, + "grad_norm": 1.3596776723861694, + "learning_rate": 0.0001, + "loss": 67.469, + "step": 2820 + }, + { + "epoch": 45.65040650406504, + "grad_norm": 1.5618443489074707, + "learning_rate": 0.0001, + "loss": 66.1808, + "step": 2830 + }, + { + "epoch": 45.8130081300813, + "grad_norm": 1.7551205158233643, + "learning_rate": 0.0001, + "loss": 65.3803, + "step": 2840 + }, + { + "epoch": 45.97560975609756, + "grad_norm": 1.433794379234314, + "learning_rate": 0.0001, + "loss": 67.8314, + "step": 2850 + }, + { + "epoch": 46.13008130081301, + "grad_norm": 1.4634851217269897, + "learning_rate": 0.0001, + "loss": 63.4219, + "step": 2860 + }, + { + "epoch": 46.292682926829265, + "grad_norm": 1.083052396774292, + "learning_rate": 0.0001, + "loss": 68.0719, + "step": 2870 + }, + { + "epoch": 46.45528455284553, + "grad_norm": 1.1784703731536865, + "learning_rate": 0.0001, + "loss": 66.1391, + "step": 2880 + }, + { + "epoch": 46.61788617886179, + "grad_norm": 1.2552465200424194, + "learning_rate": 0.0001, + "loss": 65.6006, + "step": 2890 + }, + { + "epoch": 46.78048780487805, + "grad_norm": 1.4684522151947021, + "learning_rate": 0.0001, + "loss": 66.8314, + "step": 2900 + }, + { + "epoch": 46.94308943089431, + "grad_norm": 1.741514801979065, + "learning_rate": 0.0001, + "loss": 67.5056, + "step": 2910 + }, + { + "epoch": 47.09756097560975, + "grad_norm": 5.696534156799316, + "learning_rate": 0.0001, + "loss": 62.8016, + "step": 2920 + }, + { + "epoch": 47.260162601626014, + "grad_norm": 1.5000969171524048, + "learning_rate": 0.0001, + "loss": 68.0272, + "step": 2930 + }, + { + "epoch": 47.422764227642276, + "grad_norm": 1.1044933795928955, + "learning_rate": 0.0001, + "loss": 67.0905, + "step": 2940 + }, + { + "epoch": 47.58536585365854, + "grad_norm": 0.9933589100837708, + "learning_rate": 0.0001, + "loss": 67.6077, + "step": 2950 + }, + { + "epoch": 47.7479674796748, + "grad_norm": 1.2908599376678467, + "learning_rate": 0.0001, + "loss": 65.869, + "step": 2960 + }, + { + "epoch": 47.91056910569106, + "grad_norm": 1.0470529794692993, + "learning_rate": 0.0001, + "loss": 66.5353, + "step": 2970 + }, + { + "epoch": 48.0650406504065, + "grad_norm": 1.0927976369857788, + "learning_rate": 0.0001, + "loss": 62.7541, + "step": 2980 + }, + { + "epoch": 48.22764227642276, + "grad_norm": 1.0870513916015625, + "learning_rate": 0.0001, + "loss": 64.172, + "step": 2990 + }, + { + "epoch": 48.390243902439025, + "grad_norm": 1.3492090702056885, + "learning_rate": 0.0001, + "loss": 67.6642, + "step": 3000 + }, + { + "epoch": 48.552845528455286, + "grad_norm": 1.4024657011032104, + "learning_rate": 0.0001, + "loss": 65.2722, + "step": 3010 + }, + { + "epoch": 48.71544715447155, + "grad_norm": 1.1165252923965454, + "learning_rate": 0.0001, + "loss": 66.7233, + "step": 3020 + }, + { + "epoch": 48.8780487804878, + "grad_norm": 1.2147928476333618, + "learning_rate": 0.0001, + "loss": 65.7277, + "step": 3030 + }, + { + "epoch": 49.03252032520325, + "grad_norm": 0.925310492515564, + "learning_rate": 0.0001, + "loss": 62.9256, + "step": 3040 + }, + { + "epoch": 49.19512195121951, + "grad_norm": 1.2353442907333374, + "learning_rate": 0.0001, + "loss": 66.1218, + "step": 3050 + }, + { + "epoch": 49.357723577235774, + "grad_norm": 1.2177367210388184, + "learning_rate": 0.0001, + "loss": 66.7978, + "step": 3060 + }, + { + "epoch": 49.520325203252035, + "grad_norm": 1.248044729232788, + "learning_rate": 0.0001, + "loss": 66.2132, + "step": 3070 + }, + { + "epoch": 49.68292682926829, + "grad_norm": 1.366050124168396, + "learning_rate": 0.0001, + "loss": 67.0118, + "step": 3080 + }, + { + "epoch": 49.84552845528455, + "grad_norm": 1.3701151609420776, + "learning_rate": 0.0001, + "loss": 67.6099, + "step": 3090 + }, + { + "epoch": 50.0, + "grad_norm": 1.0420141220092773, + "learning_rate": 0.0001, + "loss": 61.5289, + "step": 3100 + }, + { + "epoch": 50.16260162601626, + "grad_norm": 1.0957746505737305, + "learning_rate": 0.0001, + "loss": 66.7348, + "step": 3110 + }, + { + "epoch": 50.32520325203252, + "grad_norm": 1.108628749847412, + "learning_rate": 0.0001, + "loss": 66.6199, + "step": 3120 + }, + { + "epoch": 50.48780487804878, + "grad_norm": 1.3680156469345093, + "learning_rate": 0.0001, + "loss": 67.0825, + "step": 3130 + }, + { + "epoch": 50.65040650406504, + "grad_norm": 1.169851303100586, + "learning_rate": 0.0001, + "loss": 65.4901, + "step": 3140 + }, + { + "epoch": 50.8130081300813, + "grad_norm": 2.1822028160095215, + "learning_rate": 0.0001, + "loss": 65.6841, + "step": 3150 + }, + { + "epoch": 50.97560975609756, + "grad_norm": 1.053402304649353, + "learning_rate": 0.0001, + "loss": 66.3119, + "step": 3160 + }, + { + "epoch": 51.13008130081301, + "grad_norm": 1.0627789497375488, + "learning_rate": 0.0001, + "loss": 61.3642, + "step": 3170 + }, + { + "epoch": 51.292682926829265, + "grad_norm": 1.926291823387146, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 3180 + }, + { + "epoch": 51.45528455284553, + "grad_norm": 1.5791983604431152, + "learning_rate": 0.0001, + "loss": 63.913, + "step": 3190 + }, + { + "epoch": 51.61788617886179, + "grad_norm": 1.210265040397644, + "learning_rate": 0.0001, + "loss": 64.5261, + "step": 3200 + }, + { + "epoch": 51.78048780487805, + "grad_norm": 1.0720808506011963, + "learning_rate": 0.0001, + "loss": 67.0194, + "step": 3210 + }, + { + "epoch": 51.94308943089431, + "grad_norm": 1.94563889503479, + "learning_rate": 0.0001, + "loss": 68.4662, + "step": 3220 + }, + { + "epoch": 52.09756097560975, + "grad_norm": 1.598980188369751, + "learning_rate": 0.0001, + "loss": 62.4946, + "step": 3230 + }, + { + "epoch": 52.260162601626014, + "grad_norm": 1.0564576387405396, + "learning_rate": 0.0001, + "loss": 65.7912, + "step": 3240 + }, + { + "epoch": 52.422764227642276, + "grad_norm": 1.2189645767211914, + "learning_rate": 0.0001, + "loss": 65.5772, + "step": 3250 + }, + { + "epoch": 52.58536585365854, + "grad_norm": 1.1357049942016602, + "learning_rate": 0.0001, + "loss": 65.1559, + "step": 3260 + }, + { + "epoch": 52.7479674796748, + "grad_norm": 1.316867709159851, + "learning_rate": 0.0001, + "loss": 64.7897, + "step": 3270 + }, + { + "epoch": 52.91056910569106, + "grad_norm": 1.3870197534561157, + "learning_rate": 0.0001, + "loss": 65.5195, + "step": 3280 + }, + { + "epoch": 53.0650406504065, + "grad_norm": 1.2467927932739258, + "learning_rate": 0.0001, + "loss": 61.3392, + "step": 3290 + }, + { + "epoch": 53.22764227642276, + "grad_norm": 1.1661109924316406, + "learning_rate": 0.0001, + "loss": 66.0117, + "step": 3300 + }, + { + "epoch": 53.390243902439025, + "grad_norm": 1.3509628772735596, + "learning_rate": 0.0001, + "loss": 64.7228, + "step": 3310 + }, + { + "epoch": 53.552845528455286, + "grad_norm": 1.1093642711639404, + "learning_rate": 0.0001, + "loss": 65.2534, + "step": 3320 + }, + { + "epoch": 53.71544715447155, + "grad_norm": 1.1801207065582275, + "learning_rate": 0.0001, + "loss": 65.7097, + "step": 3330 + }, + { + "epoch": 53.8780487804878, + "grad_norm": 1.175884485244751, + "learning_rate": 0.0001, + "loss": 64.4368, + "step": 3340 + }, + { + "epoch": 54.03252032520325, + "grad_norm": 1.1727889776229858, + "learning_rate": 0.0001, + "loss": 62.9373, + "step": 3350 + }, + { + "epoch": 54.19512195121951, + "grad_norm": 1.5330945253372192, + "learning_rate": 0.0001, + "loss": 66.7763, + "step": 3360 + }, + { + "epoch": 54.357723577235774, + "grad_norm": 1.2992504835128784, + "learning_rate": 0.0001, + "loss": 65.7854, + "step": 3370 + }, + { + "epoch": 54.520325203252035, + "grad_norm": 1.0185420513153076, + "learning_rate": 0.0001, + "loss": 65.7046, + "step": 3380 + }, + { + "epoch": 54.68292682926829, + "grad_norm": 1.426229476928711, + "learning_rate": 0.0001, + "loss": 64.0286, + "step": 3390 + }, + { + "epoch": 54.84552845528455, + "grad_norm": 1.2600868940353394, + "learning_rate": 0.0001, + "loss": 66.1738, + "step": 3400 + }, + { + "epoch": 55.0, + "grad_norm": 1.917455792427063, + "learning_rate": 0.0001, + "loss": 61.9381, + "step": 3410 + }, + { + "epoch": 55.16260162601626, + "grad_norm": 1.2000917196273804, + "learning_rate": 0.0001, + "loss": 65.5626, + "step": 3420 + }, + { + "epoch": 55.32520325203252, + "grad_norm": 1.362176775932312, + "learning_rate": 0.0001, + "loss": 66.8012, + "step": 3430 + }, + { + "epoch": 55.48780487804878, + "grad_norm": 1.1611402034759521, + "learning_rate": 0.0001, + "loss": 64.5227, + "step": 3440 + }, + { + "epoch": 55.65040650406504, + "grad_norm": 1.2066621780395508, + "learning_rate": 0.0001, + "loss": 63.4351, + "step": 3450 + }, + { + "epoch": 55.8130081300813, + "grad_norm": 1.005340814590454, + "learning_rate": 0.0001, + "loss": 65.9261, + "step": 3460 + }, + { + "epoch": 55.97560975609756, + "grad_norm": 1.0906637907028198, + "learning_rate": 0.0001, + "loss": 64.6635, + "step": 3470 + }, + { + "epoch": 56.13008130081301, + "grad_norm": 1.0608748197555542, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 3480 + }, + { + "epoch": 56.292682926829265, + "grad_norm": 1.025319218635559, + "learning_rate": 0.0001, + "loss": 64.7783, + "step": 3490 + }, + { + "epoch": 56.45528455284553, + "grad_norm": 1.1596784591674805, + "learning_rate": 0.0001, + "loss": 66.4629, + "step": 3500 + }, + { + "epoch": 56.61788617886179, + "grad_norm": 0.9318640232086182, + "learning_rate": 0.0001, + "loss": 65.5533, + "step": 3510 + }, + { + "epoch": 56.78048780487805, + "grad_norm": 1.0977418422698975, + "learning_rate": 0.0001, + "loss": 63.647, + "step": 3520 + }, + { + "epoch": 56.94308943089431, + "grad_norm": 1.3912436962127686, + "learning_rate": 0.0001, + "loss": 62.8273, + "step": 3530 + }, + { + "epoch": 57.09756097560975, + "grad_norm": 1.0921591520309448, + "learning_rate": 0.0001, + "loss": 61.5968, + "step": 3540 + }, + { + "epoch": 57.260162601626014, + "grad_norm": 1.17819344997406, + "learning_rate": 0.0001, + "loss": 64.9381, + "step": 3550 + }, + { + "epoch": 57.422764227642276, + "grad_norm": 1.188829779624939, + "learning_rate": 0.0001, + "loss": 64.6498, + "step": 3560 + }, + { + "epoch": 57.58536585365854, + "grad_norm": 3.664839506149292, + "learning_rate": 0.0001, + "loss": 65.3834, + "step": 3570 + }, + { + "epoch": 57.7479674796748, + "grad_norm": 1.57958984375, + "learning_rate": 0.0001, + "loss": 63.8808, + "step": 3580 + }, + { + "epoch": 57.91056910569106, + "grad_norm": 1.1837552785873413, + "learning_rate": 0.0001, + "loss": 64.1545, + "step": 3590 + }, + { + "epoch": 58.0650406504065, + "grad_norm": 1.5986013412475586, + "learning_rate": 0.0001, + "loss": 60.5742, + "step": 3600 + }, + { + "epoch": 58.22764227642276, + "grad_norm": 1.1551810503005981, + "learning_rate": 0.0001, + "loss": 66.2378, + "step": 3610 + }, + { + "epoch": 58.390243902439025, + "grad_norm": 1.0869863033294678, + "learning_rate": 0.0001, + "loss": 65.3034, + "step": 3620 + }, + { + "epoch": 58.552845528455286, + "grad_norm": 1.2263212203979492, + "learning_rate": 0.0001, + "loss": 63.976, + "step": 3630 + }, + { + "epoch": 58.71544715447155, + "grad_norm": 1.4410817623138428, + "learning_rate": 0.0001, + "loss": 63.4778, + "step": 3640 + }, + { + "epoch": 58.8780487804878, + "grad_norm": 1.9633759260177612, + "learning_rate": 0.0001, + "loss": 65.5946, + "step": 3650 + }, + { + "epoch": 59.03252032520325, + "grad_norm": 1.1729766130447388, + "learning_rate": 0.0001, + "loss": 62.2171, + "step": 3660 + }, + { + "epoch": 59.19512195121951, + "grad_norm": 1.0049556493759155, + "learning_rate": 0.0001, + "loss": 65.1854, + "step": 3670 + }, + { + "epoch": 59.357723577235774, + "grad_norm": 1.1727055311203003, + "learning_rate": 0.0001, + "loss": 65.4839, + "step": 3680 + }, + { + "epoch": 59.520325203252035, + "grad_norm": 1.5394458770751953, + "learning_rate": 0.0001, + "loss": 64.9341, + "step": 3690 + }, + { + "epoch": 59.68292682926829, + "grad_norm": 1.1583774089813232, + "learning_rate": 0.0001, + "loss": 63.9197, + "step": 3700 + }, + { + "epoch": 59.84552845528455, + "grad_norm": 1.0835607051849365, + "learning_rate": 0.0001, + "loss": 64.0494, + "step": 3710 + }, + { + "epoch": 60.0, + "grad_norm": 1.7504246234893799, + "learning_rate": 0.0001, + "loss": 59.4134, + "step": 3720 + }, + { + "epoch": 60.16260162601626, + "grad_norm": 1.1870343685150146, + "learning_rate": 0.0001, + "loss": 63.2158, + "step": 3730 + }, + { + "epoch": 60.32520325203252, + "grad_norm": 1.1983181238174438, + "learning_rate": 0.0001, + "loss": 64.7144, + "step": 3740 + }, + { + "epoch": 60.48780487804878, + "grad_norm": 1.3014581203460693, + "learning_rate": 0.0001, + "loss": 64.6219, + "step": 3750 + }, + { + "epoch": 60.65040650406504, + "grad_norm": 1.4954087734222412, + "learning_rate": 0.0001, + "loss": 63.3785, + "step": 3760 + }, + { + "epoch": 60.8130081300813, + "grad_norm": 1.2236649990081787, + "learning_rate": 0.0001, + "loss": 64.6933, + "step": 3770 + }, + { + "epoch": 60.97560975609756, + "grad_norm": 1.0796610116958618, + "learning_rate": 0.0001, + "loss": 63.2704, + "step": 3780 + }, + { + "epoch": 61.13008130081301, + "grad_norm": 0.9969968795776367, + "learning_rate": 0.0001, + "loss": 62.1838, + "step": 3790 + }, + { + "epoch": 61.292682926829265, + "grad_norm": 1.1624361276626587, + "learning_rate": 0.0001, + "loss": 63.6249, + "step": 3800 + }, + { + "epoch": 61.45528455284553, + "grad_norm": 1.3849223852157593, + "learning_rate": 0.0001, + "loss": 64.5321, + "step": 3810 + }, + { + "epoch": 61.61788617886179, + "grad_norm": 1.6607458591461182, + "learning_rate": 0.0001, + "loss": 64.651, + "step": 3820 + }, + { + "epoch": 61.78048780487805, + "grad_norm": 1.09684419631958, + "learning_rate": 0.0001, + "loss": 64.1129, + "step": 3830 + }, + { + "epoch": 61.94308943089431, + "grad_norm": 1.406246304512024, + "learning_rate": 0.0001, + "loss": 64.3642, + "step": 3840 + }, + { + "epoch": 62.09756097560975, + "grad_norm": 0.9850211143493652, + "learning_rate": 0.0001, + "loss": 60.8703, + "step": 3850 + }, + { + "epoch": 62.260162601626014, + "grad_norm": 1.001344084739685, + "learning_rate": 0.0001, + "loss": 67.5523, + "step": 3860 + }, + { + "epoch": 62.422764227642276, + "grad_norm": 2.435969591140747, + "learning_rate": 0.0001, + "loss": 62.7363, + "step": 3870 + }, + { + "epoch": 62.58536585365854, + "grad_norm": 1.0675824880599976, + "learning_rate": 0.0001, + "loss": 64.8513, + "step": 3880 + }, + { + "epoch": 62.7479674796748, + "grad_norm": 0.9793754816055298, + "learning_rate": 0.0001, + "loss": 64.9904, + "step": 3890 + }, + { + "epoch": 62.91056910569106, + "grad_norm": 1.2187775373458862, + "learning_rate": 0.0001, + "loss": 63.2821, + "step": 3900 + }, + { + "epoch": 63.0650406504065, + "grad_norm": 1.130772590637207, + "learning_rate": 0.0001, + "loss": 61.5957, + "step": 3910 + }, + { + "epoch": 63.22764227642276, + "grad_norm": 1.0573055744171143, + "learning_rate": 0.0001, + "loss": 63.6337, + "step": 3920 + }, + { + "epoch": 63.390243902439025, + "grad_norm": 2.055490255355835, + "learning_rate": 0.0001, + "loss": 63.9598, + "step": 3930 + }, + { + "epoch": 63.552845528455286, + "grad_norm": 1.0780516862869263, + "learning_rate": 0.0001, + "loss": 62.9303, + "step": 3940 + }, + { + "epoch": 63.71544715447155, + "grad_norm": 1.8136590719223022, + "learning_rate": 0.0001, + "loss": 64.4588, + "step": 3950 + }, + { + "epoch": 63.8780487804878, + "grad_norm": 1.0525016784667969, + "learning_rate": 0.0001, + "loss": 64.192, + "step": 3960 + }, + { + "epoch": 64.03252032520325, + "grad_norm": 1.12627375125885, + "learning_rate": 0.0001, + "loss": 59.1823, + "step": 3970 + }, + { + "epoch": 64.1951219512195, + "grad_norm": 1.2558730840682983, + "learning_rate": 0.0001, + "loss": 65.126, + "step": 3980 + }, + { + "epoch": 64.35772357723577, + "grad_norm": 1.2304726839065552, + "learning_rate": 0.0001, + "loss": 62.0868, + "step": 3990 + }, + { + "epoch": 64.52032520325203, + "grad_norm": 1.4179201126098633, + "learning_rate": 0.0001, + "loss": 62.3114, + "step": 4000 + }, + { + "epoch": 64.6829268292683, + "grad_norm": 1.125758409500122, + "learning_rate": 0.0001, + "loss": 63.4103, + "step": 4010 + }, + { + "epoch": 64.84552845528455, + "grad_norm": 2.321124315261841, + "learning_rate": 0.0001, + "loss": 62.9032, + "step": 4020 + }, + { + "epoch": 65.0, + "grad_norm": 1.4433993101119995, + "learning_rate": 0.0001, + "loss": 61.8766, + "step": 4030 + }, + { + "epoch": 65.16260162601625, + "grad_norm": 1.157878041267395, + "learning_rate": 0.0001, + "loss": 62.4833, + "step": 4040 + }, + { + "epoch": 65.32520325203252, + "grad_norm": 1.4127166271209717, + "learning_rate": 0.0001, + "loss": 64.768, + "step": 4050 + }, + { + "epoch": 65.48780487804878, + "grad_norm": 1.2575284242630005, + "learning_rate": 0.0001, + "loss": 62.9401, + "step": 4060 + }, + { + "epoch": 65.65040650406505, + "grad_norm": 1.0957911014556885, + "learning_rate": 0.0001, + "loss": 64.6061, + "step": 4070 + }, + { + "epoch": 65.8130081300813, + "grad_norm": 1.1773414611816406, + "learning_rate": 0.0001, + "loss": 63.9469, + "step": 4080 + }, + { + "epoch": 65.97560975609755, + "grad_norm": 2.1189942359924316, + "learning_rate": 0.0001, + "loss": 65.1422, + "step": 4090 + }, + { + "epoch": 66.130081300813, + "grad_norm": 1.1064234972000122, + "learning_rate": 0.0001, + "loss": 59.6108, + "step": 4100 + }, + { + "epoch": 66.29268292682927, + "grad_norm": 1.1135871410369873, + "learning_rate": 0.0001, + "loss": 63.4036, + "step": 4110 + }, + { + "epoch": 66.45528455284553, + "grad_norm": 4.167781829833984, + "learning_rate": 0.0001, + "loss": 63.1174, + "step": 4120 + }, + { + "epoch": 66.6178861788618, + "grad_norm": 1.1429376602172852, + "learning_rate": 0.0001, + "loss": 64.4085, + "step": 4130 + }, + { + "epoch": 66.78048780487805, + "grad_norm": 1.3188339471817017, + "learning_rate": 0.0001, + "loss": 62.5997, + "step": 4140 + }, + { + "epoch": 66.9430894308943, + "grad_norm": 1.160335659980774, + "learning_rate": 0.0001, + "loss": 64.2911, + "step": 4150 + }, + { + "epoch": 67.09756097560975, + "grad_norm": 0.9261873960494995, + "learning_rate": 0.0001, + "loss": 61.9036, + "step": 4160 + }, + { + "epoch": 67.26016260162602, + "grad_norm": 1.2254258394241333, + "learning_rate": 0.0001, + "loss": 64.1092, + "step": 4170 + }, + { + "epoch": 67.42276422764228, + "grad_norm": 0.9519097208976746, + "learning_rate": 0.0001, + "loss": 61.6428, + "step": 4180 + }, + { + "epoch": 67.58536585365853, + "grad_norm": 1.2290887832641602, + "learning_rate": 0.0001, + "loss": 64.9842, + "step": 4190 + }, + { + "epoch": 67.7479674796748, + "grad_norm": 1.0995723009109497, + "learning_rate": 0.0001, + "loss": 64.0677, + "step": 4200 + }, + { + "epoch": 67.91056910569105, + "grad_norm": 2.727957010269165, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 4210 + }, + { + "epoch": 68.0650406504065, + "grad_norm": 1.1222411394119263, + "learning_rate": 0.0001, + "loss": 61.0693, + "step": 4220 + }, + { + "epoch": 68.22764227642277, + "grad_norm": 1.159303069114685, + "learning_rate": 0.0001, + "loss": 64.2469, + "step": 4230 + }, + { + "epoch": 68.39024390243902, + "grad_norm": 1.2205990552902222, + "learning_rate": 0.0001, + "loss": 63.8831, + "step": 4240 + }, + { + "epoch": 68.55284552845528, + "grad_norm": 1.2351205348968506, + "learning_rate": 0.0001, + "loss": 63.4294, + "step": 4250 + }, + { + "epoch": 68.71544715447155, + "grad_norm": 1.4483757019042969, + "learning_rate": 0.0001, + "loss": 63.5345, + "step": 4260 + }, + { + "epoch": 68.8780487804878, + "grad_norm": 1.0808624029159546, + "learning_rate": 0.0001, + "loss": 63.6885, + "step": 4270 + }, + { + "epoch": 69.03252032520325, + "grad_norm": 1.7775999307632446, + "learning_rate": 0.0001, + "loss": 60.7266, + "step": 4280 + }, + { + "epoch": 69.1951219512195, + "grad_norm": 1.3214768171310425, + "learning_rate": 0.0001, + "loss": 63.3496, + "step": 4290 + }, + { + "epoch": 69.35772357723577, + "grad_norm": 1.1698002815246582, + "learning_rate": 0.0001, + "loss": 63.5284, + "step": 4300 + }, + { + "epoch": 69.52032520325203, + "grad_norm": 1.2050056457519531, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 4310 + }, + { + "epoch": 69.6829268292683, + "grad_norm": 1.0873243808746338, + "learning_rate": 0.0001, + "loss": 65.1709, + "step": 4320 + }, + { + "epoch": 69.84552845528455, + "grad_norm": 1.14133620262146, + "learning_rate": 0.0001, + "loss": 63.1872, + "step": 4330 + }, + { + "epoch": 70.0, + "grad_norm": 0.9684088230133057, + "learning_rate": 0.0001, + "loss": 60.1486, + "step": 4340 + }, + { + "epoch": 70.16260162601625, + "grad_norm": 1.1206653118133545, + "learning_rate": 0.0001, + "loss": 61.117, + "step": 4350 + }, + { + "epoch": 70.32520325203252, + "grad_norm": 1.2754334211349487, + "learning_rate": 0.0001, + "loss": 62.5409, + "step": 4360 + }, + { + "epoch": 70.48780487804878, + "grad_norm": 1.0154718160629272, + "learning_rate": 0.0001, + "loss": 62.9428, + "step": 4370 + }, + { + "epoch": 70.65040650406505, + "grad_norm": 1.0597270727157593, + "learning_rate": 0.0001, + "loss": 63.079, + "step": 4380 + }, + { + "epoch": 70.8130081300813, + "grad_norm": 1.254758596420288, + "learning_rate": 0.0001, + "loss": 63.3035, + "step": 4390 + }, + { + "epoch": 70.97560975609755, + "grad_norm": 1.0185120105743408, + "learning_rate": 0.0001, + "loss": 63.8922, + "step": 4400 + }, + { + "epoch": 71.130081300813, + "grad_norm": 1.007300853729248, + "learning_rate": 0.0001, + "loss": 58.7735, + "step": 4410 + }, + { + "epoch": 71.29268292682927, + "grad_norm": 1.2012463808059692, + "learning_rate": 0.0001, + "loss": 63.1782, + "step": 4420 + }, + { + "epoch": 71.45528455284553, + "grad_norm": 1.0258845090866089, + "learning_rate": 0.0001, + "loss": 63.2483, + "step": 4430 + }, + { + "epoch": 71.6178861788618, + "grad_norm": 1.0611220598220825, + "learning_rate": 0.0001, + "loss": 63.2247, + "step": 4440 + }, + { + "epoch": 71.78048780487805, + "grad_norm": 1.060114860534668, + "learning_rate": 0.0001, + "loss": 61.8352, + "step": 4450 + }, + { + "epoch": 71.9430894308943, + "grad_norm": 1.0540589094161987, + "learning_rate": 0.0001, + "loss": 62.1156, + "step": 4460 + }, + { + "epoch": 72.09756097560975, + "grad_norm": 1.4923726320266724, + "learning_rate": 0.0001, + "loss": 59.3863, + "step": 4470 + }, + { + "epoch": 72.26016260162602, + "grad_norm": 1.13791024684906, + "learning_rate": 0.0001, + "loss": 62.203, + "step": 4480 + }, + { + "epoch": 72.42276422764228, + "grad_norm": 0.9737065434455872, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 4490 + }, + { + "epoch": 72.58536585365853, + "grad_norm": 1.3980190753936768, + "learning_rate": 0.0001, + "loss": 63.5627, + "step": 4500 + }, + { + "epoch": 72.7479674796748, + "grad_norm": 1.0274118185043335, + "learning_rate": 0.0001, + "loss": 63.6129, + "step": 4510 + }, + { + "epoch": 72.91056910569105, + "grad_norm": 1.3148925304412842, + "learning_rate": 0.0001, + "loss": 63.6506, + "step": 4520 + }, + { + "epoch": 73.0650406504065, + "grad_norm": 1.4182451963424683, + "learning_rate": 0.0001, + "loss": 60.3616, + "step": 4530 + }, + { + "epoch": 73.22764227642277, + "grad_norm": 0.9235166907310486, + "learning_rate": 0.0001, + "loss": 63.6465, + "step": 4540 + }, + { + "epoch": 73.39024390243902, + "grad_norm": 0.9293469786643982, + "learning_rate": 0.0001, + "loss": 61.0984, + "step": 4550 + }, + { + "epoch": 73.55284552845528, + "grad_norm": 1.1075892448425293, + "learning_rate": 0.0001, + "loss": 62.9356, + "step": 4560 + }, + { + "epoch": 73.71544715447155, + "grad_norm": 1.4955459833145142, + "learning_rate": 0.0001, + "loss": 61.3811, + "step": 4570 + }, + { + "epoch": 73.8780487804878, + "grad_norm": 1.220595359802246, + "learning_rate": 0.0001, + "loss": 62.4364, + "step": 4580 + }, + { + "epoch": 74.03252032520325, + "grad_norm": 0.9975704550743103, + "learning_rate": 0.0001, + "loss": 58.5194, + "step": 4590 + }, + { + "epoch": 74.1951219512195, + "grad_norm": 1.0160857439041138, + "learning_rate": 0.0001, + "loss": 64.6989, + "step": 4600 + }, + { + "epoch": 74.35772357723577, + "grad_norm": 1.0052287578582764, + "learning_rate": 0.0001, + "loss": 62.526, + "step": 4610 + }, + { + "epoch": 74.52032520325203, + "grad_norm": 1.015514612197876, + "learning_rate": 0.0001, + "loss": 63.8627, + "step": 4620 + }, + { + "epoch": 74.6829268292683, + "grad_norm": 1.139051914215088, + "learning_rate": 0.0001, + "loss": 62.4525, + "step": 4630 + }, + { + "epoch": 74.84552845528455, + "grad_norm": 1.1727714538574219, + "learning_rate": 0.0001, + "loss": 64.0046, + "step": 4640 + }, + { + "epoch": 75.0, + "grad_norm": 1.1611062288284302, + "learning_rate": 0.0001, + "loss": 57.6443, + "step": 4650 + }, + { + "epoch": 75.16260162601625, + "grad_norm": 0.9893108606338501, + "learning_rate": 0.0001, + "loss": 64.1109, + "step": 4660 + }, + { + "epoch": 75.32520325203252, + "grad_norm": 1.190146803855896, + "learning_rate": 0.0001, + "loss": 62.1988, + "step": 4670 + }, + { + "epoch": 75.48780487804878, + "grad_norm": 1.1648579835891724, + "learning_rate": 0.0001, + "loss": 63.1393, + "step": 4680 + }, + { + "epoch": 75.65040650406505, + "grad_norm": 1.3303166627883911, + "learning_rate": 0.0001, + "loss": 63.7, + "step": 4690 + }, + { + "epoch": 75.8130081300813, + "grad_norm": 1.407772421836853, + "learning_rate": 0.0001, + "loss": 60.6798, + "step": 4700 + }, + { + "epoch": 75.97560975609755, + "grad_norm": 1.039741039276123, + "learning_rate": 0.0001, + "loss": 61.6126, + "step": 4710 + }, + { + "epoch": 76.130081300813, + "grad_norm": 1.013515830039978, + "learning_rate": 0.0001, + "loss": 57.8452, + "step": 4720 + }, + { + "epoch": 76.29268292682927, + "grad_norm": 1.1279789209365845, + "learning_rate": 0.0001, + "loss": 63.0592, + "step": 4730 + }, + { + "epoch": 76.45528455284553, + "grad_norm": 1.158605694770813, + "learning_rate": 0.0001, + "loss": 62.0836, + "step": 4740 + }, + { + "epoch": 76.6178861788618, + "grad_norm": 1.0200021266937256, + "learning_rate": 0.0001, + "loss": 62.1775, + "step": 4750 + }, + { + "epoch": 76.78048780487805, + "grad_norm": 1.216412901878357, + "learning_rate": 0.0001, + "loss": 62.6316, + "step": 4760 + }, + { + "epoch": 76.9430894308943, + "grad_norm": 1.0513523817062378, + "learning_rate": 0.0001, + "loss": 60.1037, + "step": 4770 + }, + { + "epoch": 77.09756097560975, + "grad_norm": 1.10075843334198, + "learning_rate": 0.0001, + "loss": 58.7599, + "step": 4780 + }, + { + "epoch": 77.26016260162602, + "grad_norm": 1.1728724241256714, + "learning_rate": 0.0001, + "loss": 63.3476, + "step": 4790 + }, + { + "epoch": 77.42276422764228, + "grad_norm": 1.3511605262756348, + "learning_rate": 0.0001, + "loss": 60.9885, + "step": 4800 + }, + { + "epoch": 77.58536585365853, + "grad_norm": 0.9733042120933533, + "learning_rate": 0.0001, + "loss": 62.4245, + "step": 4810 + }, + { + "epoch": 77.7479674796748, + "grad_norm": 1.0380535125732422, + "learning_rate": 0.0001, + "loss": 64.0755, + "step": 4820 + }, + { + "epoch": 77.91056910569105, + "grad_norm": 1.1594985723495483, + "learning_rate": 0.0001, + "loss": 62.1099, + "step": 4830 + }, + { + "epoch": 78.0650406504065, + "grad_norm": 1.0392569303512573, + "learning_rate": 0.0001, + "loss": 58.3778, + "step": 4840 + }, + { + "epoch": 78.22764227642277, + "grad_norm": 1.0032405853271484, + "learning_rate": 0.0001, + "loss": 62.1384, + "step": 4850 + }, + { + "epoch": 78.39024390243902, + "grad_norm": 1.0576013326644897, + "learning_rate": 0.0001, + "loss": 63.3308, + "step": 4860 + }, + { + "epoch": 78.55284552845528, + "grad_norm": 1.34328031539917, + "learning_rate": 0.0001, + "loss": 62.5351, + "step": 4870 + }, + { + "epoch": 78.71544715447155, + "grad_norm": 1.1040822267532349, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 4880 + }, + { + "epoch": 78.8780487804878, + "grad_norm": 1.1554768085479736, + "learning_rate": 0.0001, + "loss": 62.7308, + "step": 4890 + }, + { + "epoch": 79.03252032520325, + "grad_norm": 1.1864408254623413, + "learning_rate": 0.0001, + "loss": 60.0741, + "step": 4900 + }, + { + "epoch": 79.1951219512195, + "grad_norm": 0.9181821346282959, + "learning_rate": 0.0001, + "loss": 63.1682, + "step": 4910 + }, + { + "epoch": 79.35772357723577, + "grad_norm": 1.1400946378707886, + "learning_rate": 0.0001, + "loss": 62.0736, + "step": 4920 + }, + { + "epoch": 79.52032520325203, + "grad_norm": 1.245202660560608, + "learning_rate": 0.0001, + "loss": 60.51, + "step": 4930 + }, + { + "epoch": 79.6829268292683, + "grad_norm": 1.033347725868225, + "learning_rate": 0.0001, + "loss": 61.7845, + "step": 4940 + }, + { + "epoch": 79.84552845528455, + "grad_norm": 1.6206163167953491, + "learning_rate": 0.0001, + "loss": 61.5232, + "step": 4950 + }, + { + "epoch": 80.0, + "grad_norm": 1.0924937725067139, + "learning_rate": 0.0001, + "loss": 57.24, + "step": 4960 + }, + { + "epoch": 80.16260162601625, + "grad_norm": 1.075985074043274, + "learning_rate": 0.0001, + "loss": 61.2615, + "step": 4970 + }, + { + "epoch": 80.32520325203252, + "grad_norm": 0.9964771866798401, + "learning_rate": 0.0001, + "loss": 62.287, + "step": 4980 + }, + { + "epoch": 80.48780487804878, + "grad_norm": 1.3707735538482666, + "learning_rate": 0.0001, + "loss": 60.9383, + "step": 4990 + }, + { + "epoch": 80.65040650406505, + "grad_norm": 0.9395170211791992, + "learning_rate": 0.0001, + "loss": 61.2697, + "step": 5000 + }, + { + "epoch": 80.8130081300813, + "grad_norm": 2.0208873748779297, + "learning_rate": 0.0001, + "loss": 61.7219, + "step": 5010 + }, + { + "epoch": 80.97560975609755, + "grad_norm": 1.468702793121338, + "learning_rate": 0.0001, + "loss": 62.6967, + "step": 5020 + }, + { + "epoch": 81.130081300813, + "grad_norm": 1.0525966882705688, + "learning_rate": 0.0001, + "loss": 59.9793, + "step": 5030 + }, + { + "epoch": 81.29268292682927, + "grad_norm": 1.0951132774353027, + "learning_rate": 0.0001, + "loss": 60.9807, + "step": 5040 + }, + { + "epoch": 81.45528455284553, + "grad_norm": 1.3457857370376587, + "learning_rate": 0.0001, + "loss": 61.4137, + "step": 5050 + }, + { + "epoch": 81.6178861788618, + "grad_norm": 1.0895649194717407, + "learning_rate": 0.0001, + "loss": 61.2747, + "step": 5060 + }, + { + "epoch": 81.78048780487805, + "grad_norm": 1.0552371740341187, + "learning_rate": 0.0001, + "loss": 62.0947, + "step": 5070 + }, + { + "epoch": 81.9430894308943, + "grad_norm": 1.039038896560669, + "learning_rate": 0.0001, + "loss": 61.2571, + "step": 5080 + }, + { + "epoch": 82.09756097560975, + "grad_norm": 0.970783531665802, + "learning_rate": 0.0001, + "loss": 57.4157, + "step": 5090 + }, + { + "epoch": 82.26016260162602, + "grad_norm": 1.0276020765304565, + "learning_rate": 0.0001, + "loss": 62.1324, + "step": 5100 + }, + { + "epoch": 82.42276422764228, + "grad_norm": 1.2092448472976685, + "learning_rate": 0.0001, + "loss": 60.6448, + "step": 5110 + }, + { + "epoch": 82.58536585365853, + "grad_norm": 0.9352007508277893, + "learning_rate": 0.0001, + "loss": 61.9299, + "step": 5120 + }, + { + "epoch": 82.7479674796748, + "grad_norm": 1.2678358554840088, + "learning_rate": 0.0001, + "loss": 61.6622, + "step": 5130 + }, + { + "epoch": 82.91056910569105, + "grad_norm": 1.200494647026062, + "learning_rate": 0.0001, + "loss": 62.3169, + "step": 5140 + }, + { + "epoch": 83.0650406504065, + "grad_norm": 1.1284401416778564, + "learning_rate": 0.0001, + "loss": 58.7768, + "step": 5150 + }, + { + "epoch": 83.22764227642277, + "grad_norm": 1.1578079462051392, + "learning_rate": 0.0001, + "loss": 62.8423, + "step": 5160 + }, + { + "epoch": 83.39024390243902, + "grad_norm": 1.3426504135131836, + "learning_rate": 0.0001, + "loss": 61.5332, + "step": 5170 + }, + { + "epoch": 83.55284552845528, + "grad_norm": 0.9397301077842712, + "learning_rate": 0.0001, + "loss": 60.1043, + "step": 5180 + }, + { + "epoch": 83.71544715447155, + "grad_norm": 1.0728143453598022, + "learning_rate": 0.0001, + "loss": 60.5927, + "step": 5190 + }, + { + "epoch": 83.8780487804878, + "grad_norm": 1.2620211839675903, + "learning_rate": 0.0001, + "loss": 60.9224, + "step": 5200 + }, + { + "epoch": 84.03252032520325, + "grad_norm": 0.8232231140136719, + "learning_rate": 0.0001, + "loss": 58.8128, + "step": 5210 + }, + { + "epoch": 84.1951219512195, + "grad_norm": 1.0222564935684204, + "learning_rate": 0.0001, + "loss": 59.6995, + "step": 5220 + }, + { + "epoch": 84.35772357723577, + "grad_norm": 0.9078415632247925, + "learning_rate": 0.0001, + "loss": 61.1521, + "step": 5230 + }, + { + "epoch": 84.52032520325203, + "grad_norm": 1.0269399881362915, + "learning_rate": 0.0001, + "loss": 61.0136, + "step": 5240 + }, + { + "epoch": 84.6829268292683, + "grad_norm": 1.2108154296875, + "learning_rate": 0.0001, + "loss": 61.3564, + "step": 5250 + }, + { + "epoch": 84.84552845528455, + "grad_norm": 1.1170622110366821, + "learning_rate": 0.0001, + "loss": 61.7047, + "step": 5260 + }, + { + "epoch": 85.0, + "grad_norm": 1.467298150062561, + "learning_rate": 0.0001, + "loss": 58.1667, + "step": 5270 + }, + { + "epoch": 85.16260162601625, + "grad_norm": 0.908743143081665, + "learning_rate": 0.0001, + "loss": 61.5694, + "step": 5280 + }, + { + "epoch": 85.32520325203252, + "grad_norm": 0.9148766398429871, + "learning_rate": 0.0001, + "loss": 61.5325, + "step": 5290 + }, + { + "epoch": 85.48780487804878, + "grad_norm": 0.940021276473999, + "learning_rate": 0.0001, + "loss": 62.8807, + "step": 5300 + }, + { + "epoch": 85.65040650406505, + "grad_norm": 1.1239454746246338, + "learning_rate": 0.0001, + "loss": 62.361, + "step": 5310 + }, + { + "epoch": 85.8130081300813, + "grad_norm": 0.840437114238739, + "learning_rate": 0.0001, + "loss": 61.9057, + "step": 5320 + }, + { + "epoch": 85.97560975609755, + "grad_norm": 0.9764263033866882, + "learning_rate": 0.0001, + "loss": 61.0075, + "step": 5330 + }, + { + "epoch": 86.130081300813, + "grad_norm": 0.998540997505188, + "learning_rate": 0.0001, + "loss": 57.7146, + "step": 5340 + }, + { + "epoch": 86.29268292682927, + "grad_norm": 1.121551513671875, + "learning_rate": 0.0001, + "loss": 62.2278, + "step": 5350 + }, + { + "epoch": 86.45528455284553, + "grad_norm": 1.101842999458313, + "learning_rate": 0.0001, + "loss": 60.8837, + "step": 5360 + }, + { + "epoch": 86.6178861788618, + "grad_norm": 1.1724908351898193, + "learning_rate": 0.0001, + "loss": 61.6788, + "step": 5370 + }, + { + "epoch": 86.78048780487805, + "grad_norm": 1.3108958005905151, + "learning_rate": 0.0001, + "loss": 60.1461, + "step": 5380 + }, + { + "epoch": 86.9430894308943, + "grad_norm": 1.482298731803894, + "learning_rate": 0.0001, + "loss": 61.0457, + "step": 5390 + }, + { + "epoch": 87.09756097560975, + "grad_norm": 1.1471034288406372, + "learning_rate": 0.0001, + "loss": 57.6835, + "step": 5400 + }, + { + "epoch": 87.26016260162602, + "grad_norm": 0.9349374175071716, + "learning_rate": 0.0001, + "loss": 62.7018, + "step": 5410 + }, + { + "epoch": 87.42276422764228, + "grad_norm": 0.9917856454849243, + "learning_rate": 0.0001, + "loss": 60.2201, + "step": 5420 + }, + { + "epoch": 87.58536585365853, + "grad_norm": 1.0112310647964478, + "learning_rate": 0.0001, + "loss": 62.1185, + "step": 5430 + }, + { + "epoch": 87.7479674796748, + "grad_norm": 1.288894534111023, + "learning_rate": 0.0001, + "loss": 61.4348, + "step": 5440 + }, + { + "epoch": 87.91056910569105, + "grad_norm": 0.8577322363853455, + "learning_rate": 0.0001, + "loss": 60.7867, + "step": 5450 + }, + { + "epoch": 88.0650406504065, + "grad_norm": 0.8162860870361328, + "learning_rate": 0.0001, + "loss": 59.5224, + "step": 5460 + }, + { + "epoch": 88.22764227642277, + "grad_norm": 0.9491421580314636, + "learning_rate": 0.0001, + "loss": 60.4936, + "step": 5470 + }, + { + "epoch": 88.39024390243902, + "grad_norm": 4.164788246154785, + "learning_rate": 0.0001, + "loss": 61.8397, + "step": 5480 + }, + { + "epoch": 88.55284552845528, + "grad_norm": 0.9954016804695129, + "learning_rate": 0.0001, + "loss": 61.286, + "step": 5490 + }, + { + "epoch": 88.71544715447155, + "grad_norm": 1.4991120100021362, + "learning_rate": 0.0001, + "loss": 60.6157, + "step": 5500 + }, + { + "epoch": 88.8780487804878, + "grad_norm": 1.3678480386734009, + "learning_rate": 0.0001, + "loss": 62.5187, + "step": 5510 + }, + { + "epoch": 89.03252032520325, + "grad_norm": 1.057661771774292, + "learning_rate": 0.0001, + "loss": 57.5134, + "step": 5520 + }, + { + "epoch": 89.1951219512195, + "grad_norm": 0.8331280946731567, + "learning_rate": 0.0001, + "loss": 62.007, + "step": 5530 + }, + { + "epoch": 89.35772357723577, + "grad_norm": 0.9842398762702942, + "learning_rate": 0.0001, + "loss": 60.6178, + "step": 5540 + }, + { + "epoch": 89.52032520325203, + "grad_norm": 1.0510483980178833, + "learning_rate": 0.0001, + "loss": 59.6382, + "step": 5550 + }, + { + "epoch": 89.6829268292683, + "grad_norm": 1.013115406036377, + "learning_rate": 0.0001, + "loss": 61.015, + "step": 5560 + }, + { + "epoch": 89.84552845528455, + "grad_norm": 1.1595944166183472, + "learning_rate": 0.0001, + "loss": 62.5374, + "step": 5570 + }, + { + "epoch": 90.0, + "grad_norm": 1.1506270170211792, + "learning_rate": 0.0001, + "loss": 61.0063, + "step": 5580 + }, + { + "epoch": 90.16260162601625, + "grad_norm": 0.9940153360366821, + "learning_rate": 0.0001, + "loss": 62.0944, + "step": 5590 + }, + { + "epoch": 90.32520325203252, + "grad_norm": 1.1302493810653687, + "learning_rate": 0.0001, + "loss": 61.616, + "step": 5600 + }, + { + "epoch": 90.48780487804878, + "grad_norm": 1.2370656728744507, + "learning_rate": 0.0001, + "loss": 61.291, + "step": 5610 + }, + { + "epoch": 90.65040650406505, + "grad_norm": 1.228763461112976, + "learning_rate": 0.0001, + "loss": 60.0629, + "step": 5620 + }, + { + "epoch": 90.8130081300813, + "grad_norm": 1.2230069637298584, + "learning_rate": 0.0001, + "loss": 61.5391, + "step": 5630 + }, + { + "epoch": 90.97560975609755, + "grad_norm": 0.9849008917808533, + "learning_rate": 0.0001, + "loss": 63.1633, + "step": 5640 + }, + { + "epoch": 91.130081300813, + "grad_norm": 1.408632755279541, + "learning_rate": 0.0001, + "loss": 58.4195, + "step": 5650 + }, + { + "epoch": 91.29268292682927, + "grad_norm": 0.929027795791626, + "learning_rate": 0.0001, + "loss": 60.9862, + "step": 5660 + }, + { + "epoch": 91.45528455284553, + "grad_norm": 0.9228377342224121, + "learning_rate": 0.0001, + "loss": 60.8337, + "step": 5670 + }, + { + "epoch": 91.6178861788618, + "grad_norm": 1.0354220867156982, + "learning_rate": 0.0001, + "loss": 59.4667, + "step": 5680 + }, + { + "epoch": 91.78048780487805, + "grad_norm": 1.0338927507400513, + "learning_rate": 0.0001, + "loss": 62.0064, + "step": 5690 + }, + { + "epoch": 91.9430894308943, + "grad_norm": 1.101967215538025, + "learning_rate": 0.0001, + "loss": 61.6317, + "step": 5700 + }, + { + "epoch": 92.09756097560975, + "grad_norm": 1.370689034461975, + "learning_rate": 0.0001, + "loss": 57.2157, + "step": 5710 + }, + { + "epoch": 92.26016260162602, + "grad_norm": 0.9904115796089172, + "learning_rate": 0.0001, + "loss": 61.2786, + "step": 5720 + }, + { + "epoch": 92.42276422764228, + "grad_norm": 1.0545448064804077, + "learning_rate": 0.0001, + "loss": 63.2384, + "step": 5730 + }, + { + "epoch": 92.58536585365853, + "grad_norm": 1.0657731294631958, + "learning_rate": 0.0001, + "loss": 60.582, + "step": 5740 + }, + { + "epoch": 92.7479674796748, + "grad_norm": 0.9333183765411377, + "learning_rate": 0.0001, + "loss": 62.4627, + "step": 5750 + }, + { + "epoch": 92.91056910569105, + "grad_norm": 0.9810492396354675, + "learning_rate": 0.0001, + "loss": 60.0055, + "step": 5760 + }, + { + "epoch": 93.0650406504065, + "grad_norm": 1.6973884105682373, + "learning_rate": 0.0001, + "loss": 57.5381, + "step": 5770 + }, + { + "epoch": 93.22764227642277, + "grad_norm": 1.0075498819351196, + "learning_rate": 0.0001, + "loss": 61.2272, + "step": 5780 + }, + { + "epoch": 93.39024390243902, + "grad_norm": 0.980633556842804, + "learning_rate": 0.0001, + "loss": 60.4794, + "step": 5790 + }, + { + "epoch": 93.55284552845528, + "grad_norm": 1.128214955329895, + "learning_rate": 0.0001, + "loss": 60.4825, + "step": 5800 + }, + { + "epoch": 93.71544715447155, + "grad_norm": 1.302827000617981, + "learning_rate": 0.0001, + "loss": 62.4397, + "step": 5810 + }, + { + "epoch": 93.8780487804878, + "grad_norm": 0.826600193977356, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 5820 + }, + { + "epoch": 94.03252032520325, + "grad_norm": 11.418835639953613, + "learning_rate": 0.0001, + "loss": 58.5383, + "step": 5830 + }, + { + "epoch": 94.1951219512195, + "grad_norm": 1.0686477422714233, + "learning_rate": 0.0001, + "loss": 60.5696, + "step": 5840 + }, + { + "epoch": 94.35772357723577, + "grad_norm": 1.1232597827911377, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 5850 + }, + { + "epoch": 94.52032520325203, + "grad_norm": 1.1051944494247437, + "learning_rate": 0.0001, + "loss": 60.5387, + "step": 5860 + }, + { + "epoch": 94.6829268292683, + "grad_norm": 0.9954739809036255, + "learning_rate": 0.0001, + "loss": 61.3899, + "step": 5870 + }, + { + "epoch": 94.84552845528455, + "grad_norm": 1.9687999486923218, + "learning_rate": 0.0001, + "loss": 60.7711, + "step": 5880 + }, + { + "epoch": 95.0, + "grad_norm": 1.3172541856765747, + "learning_rate": 0.0001, + "loss": 58.5394, + "step": 5890 + }, + { + "epoch": 95.16260162601625, + "grad_norm": 1.2251427173614502, + "learning_rate": 0.0001, + "loss": 61.8319, + "step": 5900 + }, + { + "epoch": 95.32520325203252, + "grad_norm": 1.1701544523239136, + "learning_rate": 0.0001, + "loss": 60.1928, + "step": 5910 + }, + { + "epoch": 95.48780487804878, + "grad_norm": 1.209375262260437, + "learning_rate": 0.0001, + "loss": 61.1347, + "step": 5920 + }, + { + "epoch": 95.65040650406505, + "grad_norm": 0.9307044148445129, + "learning_rate": 0.0001, + "loss": 62.6676, + "step": 5930 + }, + { + "epoch": 95.8130081300813, + "grad_norm": 0.9679144024848938, + "learning_rate": 0.0001, + "loss": 60.7623, + "step": 5940 + }, + { + "epoch": 95.97560975609755, + "grad_norm": 1.398711085319519, + "learning_rate": 0.0001, + "loss": 59.0407, + "step": 5950 + }, + { + "epoch": 96.130081300813, + "grad_norm": 1.9118247032165527, + "learning_rate": 0.0001, + "loss": 59.9417, + "step": 5960 + }, + { + "epoch": 96.29268292682927, + "grad_norm": 1.036737084388733, + "learning_rate": 0.0001, + "loss": 60.6891, + "step": 5970 + }, + { + "epoch": 96.45528455284553, + "grad_norm": 1.5916132926940918, + "learning_rate": 0.0001, + "loss": 59.44, + "step": 5980 + }, + { + "epoch": 96.6178861788618, + "grad_norm": 1.526591181755066, + "learning_rate": 0.0001, + "loss": 58.9081, + "step": 5990 + }, + { + "epoch": 96.78048780487805, + "grad_norm": 0.9556435942649841, + "learning_rate": 0.0001, + "loss": 61.5743, + "step": 6000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 323, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.288231035957097e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c7f229ebe1034caaf3e92006c8c7528c21af93d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517586849d044483051c7a59cec7c6ce389951878a7357ef837edb1cbc9983c8 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-6000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/latest b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/latest new file mode 100644 index 0000000000000000000000000000000000000000..6c558c76ac9e7515267f79d24d9fde4a7f8688f1 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/latest @@ -0,0 +1 @@ +global_step8000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d3b0626db9522e2f08ab854a3fe250428cadf6ec --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea32a570f8ced8253cf9e9acaf78d21a48240a85d5cfab93e2d8bf6f00fdf03f +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..b25226e975ec332830c7ba55d3d8fa7225bd1143 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1e1cc56766d2b11775cce4ef961cb8d0f0008b86269f37b446e9ec87cc7cb2e +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..f87c1ca7359e4e8d69eaff9e5337aeee2f9b7b48 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3e04d50e32c203b34a5ef3e38eb285d2145e5c41cb708998ff0c74a957c1b23 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..298e4eb3fdacff9ea7be863963e8e0850d051e82 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712bef7cd40fc12b9ed3977e9a3d8337b34db5789daaf4d61b0c02249360b10b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..c983b316b1edd08336a73f0e10942d300ddc4dff --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02d50a8cb5fa83ff0dbe4a10cdf693947cf37e8714d09ebf8dc9b75172880628 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..7f8cff060896a1c388cabce814a5185c4089d5d6 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6af573c2ff3e2ae7951bdeeeec6ab5fd6527d4bb096deb33f507f2ef1ccc1586 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..c4a4bf1e4a1a8817e98cabd7889c05ad924137c9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53eaef445feb261b1c8d459bccdfdf156d14010b669c3912575aaa865b440df2 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..e30859596c59f001c57e62864ff87f641a2a4d49 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cca072e7cc01ee55f73e3cf6a706fd7f3503b0fbbffacc7f8dd45fafefcaf25 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..78a028930ead792ecad0a7139fa0e79d6ffa59f0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c51971f012cba1a99ed2aecafcfeaca9343747c8f56595a5bb6d56cc4bcc945a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8fb4ef8eda5769247453e7f4c1cbe7f89242b5c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f130956eca2bbfbb04376b671db10d86b648fabcae3263ea3116e90a644e7ab1 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0e0757f06618958a1b8428624991b3464fd0dfc8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/trainer_state.json @@ -0,0 +1,5634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 129.03252032520325, + "eval_steps": 2000, + "global_step": 8000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.16260162601626016, + "grad_norm": 23.72986602783203, + "learning_rate": 0.0001, + "loss": 271.6598, + "step": 10 + }, + { + "epoch": 0.3252032520325203, + "grad_norm": 18.99613380432129, + "learning_rate": 0.0001, + "loss": 233.029, + "step": 20 + }, + { + "epoch": 0.4878048780487805, + "grad_norm": 3.9815304279327393, + "learning_rate": 0.0001, + "loss": 231.0776, + "step": 30 + }, + { + "epoch": 0.6504065040650406, + "grad_norm": 4.641753196716309, + "learning_rate": 0.0001, + "loss": 228.6894, + "step": 40 + }, + { + "epoch": 0.8130081300813008, + "grad_norm": 3.000852108001709, + "learning_rate": 0.0001, + "loss": 228.4694, + "step": 50 + }, + { + "epoch": 0.975609756097561, + "grad_norm": 12.213018417358398, + "learning_rate": 0.0001, + "loss": 226.4258, + "step": 60 + }, + { + "epoch": 1.1300813008130082, + "grad_norm": 2.535187005996704, + "learning_rate": 0.0001, + "loss": 209.9772, + "step": 70 + }, + { + "epoch": 1.2926829268292683, + "grad_norm": 2.0984582901000977, + "learning_rate": 0.0001, + "loss": 217.2421, + "step": 80 + }, + { + "epoch": 1.4552845528455285, + "grad_norm": 3.4802305698394775, + "learning_rate": 0.0001, + "loss": 210.9241, + "step": 90 + }, + { + "epoch": 1.6178861788617886, + "grad_norm": 3.813626527786255, + "learning_rate": 0.0001, + "loss": 205.7566, + "step": 100 + }, + { + "epoch": 1.7804878048780488, + "grad_norm": 2.6378512382507324, + "learning_rate": 0.0001, + "loss": 201.8789, + "step": 110 + }, + { + "epoch": 1.943089430894309, + "grad_norm": 2.7502875328063965, + "learning_rate": 0.0001, + "loss": 196.373, + "step": 120 + }, + { + "epoch": 2.097560975609756, + "grad_norm": 2.659212112426758, + "learning_rate": 0.0001, + "loss": 183.1456, + "step": 130 + }, + { + "epoch": 2.2601626016260163, + "grad_norm": 2.513044595718384, + "learning_rate": 0.0001, + "loss": 190.1661, + "step": 140 + }, + { + "epoch": 2.4227642276422765, + "grad_norm": 2.3192145824432373, + "learning_rate": 0.0001, + "loss": 184.9997, + "step": 150 + }, + { + "epoch": 2.5853658536585367, + "grad_norm": 3.056708574295044, + "learning_rate": 0.0001, + "loss": 182.0164, + "step": 160 + }, + { + "epoch": 2.747967479674797, + "grad_norm": 2.9687108993530273, + "learning_rate": 0.0001, + "loss": 179.6539, + "step": 170 + }, + { + "epoch": 2.910569105691057, + "grad_norm": 2.1498093605041504, + "learning_rate": 0.0001, + "loss": 176.7159, + "step": 180 + }, + { + "epoch": 3.065040650406504, + "grad_norm": 3.0103771686553955, + "learning_rate": 0.0001, + "loss": 166.1703, + "step": 190 + }, + { + "epoch": 3.227642276422764, + "grad_norm": 2.992462635040283, + "learning_rate": 0.0001, + "loss": 171.124, + "step": 200 + }, + { + "epoch": 3.3902439024390243, + "grad_norm": 4.8617119789123535, + "learning_rate": 0.0001, + "loss": 168.2646, + "step": 210 + }, + { + "epoch": 3.5528455284552845, + "grad_norm": 3.1047444343566895, + "learning_rate": 0.0001, + "loss": 163.0889, + "step": 220 + }, + { + "epoch": 3.7154471544715446, + "grad_norm": 6.473227500915527, + "learning_rate": 0.0001, + "loss": 151.3667, + "step": 230 + }, + { + "epoch": 3.8780487804878048, + "grad_norm": 6.16747522354126, + "learning_rate": 0.0001, + "loss": 137.5915, + "step": 240 + }, + { + "epoch": 4.032520325203252, + "grad_norm": 7.994515419006348, + "learning_rate": 0.0001, + "loss": 118.8875, + "step": 250 + }, + { + "epoch": 4.195121951219512, + "grad_norm": 5.882101058959961, + "learning_rate": 0.0001, + "loss": 119.4793, + "step": 260 + }, + { + "epoch": 4.357723577235772, + "grad_norm": 12.32968521118164, + "learning_rate": 0.0001, + "loss": 113.1209, + "step": 270 + }, + { + "epoch": 4.520325203252033, + "grad_norm": 5.697225570678711, + "learning_rate": 0.0001, + "loss": 107.51, + "step": 280 + }, + { + "epoch": 4.682926829268292, + "grad_norm": 4.654167175292969, + "learning_rate": 0.0001, + "loss": 107.0172, + "step": 290 + }, + { + "epoch": 4.845528455284553, + "grad_norm": 2.9595484733581543, + "learning_rate": 0.0001, + "loss": 100.9164, + "step": 300 + }, + { + "epoch": 5.0, + "grad_norm": 21.242591857910156, + "learning_rate": 0.0001, + "loss": 94.9884, + "step": 310 + }, + { + "epoch": 5.16260162601626, + "grad_norm": 5.760314464569092, + "learning_rate": 0.0001, + "loss": 97.5901, + "step": 320 + }, + { + "epoch": 5.32520325203252, + "grad_norm": 3.1373329162597656, + "learning_rate": 0.0001, + "loss": 94.8485, + "step": 330 + }, + { + "epoch": 5.487804878048781, + "grad_norm": 2.3777568340301514, + "learning_rate": 0.0001, + "loss": 96.9982, + "step": 340 + }, + { + "epoch": 5.650406504065041, + "grad_norm": 3.0607948303222656, + "learning_rate": 0.0001, + "loss": 96.1586, + "step": 350 + }, + { + "epoch": 5.8130081300813, + "grad_norm": 4.168269634246826, + "learning_rate": 0.0001, + "loss": 94.5743, + "step": 360 + }, + { + "epoch": 5.975609756097561, + "grad_norm": 2.4383699893951416, + "learning_rate": 0.0001, + "loss": 95.5197, + "step": 370 + }, + { + "epoch": 6.130081300813008, + "grad_norm": 2.302259683609009, + "learning_rate": 0.0001, + "loss": 85.6599, + "step": 380 + }, + { + "epoch": 6.2926829268292686, + "grad_norm": 2.552753210067749, + "learning_rate": 0.0001, + "loss": 91.2999, + "step": 390 + }, + { + "epoch": 6.455284552845528, + "grad_norm": 2.2092080116271973, + "learning_rate": 0.0001, + "loss": 90.7041, + "step": 400 + }, + { + "epoch": 6.617886178861789, + "grad_norm": 2.2935690879821777, + "learning_rate": 0.0001, + "loss": 89.9949, + "step": 410 + }, + { + "epoch": 6.780487804878049, + "grad_norm": 2.487510919570923, + "learning_rate": 0.0001, + "loss": 86.9056, + "step": 420 + }, + { + "epoch": 6.943089430894309, + "grad_norm": 2.8630545139312744, + "learning_rate": 0.0001, + "loss": 88.3347, + "step": 430 + }, + { + "epoch": 7.097560975609756, + "grad_norm": 5.87056303024292, + "learning_rate": 0.0001, + "loss": 84.9632, + "step": 440 + }, + { + "epoch": 7.260162601626016, + "grad_norm": 2.587661027908325, + "learning_rate": 0.0001, + "loss": 86.2024, + "step": 450 + }, + { + "epoch": 7.4227642276422765, + "grad_norm": 3.5671255588531494, + "learning_rate": 0.0001, + "loss": 86.218, + "step": 460 + }, + { + "epoch": 7.585365853658536, + "grad_norm": 2.5036680698394775, + "learning_rate": 0.0001, + "loss": 86.5914, + "step": 470 + }, + { + "epoch": 7.747967479674797, + "grad_norm": 3.2656471729278564, + "learning_rate": 0.0001, + "loss": 82.6081, + "step": 480 + }, + { + "epoch": 7.9105691056910565, + "grad_norm": 2.5053915977478027, + "learning_rate": 0.0001, + "loss": 84.6116, + "step": 490 + }, + { + "epoch": 8.065040650406504, + "grad_norm": 2.1669957637786865, + "learning_rate": 0.0001, + "loss": 80.4354, + "step": 500 + }, + { + "epoch": 8.227642276422765, + "grad_norm": 4.2003703117370605, + "learning_rate": 0.0001, + "loss": 85.2286, + "step": 510 + }, + { + "epoch": 8.390243902439025, + "grad_norm": 3.1909608840942383, + "learning_rate": 0.0001, + "loss": 85.0358, + "step": 520 + }, + { + "epoch": 8.552845528455284, + "grad_norm": 1.8420121669769287, + "learning_rate": 0.0001, + "loss": 85.4647, + "step": 530 + }, + { + "epoch": 8.715447154471544, + "grad_norm": 1.9822801351547241, + "learning_rate": 0.0001, + "loss": 84.7134, + "step": 540 + }, + { + "epoch": 8.878048780487806, + "grad_norm": 2.936758518218994, + "learning_rate": 0.0001, + "loss": 83.8068, + "step": 550 + }, + { + "epoch": 9.032520325203253, + "grad_norm": 1.5713881254196167, + "learning_rate": 0.0001, + "loss": 80.0695, + "step": 560 + }, + { + "epoch": 9.195121951219512, + "grad_norm": 2.0584616661071777, + "learning_rate": 0.0001, + "loss": 84.5158, + "step": 570 + }, + { + "epoch": 9.357723577235772, + "grad_norm": 2.101391553878784, + "learning_rate": 0.0001, + "loss": 82.4719, + "step": 580 + }, + { + "epoch": 9.520325203252032, + "grad_norm": 1.9956543445587158, + "learning_rate": 0.0001, + "loss": 81.6715, + "step": 590 + }, + { + "epoch": 9.682926829268293, + "grad_norm": 2.7622063159942627, + "learning_rate": 0.0001, + "loss": 85.4178, + "step": 600 + }, + { + "epoch": 9.845528455284553, + "grad_norm": 2.2084789276123047, + "learning_rate": 0.0001, + "loss": 84.5702, + "step": 610 + }, + { + "epoch": 10.0, + "grad_norm": 1.9635663032531738, + "learning_rate": 0.0001, + "loss": 77.3838, + "step": 620 + }, + { + "epoch": 10.16260162601626, + "grad_norm": 2.5895581245422363, + "learning_rate": 0.0001, + "loss": 81.723, + "step": 630 + }, + { + "epoch": 10.32520325203252, + "grad_norm": 2.743727445602417, + "learning_rate": 0.0001, + "loss": 83.1332, + "step": 640 + }, + { + "epoch": 10.487804878048781, + "grad_norm": 2.024585008621216, + "learning_rate": 0.0001, + "loss": 81.5968, + "step": 650 + }, + { + "epoch": 10.65040650406504, + "grad_norm": 1.8254153728485107, + "learning_rate": 0.0001, + "loss": 78.9589, + "step": 660 + }, + { + "epoch": 10.8130081300813, + "grad_norm": 3.2015740871429443, + "learning_rate": 0.0001, + "loss": 79.1829, + "step": 670 + }, + { + "epoch": 10.975609756097562, + "grad_norm": 1.788453459739685, + "learning_rate": 0.0001, + "loss": 81.527, + "step": 680 + }, + { + "epoch": 11.130081300813009, + "grad_norm": 1.5744423866271973, + "learning_rate": 0.0001, + "loss": 74.9869, + "step": 690 + }, + { + "epoch": 11.292682926829269, + "grad_norm": 2.006002187728882, + "learning_rate": 0.0001, + "loss": 80.2677, + "step": 700 + }, + { + "epoch": 11.455284552845528, + "grad_norm": 1.8847148418426514, + "learning_rate": 0.0001, + "loss": 78.8709, + "step": 710 + }, + { + "epoch": 11.617886178861788, + "grad_norm": 1.8384478092193604, + "learning_rate": 0.0001, + "loss": 79.008, + "step": 720 + }, + { + "epoch": 11.78048780487805, + "grad_norm": 1.9486687183380127, + "learning_rate": 0.0001, + "loss": 76.1525, + "step": 730 + }, + { + "epoch": 11.94308943089431, + "grad_norm": 1.8644076585769653, + "learning_rate": 0.0001, + "loss": 77.5802, + "step": 740 + }, + { + "epoch": 12.097560975609756, + "grad_norm": 1.6035499572753906, + "learning_rate": 0.0001, + "loss": 75.2422, + "step": 750 + }, + { + "epoch": 12.260162601626016, + "grad_norm": 1.6935622692108154, + "learning_rate": 0.0001, + "loss": 77.5838, + "step": 760 + }, + { + "epoch": 12.422764227642276, + "grad_norm": 1.9327987432479858, + "learning_rate": 0.0001, + "loss": 78.3237, + "step": 770 + }, + { + "epoch": 12.585365853658537, + "grad_norm": 1.7903852462768555, + "learning_rate": 0.0001, + "loss": 78.7459, + "step": 780 + }, + { + "epoch": 12.747967479674797, + "grad_norm": 1.4962575435638428, + "learning_rate": 0.0001, + "loss": 78.9193, + "step": 790 + }, + { + "epoch": 12.910569105691057, + "grad_norm": 42.10435104370117, + "learning_rate": 0.0001, + "loss": 80.1429, + "step": 800 + }, + { + "epoch": 13.065040650406504, + "grad_norm": 1.904065728187561, + "learning_rate": 0.0001, + "loss": 73.7798, + "step": 810 + }, + { + "epoch": 13.227642276422765, + "grad_norm": 1.8768481016159058, + "learning_rate": 0.0001, + "loss": 76.8692, + "step": 820 + }, + { + "epoch": 13.390243902439025, + "grad_norm": 16.140565872192383, + "learning_rate": 0.0001, + "loss": 78.122, + "step": 830 + }, + { + "epoch": 13.552845528455284, + "grad_norm": 2.160288095474243, + "learning_rate": 0.0001, + "loss": 76.2169, + "step": 840 + }, + { + "epoch": 13.715447154471544, + "grad_norm": 27.682231903076172, + "learning_rate": 0.0001, + "loss": 76.6266, + "step": 850 + }, + { + "epoch": 13.878048780487806, + "grad_norm": 1.779772162437439, + "learning_rate": 0.0001, + "loss": 80.0434, + "step": 860 + }, + { + "epoch": 14.032520325203253, + "grad_norm": 1.6516485214233398, + "learning_rate": 0.0001, + "loss": 73.3943, + "step": 870 + }, + { + "epoch": 14.195121951219512, + "grad_norm": 1.846498966217041, + "learning_rate": 0.0001, + "loss": 77.5157, + "step": 880 + }, + { + "epoch": 14.357723577235772, + "grad_norm": 1.942946434020996, + "learning_rate": 0.0001, + "loss": 77.6944, + "step": 890 + }, + { + "epoch": 14.520325203252032, + "grad_norm": 2.0929224491119385, + "learning_rate": 0.0001, + "loss": 76.9411, + "step": 900 + }, + { + "epoch": 14.682926829268293, + "grad_norm": 3.4498767852783203, + "learning_rate": 0.0001, + "loss": 74.9234, + "step": 910 + }, + { + "epoch": 14.845528455284553, + "grad_norm": 1.6880728006362915, + "learning_rate": 0.0001, + "loss": 75.3617, + "step": 920 + }, + { + "epoch": 15.0, + "grad_norm": 1.8403468132019043, + "learning_rate": 0.0001, + "loss": 73.2543, + "step": 930 + }, + { + "epoch": 15.16260162601626, + "grad_norm": 1.4368852376937866, + "learning_rate": 0.0001, + "loss": 76.6271, + "step": 940 + }, + { + "epoch": 15.32520325203252, + "grad_norm": 2.9467780590057373, + "learning_rate": 0.0001, + "loss": 75.8744, + "step": 950 + }, + { + "epoch": 15.487804878048781, + "grad_norm": 1.567134141921997, + "learning_rate": 0.0001, + "loss": 76.6629, + "step": 960 + }, + { + "epoch": 15.65040650406504, + "grad_norm": 1.4503871202468872, + "learning_rate": 0.0001, + "loss": 76.3525, + "step": 970 + }, + { + "epoch": 15.8130081300813, + "grad_norm": 1.8871147632598877, + "learning_rate": 0.0001, + "loss": 77.4684, + "step": 980 + }, + { + "epoch": 15.975609756097562, + "grad_norm": 1.6707026958465576, + "learning_rate": 0.0001, + "loss": 76.3063, + "step": 990 + }, + { + "epoch": 16.130081300813007, + "grad_norm": 3.6300716400146484, + "learning_rate": 0.0001, + "loss": 73.0586, + "step": 1000 + }, + { + "epoch": 16.29268292682927, + "grad_norm": 1.6669234037399292, + "learning_rate": 0.0001, + "loss": 75.8027, + "step": 1010 + }, + { + "epoch": 16.45528455284553, + "grad_norm": 1.6920552253723145, + "learning_rate": 0.0001, + "loss": 74.7252, + "step": 1020 + }, + { + "epoch": 16.617886178861788, + "grad_norm": 1.8184431791305542, + "learning_rate": 0.0001, + "loss": 75.3625, + "step": 1030 + }, + { + "epoch": 16.78048780487805, + "grad_norm": 1.5828496217727661, + "learning_rate": 0.0001, + "loss": 78.0302, + "step": 1040 + }, + { + "epoch": 16.943089430894307, + "grad_norm": 1.7372915744781494, + "learning_rate": 0.0001, + "loss": 78.6949, + "step": 1050 + }, + { + "epoch": 17.097560975609756, + "grad_norm": 1.837161898612976, + "learning_rate": 0.0001, + "loss": 74.4726, + "step": 1060 + }, + { + "epoch": 17.260162601626018, + "grad_norm": 3.8733808994293213, + "learning_rate": 0.0001, + "loss": 76.1711, + "step": 1070 + }, + { + "epoch": 17.422764227642276, + "grad_norm": 1.5907851457595825, + "learning_rate": 0.0001, + "loss": 75.8292, + "step": 1080 + }, + { + "epoch": 17.585365853658537, + "grad_norm": 1.8271747827529907, + "learning_rate": 0.0001, + "loss": 74.5316, + "step": 1090 + }, + { + "epoch": 17.747967479674795, + "grad_norm": 1.827980875968933, + "learning_rate": 0.0001, + "loss": 74.7281, + "step": 1100 + }, + { + "epoch": 17.910569105691057, + "grad_norm": 1.4184074401855469, + "learning_rate": 0.0001, + "loss": 75.736, + "step": 1110 + }, + { + "epoch": 18.065040650406505, + "grad_norm": 2.3459699153900146, + "learning_rate": 0.0001, + "loss": 71.9586, + "step": 1120 + }, + { + "epoch": 18.227642276422763, + "grad_norm": 47.673641204833984, + "learning_rate": 0.0001, + "loss": 75.6649, + "step": 1130 + }, + { + "epoch": 18.390243902439025, + "grad_norm": 1.677398443222046, + "learning_rate": 0.0001, + "loss": 76.0574, + "step": 1140 + }, + { + "epoch": 18.552845528455286, + "grad_norm": 1.4180850982666016, + "learning_rate": 0.0001, + "loss": 74.3077, + "step": 1150 + }, + { + "epoch": 18.715447154471544, + "grad_norm": 2.0041465759277344, + "learning_rate": 0.0001, + "loss": 73.5916, + "step": 1160 + }, + { + "epoch": 18.878048780487806, + "grad_norm": 1.9098238945007324, + "learning_rate": 0.0001, + "loss": 74.5002, + "step": 1170 + }, + { + "epoch": 19.03252032520325, + "grad_norm": 1.405555248260498, + "learning_rate": 0.0001, + "loss": 69.8115, + "step": 1180 + }, + { + "epoch": 19.195121951219512, + "grad_norm": 1.424198865890503, + "learning_rate": 0.0001, + "loss": 73.7289, + "step": 1190 + }, + { + "epoch": 19.357723577235774, + "grad_norm": 7.542550563812256, + "learning_rate": 0.0001, + "loss": 73.7696, + "step": 1200 + }, + { + "epoch": 19.520325203252032, + "grad_norm": 1.425329566001892, + "learning_rate": 0.0001, + "loss": 72.1261, + "step": 1210 + }, + { + "epoch": 19.682926829268293, + "grad_norm": 1.525724172592163, + "learning_rate": 0.0001, + "loss": 73.6578, + "step": 1220 + }, + { + "epoch": 19.84552845528455, + "grad_norm": 1.6387193202972412, + "learning_rate": 0.0001, + "loss": 74.3027, + "step": 1230 + }, + { + "epoch": 20.0, + "grad_norm": 1.8429840803146362, + "learning_rate": 0.0001, + "loss": 69.5056, + "step": 1240 + }, + { + "epoch": 20.16260162601626, + "grad_norm": 2.5166006088256836, + "learning_rate": 0.0001, + "loss": 73.5965, + "step": 1250 + }, + { + "epoch": 20.32520325203252, + "grad_norm": 1.6006672382354736, + "learning_rate": 0.0001, + "loss": 72.4581, + "step": 1260 + }, + { + "epoch": 20.48780487804878, + "grad_norm": 1.6722878217697144, + "learning_rate": 0.0001, + "loss": 72.3097, + "step": 1270 + }, + { + "epoch": 20.65040650406504, + "grad_norm": 1.8715728521347046, + "learning_rate": 0.0001, + "loss": 73.4752, + "step": 1280 + }, + { + "epoch": 20.8130081300813, + "grad_norm": 1.3280900716781616, + "learning_rate": 0.0001, + "loss": 73.2145, + "step": 1290 + }, + { + "epoch": 20.975609756097562, + "grad_norm": 1.941451907157898, + "learning_rate": 0.0001, + "loss": 73.2085, + "step": 1300 + }, + { + "epoch": 21.130081300813007, + "grad_norm": 4.277497291564941, + "learning_rate": 0.0001, + "loss": 68.8305, + "step": 1310 + }, + { + "epoch": 21.29268292682927, + "grad_norm": 1.9949846267700195, + "learning_rate": 0.0001, + "loss": 72.6689, + "step": 1320 + }, + { + "epoch": 21.45528455284553, + "grad_norm": 1.524044394493103, + "learning_rate": 0.0001, + "loss": 72.9705, + "step": 1330 + }, + { + "epoch": 21.617886178861788, + "grad_norm": 1.384194016456604, + "learning_rate": 0.0001, + "loss": 72.5465, + "step": 1340 + }, + { + "epoch": 21.78048780487805, + "grad_norm": 1.724225640296936, + "learning_rate": 0.0001, + "loss": 72.9203, + "step": 1350 + }, + { + "epoch": 21.943089430894307, + "grad_norm": 2.0856704711914062, + "learning_rate": 0.0001, + "loss": 72.4034, + "step": 1360 + }, + { + "epoch": 22.097560975609756, + "grad_norm": 5.128044128417969, + "learning_rate": 0.0001, + "loss": 68.7754, + "step": 1370 + }, + { + "epoch": 22.260162601626018, + "grad_norm": 1.7385634183883667, + "learning_rate": 0.0001, + "loss": 73.9349, + "step": 1380 + }, + { + "epoch": 22.422764227642276, + "grad_norm": 1.3946012258529663, + "learning_rate": 0.0001, + "loss": 70.9627, + "step": 1390 + }, + { + "epoch": 22.585365853658537, + "grad_norm": 1.285414457321167, + "learning_rate": 0.0001, + "loss": 72.4578, + "step": 1400 + }, + { + "epoch": 22.747967479674795, + "grad_norm": 1.4197710752487183, + "learning_rate": 0.0001, + "loss": 72.5252, + "step": 1410 + }, + { + "epoch": 22.910569105691057, + "grad_norm": 3.048942804336548, + "learning_rate": 0.0001, + "loss": 71.1107, + "step": 1420 + }, + { + "epoch": 23.065040650406505, + "grad_norm": 1.3114765882492065, + "learning_rate": 0.0001, + "loss": 68.2725, + "step": 1430 + }, + { + "epoch": 23.227642276422763, + "grad_norm": 1.2763880491256714, + "learning_rate": 0.0001, + "loss": 71.5783, + "step": 1440 + }, + { + "epoch": 23.390243902439025, + "grad_norm": 1.3913089036941528, + "learning_rate": 0.0001, + "loss": 72.5986, + "step": 1450 + }, + { + "epoch": 23.552845528455286, + "grad_norm": 1.628751516342163, + "learning_rate": 0.0001, + "loss": 71.8383, + "step": 1460 + }, + { + "epoch": 23.715447154471544, + "grad_norm": 1.4331274032592773, + "learning_rate": 0.0001, + "loss": 70.8051, + "step": 1470 + }, + { + "epoch": 23.878048780487806, + "grad_norm": 1.4154727458953857, + "learning_rate": 0.0001, + "loss": 71.3283, + "step": 1480 + }, + { + "epoch": 24.03252032520325, + "grad_norm": 1.7208784818649292, + "learning_rate": 0.0001, + "loss": 68.4719, + "step": 1490 + }, + { + "epoch": 24.195121951219512, + "grad_norm": 1.466123342514038, + "learning_rate": 0.0001, + "loss": 71.9597, + "step": 1500 + }, + { + "epoch": 24.357723577235774, + "grad_norm": 1.4823886156082153, + "learning_rate": 0.0001, + "loss": 72.4533, + "step": 1510 + }, + { + "epoch": 24.520325203252032, + "grad_norm": 1.8407355546951294, + "learning_rate": 0.0001, + "loss": 71.1468, + "step": 1520 + }, + { + "epoch": 24.682926829268293, + "grad_norm": 1.630565881729126, + "learning_rate": 0.0001, + "loss": 71.5192, + "step": 1530 + }, + { + "epoch": 24.84552845528455, + "grad_norm": 1.5248785018920898, + "learning_rate": 0.0001, + "loss": 68.4804, + "step": 1540 + }, + { + "epoch": 25.0, + "grad_norm": 1.4690219163894653, + "learning_rate": 0.0001, + "loss": 67.5716, + "step": 1550 + }, + { + "epoch": 25.16260162601626, + "grad_norm": 2.0246646404266357, + "learning_rate": 0.0001, + "loss": 71.2554, + "step": 1560 + }, + { + "epoch": 25.32520325203252, + "grad_norm": 1.3611458539962769, + "learning_rate": 0.0001, + "loss": 69.8558, + "step": 1570 + }, + { + "epoch": 25.48780487804878, + "grad_norm": 2.0006039142608643, + "learning_rate": 0.0001, + "loss": 72.5344, + "step": 1580 + }, + { + "epoch": 25.65040650406504, + "grad_norm": 1.7373167276382446, + "learning_rate": 0.0001, + "loss": 70.6253, + "step": 1590 + }, + { + "epoch": 25.8130081300813, + "grad_norm": 1.2878378629684448, + "learning_rate": 0.0001, + "loss": 72.165, + "step": 1600 + }, + { + "epoch": 25.975609756097562, + "grad_norm": 1.5337013006210327, + "learning_rate": 0.0001, + "loss": 70.6989, + "step": 1610 + }, + { + "epoch": 26.130081300813007, + "grad_norm": 1.364615559577942, + "learning_rate": 0.0001, + "loss": 67.3898, + "step": 1620 + }, + { + "epoch": 26.29268292682927, + "grad_norm": 1.3249316215515137, + "learning_rate": 0.0001, + "loss": 72.7102, + "step": 1630 + }, + { + "epoch": 26.45528455284553, + "grad_norm": 4.209898471832275, + "learning_rate": 0.0001, + "loss": 70.3185, + "step": 1640 + }, + { + "epoch": 26.617886178861788, + "grad_norm": 1.5488810539245605, + "learning_rate": 0.0001, + "loss": 72.3678, + "step": 1650 + }, + { + "epoch": 26.78048780487805, + "grad_norm": 1.7743597030639648, + "learning_rate": 0.0001, + "loss": 71.0917, + "step": 1660 + }, + { + "epoch": 26.943089430894307, + "grad_norm": 1.5433533191680908, + "learning_rate": 0.0001, + "loss": 71.4001, + "step": 1670 + }, + { + "epoch": 27.097560975609756, + "grad_norm": 1.6320385932922363, + "learning_rate": 0.0001, + "loss": 66.7441, + "step": 1680 + }, + { + "epoch": 27.260162601626018, + "grad_norm": 1.3137969970703125, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 1690 + }, + { + "epoch": 27.422764227642276, + "grad_norm": 1.9150484800338745, + "learning_rate": 0.0001, + "loss": 69.8956, + "step": 1700 + }, + { + "epoch": 27.585365853658537, + "grad_norm": 1.4442802667617798, + "learning_rate": 0.0001, + "loss": 70.7185, + "step": 1710 + }, + { + "epoch": 27.747967479674795, + "grad_norm": 2.572497606277466, + "learning_rate": 0.0001, + "loss": 70.8934, + "step": 1720 + }, + { + "epoch": 27.910569105691057, + "grad_norm": 1.6669617891311646, + "learning_rate": 0.0001, + "loss": 71.8221, + "step": 1730 + }, + { + "epoch": 28.065040650406505, + "grad_norm": 1.8449876308441162, + "learning_rate": 0.0001, + "loss": 66.3923, + "step": 1740 + }, + { + "epoch": 28.227642276422763, + "grad_norm": 1.6600004434585571, + "learning_rate": 0.0001, + "loss": 69.5301, + "step": 1750 + }, + { + "epoch": 28.390243902439025, + "grad_norm": 1.4655731916427612, + "learning_rate": 0.0001, + "loss": 72.1924, + "step": 1760 + }, + { + "epoch": 28.552845528455286, + "grad_norm": 1.134505033493042, + "learning_rate": 0.0001, + "loss": 69.5855, + "step": 1770 + }, + { + "epoch": 28.715447154471544, + "grad_norm": 1.301984190940857, + "learning_rate": 0.0001, + "loss": 70.7999, + "step": 1780 + }, + { + "epoch": 28.878048780487806, + "grad_norm": 1.5641013383865356, + "learning_rate": 0.0001, + "loss": 69.9091, + "step": 1790 + }, + { + "epoch": 29.03252032520325, + "grad_norm": 1.4850797653198242, + "learning_rate": 0.0001, + "loss": 66.6266, + "step": 1800 + }, + { + "epoch": 29.195121951219512, + "grad_norm": 1.2069118022918701, + "learning_rate": 0.0001, + "loss": 69.3802, + "step": 1810 + }, + { + "epoch": 29.357723577235774, + "grad_norm": 1.2890352010726929, + "learning_rate": 0.0001, + "loss": 68.5044, + "step": 1820 + }, + { + "epoch": 29.520325203252032, + "grad_norm": 1.4150917530059814, + "learning_rate": 0.0001, + "loss": 69.807, + "step": 1830 + }, + { + "epoch": 29.682926829268293, + "grad_norm": 1.514807939529419, + "learning_rate": 0.0001, + "loss": 71.3858, + "step": 1840 + }, + { + "epoch": 29.84552845528455, + "grad_norm": 1.4246330261230469, + "learning_rate": 0.0001, + "loss": 70.494, + "step": 1850 + }, + { + "epoch": 30.0, + "grad_norm": 1.1789597272872925, + "learning_rate": 0.0001, + "loss": 67.1056, + "step": 1860 + }, + { + "epoch": 30.16260162601626, + "grad_norm": 1.23678719997406, + "learning_rate": 0.0001, + "loss": 70.336, + "step": 1870 + }, + { + "epoch": 30.32520325203252, + "grad_norm": 1.1756274700164795, + "learning_rate": 0.0001, + "loss": 67.9909, + "step": 1880 + }, + { + "epoch": 30.48780487804878, + "grad_norm": 1.4175690412521362, + "learning_rate": 0.0001, + "loss": 69.2617, + "step": 1890 + }, + { + "epoch": 30.65040650406504, + "grad_norm": 1.3084465265274048, + "learning_rate": 0.0001, + "loss": 70.2963, + "step": 1900 + }, + { + "epoch": 30.8130081300813, + "grad_norm": 1.486464500427246, + "learning_rate": 0.0001, + "loss": 68.6317, + "step": 1910 + }, + { + "epoch": 30.975609756097562, + "grad_norm": 10.415762901306152, + "learning_rate": 0.0001, + "loss": 69.7767, + "step": 1920 + }, + { + "epoch": 31.130081300813007, + "grad_norm": 1.4484038352966309, + "learning_rate": 0.0001, + "loss": 65.6345, + "step": 1930 + }, + { + "epoch": 31.29268292682927, + "grad_norm": 2.297306537628174, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 1940 + }, + { + "epoch": 31.45528455284553, + "grad_norm": 1.223490595817566, + "learning_rate": 0.0001, + "loss": 69.2139, + "step": 1950 + }, + { + "epoch": 31.617886178861788, + "grad_norm": 1.2144125699996948, + "learning_rate": 0.0001, + "loss": 70.3061, + "step": 1960 + }, + { + "epoch": 31.78048780487805, + "grad_norm": 2.633528470993042, + "learning_rate": 0.0001, + "loss": 68.7096, + "step": 1970 + }, + { + "epoch": 31.943089430894307, + "grad_norm": 1.5048350095748901, + "learning_rate": 0.0001, + "loss": 68.6711, + "step": 1980 + }, + { + "epoch": 32.09756097560975, + "grad_norm": 1.5154467821121216, + "learning_rate": 0.0001, + "loss": 66.7989, + "step": 1990 + }, + { + "epoch": 32.260162601626014, + "grad_norm": 2.721277952194214, + "learning_rate": 0.0001, + "loss": 70.4987, + "step": 2000 + }, + { + "epoch": 32.422764227642276, + "grad_norm": 1.44464910030365, + "learning_rate": 0.0001, + "loss": 69.9712, + "step": 2010 + }, + { + "epoch": 32.58536585365854, + "grad_norm": 1.3462835550308228, + "learning_rate": 0.0001, + "loss": 68.9381, + "step": 2020 + }, + { + "epoch": 32.7479674796748, + "grad_norm": 1.3283029794692993, + "learning_rate": 0.0001, + "loss": 68.5301, + "step": 2030 + }, + { + "epoch": 32.91056910569106, + "grad_norm": 1.5172960758209229, + "learning_rate": 0.0001, + "loss": 70.1894, + "step": 2040 + }, + { + "epoch": 33.0650406504065, + "grad_norm": 1.1872576475143433, + "learning_rate": 0.0001, + "loss": 65.0158, + "step": 2050 + }, + { + "epoch": 33.22764227642276, + "grad_norm": 1.5414419174194336, + "learning_rate": 0.0001, + "loss": 69.1537, + "step": 2060 + }, + { + "epoch": 33.390243902439025, + "grad_norm": 1.3143430948257446, + "learning_rate": 0.0001, + "loss": 68.6792, + "step": 2070 + }, + { + "epoch": 33.552845528455286, + "grad_norm": 1.3863723278045654, + "learning_rate": 0.0001, + "loss": 70.3867, + "step": 2080 + }, + { + "epoch": 33.71544715447155, + "grad_norm": 1.4060479402542114, + "learning_rate": 0.0001, + "loss": 68.4981, + "step": 2090 + }, + { + "epoch": 33.8780487804878, + "grad_norm": 1.34132981300354, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 2100 + }, + { + "epoch": 34.03252032520325, + "grad_norm": 1.8303673267364502, + "learning_rate": 0.0001, + "loss": 65.819, + "step": 2110 + }, + { + "epoch": 34.19512195121951, + "grad_norm": 1.387339472770691, + "learning_rate": 0.0001, + "loss": 69.9825, + "step": 2120 + }, + { + "epoch": 34.357723577235774, + "grad_norm": 1.4125195741653442, + "learning_rate": 0.0001, + "loss": 67.5149, + "step": 2130 + }, + { + "epoch": 34.520325203252035, + "grad_norm": 1.2909412384033203, + "learning_rate": 0.0001, + "loss": 69.7848, + "step": 2140 + }, + { + "epoch": 34.68292682926829, + "grad_norm": 1.8860795497894287, + "learning_rate": 0.0001, + "loss": 68.6145, + "step": 2150 + }, + { + "epoch": 34.84552845528455, + "grad_norm": 1.5111079216003418, + "learning_rate": 0.0001, + "loss": 68.2612, + "step": 2160 + }, + { + "epoch": 35.0, + "grad_norm": 1.3275264501571655, + "learning_rate": 0.0001, + "loss": 64.0298, + "step": 2170 + }, + { + "epoch": 35.16260162601626, + "grad_norm": 1.174864411354065, + "learning_rate": 0.0001, + "loss": 66.6153, + "step": 2180 + }, + { + "epoch": 35.32520325203252, + "grad_norm": 1.6873915195465088, + "learning_rate": 0.0001, + "loss": 69.6278, + "step": 2190 + }, + { + "epoch": 35.48780487804878, + "grad_norm": 2.4457507133483887, + "learning_rate": 0.0001, + "loss": 68.3611, + "step": 2200 + }, + { + "epoch": 35.65040650406504, + "grad_norm": 1.4779958724975586, + "learning_rate": 0.0001, + "loss": 69.2609, + "step": 2210 + }, + { + "epoch": 35.8130081300813, + "grad_norm": 2.273709774017334, + "learning_rate": 0.0001, + "loss": 69.1312, + "step": 2220 + }, + { + "epoch": 35.97560975609756, + "grad_norm": 1.5741862058639526, + "learning_rate": 0.0001, + "loss": 69.1191, + "step": 2230 + }, + { + "epoch": 36.13008130081301, + "grad_norm": 1.2730036973953247, + "learning_rate": 0.0001, + "loss": 65.692, + "step": 2240 + }, + { + "epoch": 36.292682926829265, + "grad_norm": 1.2936429977416992, + "learning_rate": 0.0001, + "loss": 68.2962, + "step": 2250 + }, + { + "epoch": 36.45528455284553, + "grad_norm": 1.354279637336731, + "learning_rate": 0.0001, + "loss": 70.5507, + "step": 2260 + }, + { + "epoch": 36.61788617886179, + "grad_norm": 1.4165853261947632, + "learning_rate": 0.0001, + "loss": 68.0048, + "step": 2270 + }, + { + "epoch": 36.78048780487805, + "grad_norm": 1.2992445230484009, + "learning_rate": 0.0001, + "loss": 67.7072, + "step": 2280 + }, + { + "epoch": 36.94308943089431, + "grad_norm": 1.5053461790084839, + "learning_rate": 0.0001, + "loss": 66.7587, + "step": 2290 + }, + { + "epoch": 37.09756097560975, + "grad_norm": 1.337390661239624, + "learning_rate": 0.0001, + "loss": 63.786, + "step": 2300 + }, + { + "epoch": 37.260162601626014, + "grad_norm": 1.0407774448394775, + "learning_rate": 0.0001, + "loss": 66.6253, + "step": 2310 + }, + { + "epoch": 37.422764227642276, + "grad_norm": 1.564152479171753, + "learning_rate": 0.0001, + "loss": 68.3606, + "step": 2320 + }, + { + "epoch": 37.58536585365854, + "grad_norm": 1.2028454542160034, + "learning_rate": 0.0001, + "loss": 66.2841, + "step": 2330 + }, + { + "epoch": 37.7479674796748, + "grad_norm": 1.4528603553771973, + "learning_rate": 0.0001, + "loss": 67.4277, + "step": 2340 + }, + { + "epoch": 37.91056910569106, + "grad_norm": 1.4954335689544678, + "learning_rate": 0.0001, + "loss": 68.9954, + "step": 2350 + }, + { + "epoch": 38.0650406504065, + "grad_norm": 1.2820416688919067, + "learning_rate": 0.0001, + "loss": 65.2449, + "step": 2360 + }, + { + "epoch": 38.22764227642276, + "grad_norm": 1.2392579317092896, + "learning_rate": 0.0001, + "loss": 67.5006, + "step": 2370 + }, + { + "epoch": 38.390243902439025, + "grad_norm": 1.2135238647460938, + "learning_rate": 0.0001, + "loss": 69.2214, + "step": 2380 + }, + { + "epoch": 38.552845528455286, + "grad_norm": 1.5964744091033936, + "learning_rate": 0.0001, + "loss": 67.934, + "step": 2390 + }, + { + "epoch": 38.71544715447155, + "grad_norm": 1.2841702699661255, + "learning_rate": 0.0001, + "loss": 67.0848, + "step": 2400 + }, + { + "epoch": 38.8780487804878, + "grad_norm": 1.8026081323623657, + "learning_rate": 0.0001, + "loss": 67.5358, + "step": 2410 + }, + { + "epoch": 39.03252032520325, + "grad_norm": 1.4178807735443115, + "learning_rate": 0.0001, + "loss": 64.771, + "step": 2420 + }, + { + "epoch": 39.19512195121951, + "grad_norm": 1.3671854734420776, + "learning_rate": 0.0001, + "loss": 68.0531, + "step": 2430 + }, + { + "epoch": 39.357723577235774, + "grad_norm": 1.43596613407135, + "learning_rate": 0.0001, + "loss": 67.7735, + "step": 2440 + }, + { + "epoch": 39.520325203252035, + "grad_norm": 1.464250922203064, + "learning_rate": 0.0001, + "loss": 67.8331, + "step": 2450 + }, + { + "epoch": 39.68292682926829, + "grad_norm": 3.101461410522461, + "learning_rate": 0.0001, + "loss": 65.8956, + "step": 2460 + }, + { + "epoch": 39.84552845528455, + "grad_norm": 1.8332289457321167, + "learning_rate": 0.0001, + "loss": 66.8218, + "step": 2470 + }, + { + "epoch": 40.0, + "grad_norm": 1.2660984992980957, + "learning_rate": 0.0001, + "loss": 63.9969, + "step": 2480 + }, + { + "epoch": 40.16260162601626, + "grad_norm": 2.0309898853302, + "learning_rate": 0.0001, + "loss": 67.3538, + "step": 2490 + }, + { + "epoch": 40.32520325203252, + "grad_norm": 1.207000494003296, + "learning_rate": 0.0001, + "loss": 68.0898, + "step": 2500 + }, + { + "epoch": 40.48780487804878, + "grad_norm": 1.3609018325805664, + "learning_rate": 0.0001, + "loss": 67.0381, + "step": 2510 + }, + { + "epoch": 40.65040650406504, + "grad_norm": 1.310826301574707, + "learning_rate": 0.0001, + "loss": 69.0267, + "step": 2520 + }, + { + "epoch": 40.8130081300813, + "grad_norm": 1.4150960445404053, + "learning_rate": 0.0001, + "loss": 69.1934, + "step": 2530 + }, + { + "epoch": 40.97560975609756, + "grad_norm": 1.5414319038391113, + "learning_rate": 0.0001, + "loss": 68.0604, + "step": 2540 + }, + { + "epoch": 41.13008130081301, + "grad_norm": 2.3246140480041504, + "learning_rate": 0.0001, + "loss": 61.5698, + "step": 2550 + }, + { + "epoch": 41.292682926829265, + "grad_norm": 1.206296443939209, + "learning_rate": 0.0001, + "loss": 64.9898, + "step": 2560 + }, + { + "epoch": 41.45528455284553, + "grad_norm": 1.6080626249313354, + "learning_rate": 0.0001, + "loss": 68.2685, + "step": 2570 + }, + { + "epoch": 41.61788617886179, + "grad_norm": 1.2404112815856934, + "learning_rate": 0.0001, + "loss": 67.8457, + "step": 2580 + }, + { + "epoch": 41.78048780487805, + "grad_norm": 1.2304514646530151, + "learning_rate": 0.0001, + "loss": 67.9052, + "step": 2590 + }, + { + "epoch": 41.94308943089431, + "grad_norm": 1.5806187391281128, + "learning_rate": 0.0001, + "loss": 66.0663, + "step": 2600 + }, + { + "epoch": 42.09756097560975, + "grad_norm": 1.2564151287078857, + "learning_rate": 0.0001, + "loss": 65.7763, + "step": 2610 + }, + { + "epoch": 42.260162601626014, + "grad_norm": 1.4972114562988281, + "learning_rate": 0.0001, + "loss": 66.4269, + "step": 2620 + }, + { + "epoch": 42.422764227642276, + "grad_norm": 1.2913843393325806, + "learning_rate": 0.0001, + "loss": 67.7554, + "step": 2630 + }, + { + "epoch": 42.58536585365854, + "grad_norm": 1.3743863105773926, + "learning_rate": 0.0001, + "loss": 67.2627, + "step": 2640 + }, + { + "epoch": 42.7479674796748, + "grad_norm": 1.1267883777618408, + "learning_rate": 0.0001, + "loss": 67.4597, + "step": 2650 + }, + { + "epoch": 42.91056910569106, + "grad_norm": 2.0194661617279053, + "learning_rate": 0.0001, + "loss": 66.9295, + "step": 2660 + }, + { + "epoch": 43.0650406504065, + "grad_norm": 1.273779273033142, + "learning_rate": 0.0001, + "loss": 60.7906, + "step": 2670 + }, + { + "epoch": 43.22764227642276, + "grad_norm": 1.1498609781265259, + "learning_rate": 0.0001, + "loss": 68.0405, + "step": 2680 + }, + { + "epoch": 43.390243902439025, + "grad_norm": 1.4689130783081055, + "learning_rate": 0.0001, + "loss": 66.2358, + "step": 2690 + }, + { + "epoch": 43.552845528455286, + "grad_norm": 1.4495564699172974, + "learning_rate": 0.0001, + "loss": 67.4267, + "step": 2700 + }, + { + "epoch": 43.71544715447155, + "grad_norm": 2.5769612789154053, + "learning_rate": 0.0001, + "loss": 66.953, + "step": 2710 + }, + { + "epoch": 43.8780487804878, + "grad_norm": 2.1684205532073975, + "learning_rate": 0.0001, + "loss": 65.6393, + "step": 2720 + }, + { + "epoch": 44.03252032520325, + "grad_norm": 1.4968472719192505, + "learning_rate": 0.0001, + "loss": 60.8256, + "step": 2730 + }, + { + "epoch": 44.19512195121951, + "grad_norm": 1.1874353885650635, + "learning_rate": 0.0001, + "loss": 66.818, + "step": 2740 + }, + { + "epoch": 44.357723577235774, + "grad_norm": 2.271235227584839, + "learning_rate": 0.0001, + "loss": 66.5048, + "step": 2750 + }, + { + "epoch": 44.520325203252035, + "grad_norm": 1.20489501953125, + "learning_rate": 0.0001, + "loss": 67.0487, + "step": 2760 + }, + { + "epoch": 44.68292682926829, + "grad_norm": 1.6273691654205322, + "learning_rate": 0.0001, + "loss": 66.2228, + "step": 2770 + }, + { + "epoch": 44.84552845528455, + "grad_norm": 1.4935433864593506, + "learning_rate": 0.0001, + "loss": 67.2039, + "step": 2780 + }, + { + "epoch": 45.0, + "grad_norm": 1.2668659687042236, + "learning_rate": 0.0001, + "loss": 62.7855, + "step": 2790 + }, + { + "epoch": 45.16260162601626, + "grad_norm": 1.6439573764801025, + "learning_rate": 0.0001, + "loss": 67.919, + "step": 2800 + }, + { + "epoch": 45.32520325203252, + "grad_norm": 1.311854600906372, + "learning_rate": 0.0001, + "loss": 65.1797, + "step": 2810 + }, + { + "epoch": 45.48780487804878, + "grad_norm": 1.3596776723861694, + "learning_rate": 0.0001, + "loss": 67.469, + "step": 2820 + }, + { + "epoch": 45.65040650406504, + "grad_norm": 1.5618443489074707, + "learning_rate": 0.0001, + "loss": 66.1808, + "step": 2830 + }, + { + "epoch": 45.8130081300813, + "grad_norm": 1.7551205158233643, + "learning_rate": 0.0001, + "loss": 65.3803, + "step": 2840 + }, + { + "epoch": 45.97560975609756, + "grad_norm": 1.433794379234314, + "learning_rate": 0.0001, + "loss": 67.8314, + "step": 2850 + }, + { + "epoch": 46.13008130081301, + "grad_norm": 1.4634851217269897, + "learning_rate": 0.0001, + "loss": 63.4219, + "step": 2860 + }, + { + "epoch": 46.292682926829265, + "grad_norm": 1.083052396774292, + "learning_rate": 0.0001, + "loss": 68.0719, + "step": 2870 + }, + { + "epoch": 46.45528455284553, + "grad_norm": 1.1784703731536865, + "learning_rate": 0.0001, + "loss": 66.1391, + "step": 2880 + }, + { + "epoch": 46.61788617886179, + "grad_norm": 1.2552465200424194, + "learning_rate": 0.0001, + "loss": 65.6006, + "step": 2890 + }, + { + "epoch": 46.78048780487805, + "grad_norm": 1.4684522151947021, + "learning_rate": 0.0001, + "loss": 66.8314, + "step": 2900 + }, + { + "epoch": 46.94308943089431, + "grad_norm": 1.741514801979065, + "learning_rate": 0.0001, + "loss": 67.5056, + "step": 2910 + }, + { + "epoch": 47.09756097560975, + "grad_norm": 5.696534156799316, + "learning_rate": 0.0001, + "loss": 62.8016, + "step": 2920 + }, + { + "epoch": 47.260162601626014, + "grad_norm": 1.5000969171524048, + "learning_rate": 0.0001, + "loss": 68.0272, + "step": 2930 + }, + { + "epoch": 47.422764227642276, + "grad_norm": 1.1044933795928955, + "learning_rate": 0.0001, + "loss": 67.0905, + "step": 2940 + }, + { + "epoch": 47.58536585365854, + "grad_norm": 0.9933589100837708, + "learning_rate": 0.0001, + "loss": 67.6077, + "step": 2950 + }, + { + "epoch": 47.7479674796748, + "grad_norm": 1.2908599376678467, + "learning_rate": 0.0001, + "loss": 65.869, + "step": 2960 + }, + { + "epoch": 47.91056910569106, + "grad_norm": 1.0470529794692993, + "learning_rate": 0.0001, + "loss": 66.5353, + "step": 2970 + }, + { + "epoch": 48.0650406504065, + "grad_norm": 1.0927976369857788, + "learning_rate": 0.0001, + "loss": 62.7541, + "step": 2980 + }, + { + "epoch": 48.22764227642276, + "grad_norm": 1.0870513916015625, + "learning_rate": 0.0001, + "loss": 64.172, + "step": 2990 + }, + { + "epoch": 48.390243902439025, + "grad_norm": 1.3492090702056885, + "learning_rate": 0.0001, + "loss": 67.6642, + "step": 3000 + }, + { + "epoch": 48.552845528455286, + "grad_norm": 1.4024657011032104, + "learning_rate": 0.0001, + "loss": 65.2722, + "step": 3010 + }, + { + "epoch": 48.71544715447155, + "grad_norm": 1.1165252923965454, + "learning_rate": 0.0001, + "loss": 66.7233, + "step": 3020 + }, + { + "epoch": 48.8780487804878, + "grad_norm": 1.2147928476333618, + "learning_rate": 0.0001, + "loss": 65.7277, + "step": 3030 + }, + { + "epoch": 49.03252032520325, + "grad_norm": 0.925310492515564, + "learning_rate": 0.0001, + "loss": 62.9256, + "step": 3040 + }, + { + "epoch": 49.19512195121951, + "grad_norm": 1.2353442907333374, + "learning_rate": 0.0001, + "loss": 66.1218, + "step": 3050 + }, + { + "epoch": 49.357723577235774, + "grad_norm": 1.2177367210388184, + "learning_rate": 0.0001, + "loss": 66.7978, + "step": 3060 + }, + { + "epoch": 49.520325203252035, + "grad_norm": 1.248044729232788, + "learning_rate": 0.0001, + "loss": 66.2132, + "step": 3070 + }, + { + "epoch": 49.68292682926829, + "grad_norm": 1.366050124168396, + "learning_rate": 0.0001, + "loss": 67.0118, + "step": 3080 + }, + { + "epoch": 49.84552845528455, + "grad_norm": 1.3701151609420776, + "learning_rate": 0.0001, + "loss": 67.6099, + "step": 3090 + }, + { + "epoch": 50.0, + "grad_norm": 1.0420141220092773, + "learning_rate": 0.0001, + "loss": 61.5289, + "step": 3100 + }, + { + "epoch": 50.16260162601626, + "grad_norm": 1.0957746505737305, + "learning_rate": 0.0001, + "loss": 66.7348, + "step": 3110 + }, + { + "epoch": 50.32520325203252, + "grad_norm": 1.108628749847412, + "learning_rate": 0.0001, + "loss": 66.6199, + "step": 3120 + }, + { + "epoch": 50.48780487804878, + "grad_norm": 1.3680156469345093, + "learning_rate": 0.0001, + "loss": 67.0825, + "step": 3130 + }, + { + "epoch": 50.65040650406504, + "grad_norm": 1.169851303100586, + "learning_rate": 0.0001, + "loss": 65.4901, + "step": 3140 + }, + { + "epoch": 50.8130081300813, + "grad_norm": 2.1822028160095215, + "learning_rate": 0.0001, + "loss": 65.6841, + "step": 3150 + }, + { + "epoch": 50.97560975609756, + "grad_norm": 1.053402304649353, + "learning_rate": 0.0001, + "loss": 66.3119, + "step": 3160 + }, + { + "epoch": 51.13008130081301, + "grad_norm": 1.0627789497375488, + "learning_rate": 0.0001, + "loss": 61.3642, + "step": 3170 + }, + { + "epoch": 51.292682926829265, + "grad_norm": 1.926291823387146, + "learning_rate": 0.0001, + "loss": 66.6562, + "step": 3180 + }, + { + "epoch": 51.45528455284553, + "grad_norm": 1.5791983604431152, + "learning_rate": 0.0001, + "loss": 63.913, + "step": 3190 + }, + { + "epoch": 51.61788617886179, + "grad_norm": 1.210265040397644, + "learning_rate": 0.0001, + "loss": 64.5261, + "step": 3200 + }, + { + "epoch": 51.78048780487805, + "grad_norm": 1.0720808506011963, + "learning_rate": 0.0001, + "loss": 67.0194, + "step": 3210 + }, + { + "epoch": 51.94308943089431, + "grad_norm": 1.94563889503479, + "learning_rate": 0.0001, + "loss": 68.4662, + "step": 3220 + }, + { + "epoch": 52.09756097560975, + "grad_norm": 1.598980188369751, + "learning_rate": 0.0001, + "loss": 62.4946, + "step": 3230 + }, + { + "epoch": 52.260162601626014, + "grad_norm": 1.0564576387405396, + "learning_rate": 0.0001, + "loss": 65.7912, + "step": 3240 + }, + { + "epoch": 52.422764227642276, + "grad_norm": 1.2189645767211914, + "learning_rate": 0.0001, + "loss": 65.5772, + "step": 3250 + }, + { + "epoch": 52.58536585365854, + "grad_norm": 1.1357049942016602, + "learning_rate": 0.0001, + "loss": 65.1559, + "step": 3260 + }, + { + "epoch": 52.7479674796748, + "grad_norm": 1.316867709159851, + "learning_rate": 0.0001, + "loss": 64.7897, + "step": 3270 + }, + { + "epoch": 52.91056910569106, + "grad_norm": 1.3870197534561157, + "learning_rate": 0.0001, + "loss": 65.5195, + "step": 3280 + }, + { + "epoch": 53.0650406504065, + "grad_norm": 1.2467927932739258, + "learning_rate": 0.0001, + "loss": 61.3392, + "step": 3290 + }, + { + "epoch": 53.22764227642276, + "grad_norm": 1.1661109924316406, + "learning_rate": 0.0001, + "loss": 66.0117, + "step": 3300 + }, + { + "epoch": 53.390243902439025, + "grad_norm": 1.3509628772735596, + "learning_rate": 0.0001, + "loss": 64.7228, + "step": 3310 + }, + { + "epoch": 53.552845528455286, + "grad_norm": 1.1093642711639404, + "learning_rate": 0.0001, + "loss": 65.2534, + "step": 3320 + }, + { + "epoch": 53.71544715447155, + "grad_norm": 1.1801207065582275, + "learning_rate": 0.0001, + "loss": 65.7097, + "step": 3330 + }, + { + "epoch": 53.8780487804878, + "grad_norm": 1.175884485244751, + "learning_rate": 0.0001, + "loss": 64.4368, + "step": 3340 + }, + { + "epoch": 54.03252032520325, + "grad_norm": 1.1727889776229858, + "learning_rate": 0.0001, + "loss": 62.9373, + "step": 3350 + }, + { + "epoch": 54.19512195121951, + "grad_norm": 1.5330945253372192, + "learning_rate": 0.0001, + "loss": 66.7763, + "step": 3360 + }, + { + "epoch": 54.357723577235774, + "grad_norm": 1.2992504835128784, + "learning_rate": 0.0001, + "loss": 65.7854, + "step": 3370 + }, + { + "epoch": 54.520325203252035, + "grad_norm": 1.0185420513153076, + "learning_rate": 0.0001, + "loss": 65.7046, + "step": 3380 + }, + { + "epoch": 54.68292682926829, + "grad_norm": 1.426229476928711, + "learning_rate": 0.0001, + "loss": 64.0286, + "step": 3390 + }, + { + "epoch": 54.84552845528455, + "grad_norm": 1.2600868940353394, + "learning_rate": 0.0001, + "loss": 66.1738, + "step": 3400 + }, + { + "epoch": 55.0, + "grad_norm": 1.917455792427063, + "learning_rate": 0.0001, + "loss": 61.9381, + "step": 3410 + }, + { + "epoch": 55.16260162601626, + "grad_norm": 1.2000917196273804, + "learning_rate": 0.0001, + "loss": 65.5626, + "step": 3420 + }, + { + "epoch": 55.32520325203252, + "grad_norm": 1.362176775932312, + "learning_rate": 0.0001, + "loss": 66.8012, + "step": 3430 + }, + { + "epoch": 55.48780487804878, + "grad_norm": 1.1611402034759521, + "learning_rate": 0.0001, + "loss": 64.5227, + "step": 3440 + }, + { + "epoch": 55.65040650406504, + "grad_norm": 1.2066621780395508, + "learning_rate": 0.0001, + "loss": 63.4351, + "step": 3450 + }, + { + "epoch": 55.8130081300813, + "grad_norm": 1.005340814590454, + "learning_rate": 0.0001, + "loss": 65.9261, + "step": 3460 + }, + { + "epoch": 55.97560975609756, + "grad_norm": 1.0906637907028198, + "learning_rate": 0.0001, + "loss": 64.6635, + "step": 3470 + }, + { + "epoch": 56.13008130081301, + "grad_norm": 1.0608748197555542, + "learning_rate": 0.0001, + "loss": 59.8946, + "step": 3480 + }, + { + "epoch": 56.292682926829265, + "grad_norm": 1.025319218635559, + "learning_rate": 0.0001, + "loss": 64.7783, + "step": 3490 + }, + { + "epoch": 56.45528455284553, + "grad_norm": 1.1596784591674805, + "learning_rate": 0.0001, + "loss": 66.4629, + "step": 3500 + }, + { + "epoch": 56.61788617886179, + "grad_norm": 0.9318640232086182, + "learning_rate": 0.0001, + "loss": 65.5533, + "step": 3510 + }, + { + "epoch": 56.78048780487805, + "grad_norm": 1.0977418422698975, + "learning_rate": 0.0001, + "loss": 63.647, + "step": 3520 + }, + { + "epoch": 56.94308943089431, + "grad_norm": 1.3912436962127686, + "learning_rate": 0.0001, + "loss": 62.8273, + "step": 3530 + }, + { + "epoch": 57.09756097560975, + "grad_norm": 1.0921591520309448, + "learning_rate": 0.0001, + "loss": 61.5968, + "step": 3540 + }, + { + "epoch": 57.260162601626014, + "grad_norm": 1.17819344997406, + "learning_rate": 0.0001, + "loss": 64.9381, + "step": 3550 + }, + { + "epoch": 57.422764227642276, + "grad_norm": 1.188829779624939, + "learning_rate": 0.0001, + "loss": 64.6498, + "step": 3560 + }, + { + "epoch": 57.58536585365854, + "grad_norm": 3.664839506149292, + "learning_rate": 0.0001, + "loss": 65.3834, + "step": 3570 + }, + { + "epoch": 57.7479674796748, + "grad_norm": 1.57958984375, + "learning_rate": 0.0001, + "loss": 63.8808, + "step": 3580 + }, + { + "epoch": 57.91056910569106, + "grad_norm": 1.1837552785873413, + "learning_rate": 0.0001, + "loss": 64.1545, + "step": 3590 + }, + { + "epoch": 58.0650406504065, + "grad_norm": 1.5986013412475586, + "learning_rate": 0.0001, + "loss": 60.5742, + "step": 3600 + }, + { + "epoch": 58.22764227642276, + "grad_norm": 1.1551810503005981, + "learning_rate": 0.0001, + "loss": 66.2378, + "step": 3610 + }, + { + "epoch": 58.390243902439025, + "grad_norm": 1.0869863033294678, + "learning_rate": 0.0001, + "loss": 65.3034, + "step": 3620 + }, + { + "epoch": 58.552845528455286, + "grad_norm": 1.2263212203979492, + "learning_rate": 0.0001, + "loss": 63.976, + "step": 3630 + }, + { + "epoch": 58.71544715447155, + "grad_norm": 1.4410817623138428, + "learning_rate": 0.0001, + "loss": 63.4778, + "step": 3640 + }, + { + "epoch": 58.8780487804878, + "grad_norm": 1.9633759260177612, + "learning_rate": 0.0001, + "loss": 65.5946, + "step": 3650 + }, + { + "epoch": 59.03252032520325, + "grad_norm": 1.1729766130447388, + "learning_rate": 0.0001, + "loss": 62.2171, + "step": 3660 + }, + { + "epoch": 59.19512195121951, + "grad_norm": 1.0049556493759155, + "learning_rate": 0.0001, + "loss": 65.1854, + "step": 3670 + }, + { + "epoch": 59.357723577235774, + "grad_norm": 1.1727055311203003, + "learning_rate": 0.0001, + "loss": 65.4839, + "step": 3680 + }, + { + "epoch": 59.520325203252035, + "grad_norm": 1.5394458770751953, + "learning_rate": 0.0001, + "loss": 64.9341, + "step": 3690 + }, + { + "epoch": 59.68292682926829, + "grad_norm": 1.1583774089813232, + "learning_rate": 0.0001, + "loss": 63.9197, + "step": 3700 + }, + { + "epoch": 59.84552845528455, + "grad_norm": 1.0835607051849365, + "learning_rate": 0.0001, + "loss": 64.0494, + "step": 3710 + }, + { + "epoch": 60.0, + "grad_norm": 1.7504246234893799, + "learning_rate": 0.0001, + "loss": 59.4134, + "step": 3720 + }, + { + "epoch": 60.16260162601626, + "grad_norm": 1.1870343685150146, + "learning_rate": 0.0001, + "loss": 63.2158, + "step": 3730 + }, + { + "epoch": 60.32520325203252, + "grad_norm": 1.1983181238174438, + "learning_rate": 0.0001, + "loss": 64.7144, + "step": 3740 + }, + { + "epoch": 60.48780487804878, + "grad_norm": 1.3014581203460693, + "learning_rate": 0.0001, + "loss": 64.6219, + "step": 3750 + }, + { + "epoch": 60.65040650406504, + "grad_norm": 1.4954087734222412, + "learning_rate": 0.0001, + "loss": 63.3785, + "step": 3760 + }, + { + "epoch": 60.8130081300813, + "grad_norm": 1.2236649990081787, + "learning_rate": 0.0001, + "loss": 64.6933, + "step": 3770 + }, + { + "epoch": 60.97560975609756, + "grad_norm": 1.0796610116958618, + "learning_rate": 0.0001, + "loss": 63.2704, + "step": 3780 + }, + { + "epoch": 61.13008130081301, + "grad_norm": 0.9969968795776367, + "learning_rate": 0.0001, + "loss": 62.1838, + "step": 3790 + }, + { + "epoch": 61.292682926829265, + "grad_norm": 1.1624361276626587, + "learning_rate": 0.0001, + "loss": 63.6249, + "step": 3800 + }, + { + "epoch": 61.45528455284553, + "grad_norm": 1.3849223852157593, + "learning_rate": 0.0001, + "loss": 64.5321, + "step": 3810 + }, + { + "epoch": 61.61788617886179, + "grad_norm": 1.6607458591461182, + "learning_rate": 0.0001, + "loss": 64.651, + "step": 3820 + }, + { + "epoch": 61.78048780487805, + "grad_norm": 1.09684419631958, + "learning_rate": 0.0001, + "loss": 64.1129, + "step": 3830 + }, + { + "epoch": 61.94308943089431, + "grad_norm": 1.406246304512024, + "learning_rate": 0.0001, + "loss": 64.3642, + "step": 3840 + }, + { + "epoch": 62.09756097560975, + "grad_norm": 0.9850211143493652, + "learning_rate": 0.0001, + "loss": 60.8703, + "step": 3850 + }, + { + "epoch": 62.260162601626014, + "grad_norm": 1.001344084739685, + "learning_rate": 0.0001, + "loss": 67.5523, + "step": 3860 + }, + { + "epoch": 62.422764227642276, + "grad_norm": 2.435969591140747, + "learning_rate": 0.0001, + "loss": 62.7363, + "step": 3870 + }, + { + "epoch": 62.58536585365854, + "grad_norm": 1.0675824880599976, + "learning_rate": 0.0001, + "loss": 64.8513, + "step": 3880 + }, + { + "epoch": 62.7479674796748, + "grad_norm": 0.9793754816055298, + "learning_rate": 0.0001, + "loss": 64.9904, + "step": 3890 + }, + { + "epoch": 62.91056910569106, + "grad_norm": 1.2187775373458862, + "learning_rate": 0.0001, + "loss": 63.2821, + "step": 3900 + }, + { + "epoch": 63.0650406504065, + "grad_norm": 1.130772590637207, + "learning_rate": 0.0001, + "loss": 61.5957, + "step": 3910 + }, + { + "epoch": 63.22764227642276, + "grad_norm": 1.0573055744171143, + "learning_rate": 0.0001, + "loss": 63.6337, + "step": 3920 + }, + { + "epoch": 63.390243902439025, + "grad_norm": 2.055490255355835, + "learning_rate": 0.0001, + "loss": 63.9598, + "step": 3930 + }, + { + "epoch": 63.552845528455286, + "grad_norm": 1.0780516862869263, + "learning_rate": 0.0001, + "loss": 62.9303, + "step": 3940 + }, + { + "epoch": 63.71544715447155, + "grad_norm": 1.8136590719223022, + "learning_rate": 0.0001, + "loss": 64.4588, + "step": 3950 + }, + { + "epoch": 63.8780487804878, + "grad_norm": 1.0525016784667969, + "learning_rate": 0.0001, + "loss": 64.192, + "step": 3960 + }, + { + "epoch": 64.03252032520325, + "grad_norm": 1.12627375125885, + "learning_rate": 0.0001, + "loss": 59.1823, + "step": 3970 + }, + { + "epoch": 64.1951219512195, + "grad_norm": 1.2558730840682983, + "learning_rate": 0.0001, + "loss": 65.126, + "step": 3980 + }, + { + "epoch": 64.35772357723577, + "grad_norm": 1.2304726839065552, + "learning_rate": 0.0001, + "loss": 62.0868, + "step": 3990 + }, + { + "epoch": 64.52032520325203, + "grad_norm": 1.4179201126098633, + "learning_rate": 0.0001, + "loss": 62.3114, + "step": 4000 + }, + { + "epoch": 64.6829268292683, + "grad_norm": 1.125758409500122, + "learning_rate": 0.0001, + "loss": 63.4103, + "step": 4010 + }, + { + "epoch": 64.84552845528455, + "grad_norm": 2.321124315261841, + "learning_rate": 0.0001, + "loss": 62.9032, + "step": 4020 + }, + { + "epoch": 65.0, + "grad_norm": 1.4433993101119995, + "learning_rate": 0.0001, + "loss": 61.8766, + "step": 4030 + }, + { + "epoch": 65.16260162601625, + "grad_norm": 1.157878041267395, + "learning_rate": 0.0001, + "loss": 62.4833, + "step": 4040 + }, + { + "epoch": 65.32520325203252, + "grad_norm": 1.4127166271209717, + "learning_rate": 0.0001, + "loss": 64.768, + "step": 4050 + }, + { + "epoch": 65.48780487804878, + "grad_norm": 1.2575284242630005, + "learning_rate": 0.0001, + "loss": 62.9401, + "step": 4060 + }, + { + "epoch": 65.65040650406505, + "grad_norm": 1.0957911014556885, + "learning_rate": 0.0001, + "loss": 64.6061, + "step": 4070 + }, + { + "epoch": 65.8130081300813, + "grad_norm": 1.1773414611816406, + "learning_rate": 0.0001, + "loss": 63.9469, + "step": 4080 + }, + { + "epoch": 65.97560975609755, + "grad_norm": 2.1189942359924316, + "learning_rate": 0.0001, + "loss": 65.1422, + "step": 4090 + }, + { + "epoch": 66.130081300813, + "grad_norm": 1.1064234972000122, + "learning_rate": 0.0001, + "loss": 59.6108, + "step": 4100 + }, + { + "epoch": 66.29268292682927, + "grad_norm": 1.1135871410369873, + "learning_rate": 0.0001, + "loss": 63.4036, + "step": 4110 + }, + { + "epoch": 66.45528455284553, + "grad_norm": 4.167781829833984, + "learning_rate": 0.0001, + "loss": 63.1174, + "step": 4120 + }, + { + "epoch": 66.6178861788618, + "grad_norm": 1.1429376602172852, + "learning_rate": 0.0001, + "loss": 64.4085, + "step": 4130 + }, + { + "epoch": 66.78048780487805, + "grad_norm": 1.3188339471817017, + "learning_rate": 0.0001, + "loss": 62.5997, + "step": 4140 + }, + { + "epoch": 66.9430894308943, + "grad_norm": 1.160335659980774, + "learning_rate": 0.0001, + "loss": 64.2911, + "step": 4150 + }, + { + "epoch": 67.09756097560975, + "grad_norm": 0.9261873960494995, + "learning_rate": 0.0001, + "loss": 61.9036, + "step": 4160 + }, + { + "epoch": 67.26016260162602, + "grad_norm": 1.2254258394241333, + "learning_rate": 0.0001, + "loss": 64.1092, + "step": 4170 + }, + { + "epoch": 67.42276422764228, + "grad_norm": 0.9519097208976746, + "learning_rate": 0.0001, + "loss": 61.6428, + "step": 4180 + }, + { + "epoch": 67.58536585365853, + "grad_norm": 1.2290887832641602, + "learning_rate": 0.0001, + "loss": 64.9842, + "step": 4190 + }, + { + "epoch": 67.7479674796748, + "grad_norm": 1.0995723009109497, + "learning_rate": 0.0001, + "loss": 64.0677, + "step": 4200 + }, + { + "epoch": 67.91056910569105, + "grad_norm": 2.727957010269165, + "learning_rate": 0.0001, + "loss": 62.624, + "step": 4210 + }, + { + "epoch": 68.0650406504065, + "grad_norm": 1.1222411394119263, + "learning_rate": 0.0001, + "loss": 61.0693, + "step": 4220 + }, + { + "epoch": 68.22764227642277, + "grad_norm": 1.159303069114685, + "learning_rate": 0.0001, + "loss": 64.2469, + "step": 4230 + }, + { + "epoch": 68.39024390243902, + "grad_norm": 1.2205990552902222, + "learning_rate": 0.0001, + "loss": 63.8831, + "step": 4240 + }, + { + "epoch": 68.55284552845528, + "grad_norm": 1.2351205348968506, + "learning_rate": 0.0001, + "loss": 63.4294, + "step": 4250 + }, + { + "epoch": 68.71544715447155, + "grad_norm": 1.4483757019042969, + "learning_rate": 0.0001, + "loss": 63.5345, + "step": 4260 + }, + { + "epoch": 68.8780487804878, + "grad_norm": 1.0808624029159546, + "learning_rate": 0.0001, + "loss": 63.6885, + "step": 4270 + }, + { + "epoch": 69.03252032520325, + "grad_norm": 1.7775999307632446, + "learning_rate": 0.0001, + "loss": 60.7266, + "step": 4280 + }, + { + "epoch": 69.1951219512195, + "grad_norm": 1.3214768171310425, + "learning_rate": 0.0001, + "loss": 63.3496, + "step": 4290 + }, + { + "epoch": 69.35772357723577, + "grad_norm": 1.1698002815246582, + "learning_rate": 0.0001, + "loss": 63.5284, + "step": 4300 + }, + { + "epoch": 69.52032520325203, + "grad_norm": 1.2050056457519531, + "learning_rate": 0.0001, + "loss": 64.992, + "step": 4310 + }, + { + "epoch": 69.6829268292683, + "grad_norm": 1.0873243808746338, + "learning_rate": 0.0001, + "loss": 65.1709, + "step": 4320 + }, + { + "epoch": 69.84552845528455, + "grad_norm": 1.14133620262146, + "learning_rate": 0.0001, + "loss": 63.1872, + "step": 4330 + }, + { + "epoch": 70.0, + "grad_norm": 0.9684088230133057, + "learning_rate": 0.0001, + "loss": 60.1486, + "step": 4340 + }, + { + "epoch": 70.16260162601625, + "grad_norm": 1.1206653118133545, + "learning_rate": 0.0001, + "loss": 61.117, + "step": 4350 + }, + { + "epoch": 70.32520325203252, + "grad_norm": 1.2754334211349487, + "learning_rate": 0.0001, + "loss": 62.5409, + "step": 4360 + }, + { + "epoch": 70.48780487804878, + "grad_norm": 1.0154718160629272, + "learning_rate": 0.0001, + "loss": 62.9428, + "step": 4370 + }, + { + "epoch": 70.65040650406505, + "grad_norm": 1.0597270727157593, + "learning_rate": 0.0001, + "loss": 63.079, + "step": 4380 + }, + { + "epoch": 70.8130081300813, + "grad_norm": 1.254758596420288, + "learning_rate": 0.0001, + "loss": 63.3035, + "step": 4390 + }, + { + "epoch": 70.97560975609755, + "grad_norm": 1.0185120105743408, + "learning_rate": 0.0001, + "loss": 63.8922, + "step": 4400 + }, + { + "epoch": 71.130081300813, + "grad_norm": 1.007300853729248, + "learning_rate": 0.0001, + "loss": 58.7735, + "step": 4410 + }, + { + "epoch": 71.29268292682927, + "grad_norm": 1.2012463808059692, + "learning_rate": 0.0001, + "loss": 63.1782, + "step": 4420 + }, + { + "epoch": 71.45528455284553, + "grad_norm": 1.0258845090866089, + "learning_rate": 0.0001, + "loss": 63.2483, + "step": 4430 + }, + { + "epoch": 71.6178861788618, + "grad_norm": 1.0611220598220825, + "learning_rate": 0.0001, + "loss": 63.2247, + "step": 4440 + }, + { + "epoch": 71.78048780487805, + "grad_norm": 1.060114860534668, + "learning_rate": 0.0001, + "loss": 61.8352, + "step": 4450 + }, + { + "epoch": 71.9430894308943, + "grad_norm": 1.0540589094161987, + "learning_rate": 0.0001, + "loss": 62.1156, + "step": 4460 + }, + { + "epoch": 72.09756097560975, + "grad_norm": 1.4923726320266724, + "learning_rate": 0.0001, + "loss": 59.3863, + "step": 4470 + }, + { + "epoch": 72.26016260162602, + "grad_norm": 1.13791024684906, + "learning_rate": 0.0001, + "loss": 62.203, + "step": 4480 + }, + { + "epoch": 72.42276422764228, + "grad_norm": 0.9737065434455872, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 4490 + }, + { + "epoch": 72.58536585365853, + "grad_norm": 1.3980190753936768, + "learning_rate": 0.0001, + "loss": 63.5627, + "step": 4500 + }, + { + "epoch": 72.7479674796748, + "grad_norm": 1.0274118185043335, + "learning_rate": 0.0001, + "loss": 63.6129, + "step": 4510 + }, + { + "epoch": 72.91056910569105, + "grad_norm": 1.3148925304412842, + "learning_rate": 0.0001, + "loss": 63.6506, + "step": 4520 + }, + { + "epoch": 73.0650406504065, + "grad_norm": 1.4182451963424683, + "learning_rate": 0.0001, + "loss": 60.3616, + "step": 4530 + }, + { + "epoch": 73.22764227642277, + "grad_norm": 0.9235166907310486, + "learning_rate": 0.0001, + "loss": 63.6465, + "step": 4540 + }, + { + "epoch": 73.39024390243902, + "grad_norm": 0.9293469786643982, + "learning_rate": 0.0001, + "loss": 61.0984, + "step": 4550 + }, + { + "epoch": 73.55284552845528, + "grad_norm": 1.1075892448425293, + "learning_rate": 0.0001, + "loss": 62.9356, + "step": 4560 + }, + { + "epoch": 73.71544715447155, + "grad_norm": 1.4955459833145142, + "learning_rate": 0.0001, + "loss": 61.3811, + "step": 4570 + }, + { + "epoch": 73.8780487804878, + "grad_norm": 1.220595359802246, + "learning_rate": 0.0001, + "loss": 62.4364, + "step": 4580 + }, + { + "epoch": 74.03252032520325, + "grad_norm": 0.9975704550743103, + "learning_rate": 0.0001, + "loss": 58.5194, + "step": 4590 + }, + { + "epoch": 74.1951219512195, + "grad_norm": 1.0160857439041138, + "learning_rate": 0.0001, + "loss": 64.6989, + "step": 4600 + }, + { + "epoch": 74.35772357723577, + "grad_norm": 1.0052287578582764, + "learning_rate": 0.0001, + "loss": 62.526, + "step": 4610 + }, + { + "epoch": 74.52032520325203, + "grad_norm": 1.015514612197876, + "learning_rate": 0.0001, + "loss": 63.8627, + "step": 4620 + }, + { + "epoch": 74.6829268292683, + "grad_norm": 1.139051914215088, + "learning_rate": 0.0001, + "loss": 62.4525, + "step": 4630 + }, + { + "epoch": 74.84552845528455, + "grad_norm": 1.1727714538574219, + "learning_rate": 0.0001, + "loss": 64.0046, + "step": 4640 + }, + { + "epoch": 75.0, + "grad_norm": 1.1611062288284302, + "learning_rate": 0.0001, + "loss": 57.6443, + "step": 4650 + }, + { + "epoch": 75.16260162601625, + "grad_norm": 0.9893108606338501, + "learning_rate": 0.0001, + "loss": 64.1109, + "step": 4660 + }, + { + "epoch": 75.32520325203252, + "grad_norm": 1.190146803855896, + "learning_rate": 0.0001, + "loss": 62.1988, + "step": 4670 + }, + { + "epoch": 75.48780487804878, + "grad_norm": 1.1648579835891724, + "learning_rate": 0.0001, + "loss": 63.1393, + "step": 4680 + }, + { + "epoch": 75.65040650406505, + "grad_norm": 1.3303166627883911, + "learning_rate": 0.0001, + "loss": 63.7, + "step": 4690 + }, + { + "epoch": 75.8130081300813, + "grad_norm": 1.407772421836853, + "learning_rate": 0.0001, + "loss": 60.6798, + "step": 4700 + }, + { + "epoch": 75.97560975609755, + "grad_norm": 1.039741039276123, + "learning_rate": 0.0001, + "loss": 61.6126, + "step": 4710 + }, + { + "epoch": 76.130081300813, + "grad_norm": 1.013515830039978, + "learning_rate": 0.0001, + "loss": 57.8452, + "step": 4720 + }, + { + "epoch": 76.29268292682927, + "grad_norm": 1.1279789209365845, + "learning_rate": 0.0001, + "loss": 63.0592, + "step": 4730 + }, + { + "epoch": 76.45528455284553, + "grad_norm": 1.158605694770813, + "learning_rate": 0.0001, + "loss": 62.0836, + "step": 4740 + }, + { + "epoch": 76.6178861788618, + "grad_norm": 1.0200021266937256, + "learning_rate": 0.0001, + "loss": 62.1775, + "step": 4750 + }, + { + "epoch": 76.78048780487805, + "grad_norm": 1.216412901878357, + "learning_rate": 0.0001, + "loss": 62.6316, + "step": 4760 + }, + { + "epoch": 76.9430894308943, + "grad_norm": 1.0513523817062378, + "learning_rate": 0.0001, + "loss": 60.1037, + "step": 4770 + }, + { + "epoch": 77.09756097560975, + "grad_norm": 1.10075843334198, + "learning_rate": 0.0001, + "loss": 58.7599, + "step": 4780 + }, + { + "epoch": 77.26016260162602, + "grad_norm": 1.1728724241256714, + "learning_rate": 0.0001, + "loss": 63.3476, + "step": 4790 + }, + { + "epoch": 77.42276422764228, + "grad_norm": 1.3511605262756348, + "learning_rate": 0.0001, + "loss": 60.9885, + "step": 4800 + }, + { + "epoch": 77.58536585365853, + "grad_norm": 0.9733042120933533, + "learning_rate": 0.0001, + "loss": 62.4245, + "step": 4810 + }, + { + "epoch": 77.7479674796748, + "grad_norm": 1.0380535125732422, + "learning_rate": 0.0001, + "loss": 64.0755, + "step": 4820 + }, + { + "epoch": 77.91056910569105, + "grad_norm": 1.1594985723495483, + "learning_rate": 0.0001, + "loss": 62.1099, + "step": 4830 + }, + { + "epoch": 78.0650406504065, + "grad_norm": 1.0392569303512573, + "learning_rate": 0.0001, + "loss": 58.3778, + "step": 4840 + }, + { + "epoch": 78.22764227642277, + "grad_norm": 1.0032405853271484, + "learning_rate": 0.0001, + "loss": 62.1384, + "step": 4850 + }, + { + "epoch": 78.39024390243902, + "grad_norm": 1.0576013326644897, + "learning_rate": 0.0001, + "loss": 63.3308, + "step": 4860 + }, + { + "epoch": 78.55284552845528, + "grad_norm": 1.34328031539917, + "learning_rate": 0.0001, + "loss": 62.5351, + "step": 4870 + }, + { + "epoch": 78.71544715447155, + "grad_norm": 1.1040822267532349, + "learning_rate": 0.0001, + "loss": 61.5974, + "step": 4880 + }, + { + "epoch": 78.8780487804878, + "grad_norm": 1.1554768085479736, + "learning_rate": 0.0001, + "loss": 62.7308, + "step": 4890 + }, + { + "epoch": 79.03252032520325, + "grad_norm": 1.1864408254623413, + "learning_rate": 0.0001, + "loss": 60.0741, + "step": 4900 + }, + { + "epoch": 79.1951219512195, + "grad_norm": 0.9181821346282959, + "learning_rate": 0.0001, + "loss": 63.1682, + "step": 4910 + }, + { + "epoch": 79.35772357723577, + "grad_norm": 1.1400946378707886, + "learning_rate": 0.0001, + "loss": 62.0736, + "step": 4920 + }, + { + "epoch": 79.52032520325203, + "grad_norm": 1.245202660560608, + "learning_rate": 0.0001, + "loss": 60.51, + "step": 4930 + }, + { + "epoch": 79.6829268292683, + "grad_norm": 1.033347725868225, + "learning_rate": 0.0001, + "loss": 61.7845, + "step": 4940 + }, + { + "epoch": 79.84552845528455, + "grad_norm": 1.6206163167953491, + "learning_rate": 0.0001, + "loss": 61.5232, + "step": 4950 + }, + { + "epoch": 80.0, + "grad_norm": 1.0924937725067139, + "learning_rate": 0.0001, + "loss": 57.24, + "step": 4960 + }, + { + "epoch": 80.16260162601625, + "grad_norm": 1.075985074043274, + "learning_rate": 0.0001, + "loss": 61.2615, + "step": 4970 + }, + { + "epoch": 80.32520325203252, + "grad_norm": 0.9964771866798401, + "learning_rate": 0.0001, + "loss": 62.287, + "step": 4980 + }, + { + "epoch": 80.48780487804878, + "grad_norm": 1.3707735538482666, + "learning_rate": 0.0001, + "loss": 60.9383, + "step": 4990 + }, + { + "epoch": 80.65040650406505, + "grad_norm": 0.9395170211791992, + "learning_rate": 0.0001, + "loss": 61.2697, + "step": 5000 + }, + { + "epoch": 80.8130081300813, + "grad_norm": 2.0208873748779297, + "learning_rate": 0.0001, + "loss": 61.7219, + "step": 5010 + }, + { + "epoch": 80.97560975609755, + "grad_norm": 1.468702793121338, + "learning_rate": 0.0001, + "loss": 62.6967, + "step": 5020 + }, + { + "epoch": 81.130081300813, + "grad_norm": 1.0525966882705688, + "learning_rate": 0.0001, + "loss": 59.9793, + "step": 5030 + }, + { + "epoch": 81.29268292682927, + "grad_norm": 1.0951132774353027, + "learning_rate": 0.0001, + "loss": 60.9807, + "step": 5040 + }, + { + "epoch": 81.45528455284553, + "grad_norm": 1.3457857370376587, + "learning_rate": 0.0001, + "loss": 61.4137, + "step": 5050 + }, + { + "epoch": 81.6178861788618, + "grad_norm": 1.0895649194717407, + "learning_rate": 0.0001, + "loss": 61.2747, + "step": 5060 + }, + { + "epoch": 81.78048780487805, + "grad_norm": 1.0552371740341187, + "learning_rate": 0.0001, + "loss": 62.0947, + "step": 5070 + }, + { + "epoch": 81.9430894308943, + "grad_norm": 1.039038896560669, + "learning_rate": 0.0001, + "loss": 61.2571, + "step": 5080 + }, + { + "epoch": 82.09756097560975, + "grad_norm": 0.970783531665802, + "learning_rate": 0.0001, + "loss": 57.4157, + "step": 5090 + }, + { + "epoch": 82.26016260162602, + "grad_norm": 1.0276020765304565, + "learning_rate": 0.0001, + "loss": 62.1324, + "step": 5100 + }, + { + "epoch": 82.42276422764228, + "grad_norm": 1.2092448472976685, + "learning_rate": 0.0001, + "loss": 60.6448, + "step": 5110 + }, + { + "epoch": 82.58536585365853, + "grad_norm": 0.9352007508277893, + "learning_rate": 0.0001, + "loss": 61.9299, + "step": 5120 + }, + { + "epoch": 82.7479674796748, + "grad_norm": 1.2678358554840088, + "learning_rate": 0.0001, + "loss": 61.6622, + "step": 5130 + }, + { + "epoch": 82.91056910569105, + "grad_norm": 1.200494647026062, + "learning_rate": 0.0001, + "loss": 62.3169, + "step": 5140 + }, + { + "epoch": 83.0650406504065, + "grad_norm": 1.1284401416778564, + "learning_rate": 0.0001, + "loss": 58.7768, + "step": 5150 + }, + { + "epoch": 83.22764227642277, + "grad_norm": 1.1578079462051392, + "learning_rate": 0.0001, + "loss": 62.8423, + "step": 5160 + }, + { + "epoch": 83.39024390243902, + "grad_norm": 1.3426504135131836, + "learning_rate": 0.0001, + "loss": 61.5332, + "step": 5170 + }, + { + "epoch": 83.55284552845528, + "grad_norm": 0.9397301077842712, + "learning_rate": 0.0001, + "loss": 60.1043, + "step": 5180 + }, + { + "epoch": 83.71544715447155, + "grad_norm": 1.0728143453598022, + "learning_rate": 0.0001, + "loss": 60.5927, + "step": 5190 + }, + { + "epoch": 83.8780487804878, + "grad_norm": 1.2620211839675903, + "learning_rate": 0.0001, + "loss": 60.9224, + "step": 5200 + }, + { + "epoch": 84.03252032520325, + "grad_norm": 0.8232231140136719, + "learning_rate": 0.0001, + "loss": 58.8128, + "step": 5210 + }, + { + "epoch": 84.1951219512195, + "grad_norm": 1.0222564935684204, + "learning_rate": 0.0001, + "loss": 59.6995, + "step": 5220 + }, + { + "epoch": 84.35772357723577, + "grad_norm": 0.9078415632247925, + "learning_rate": 0.0001, + "loss": 61.1521, + "step": 5230 + }, + { + "epoch": 84.52032520325203, + "grad_norm": 1.0269399881362915, + "learning_rate": 0.0001, + "loss": 61.0136, + "step": 5240 + }, + { + "epoch": 84.6829268292683, + "grad_norm": 1.2108154296875, + "learning_rate": 0.0001, + "loss": 61.3564, + "step": 5250 + }, + { + "epoch": 84.84552845528455, + "grad_norm": 1.1170622110366821, + "learning_rate": 0.0001, + "loss": 61.7047, + "step": 5260 + }, + { + "epoch": 85.0, + "grad_norm": 1.467298150062561, + "learning_rate": 0.0001, + "loss": 58.1667, + "step": 5270 + }, + { + "epoch": 85.16260162601625, + "grad_norm": 0.908743143081665, + "learning_rate": 0.0001, + "loss": 61.5694, + "step": 5280 + }, + { + "epoch": 85.32520325203252, + "grad_norm": 0.9148766398429871, + "learning_rate": 0.0001, + "loss": 61.5325, + "step": 5290 + }, + { + "epoch": 85.48780487804878, + "grad_norm": 0.940021276473999, + "learning_rate": 0.0001, + "loss": 62.8807, + "step": 5300 + }, + { + "epoch": 85.65040650406505, + "grad_norm": 1.1239454746246338, + "learning_rate": 0.0001, + "loss": 62.361, + "step": 5310 + }, + { + "epoch": 85.8130081300813, + "grad_norm": 0.840437114238739, + "learning_rate": 0.0001, + "loss": 61.9057, + "step": 5320 + }, + { + "epoch": 85.97560975609755, + "grad_norm": 0.9764263033866882, + "learning_rate": 0.0001, + "loss": 61.0075, + "step": 5330 + }, + { + "epoch": 86.130081300813, + "grad_norm": 0.998540997505188, + "learning_rate": 0.0001, + "loss": 57.7146, + "step": 5340 + }, + { + "epoch": 86.29268292682927, + "grad_norm": 1.121551513671875, + "learning_rate": 0.0001, + "loss": 62.2278, + "step": 5350 + }, + { + "epoch": 86.45528455284553, + "grad_norm": 1.101842999458313, + "learning_rate": 0.0001, + "loss": 60.8837, + "step": 5360 + }, + { + "epoch": 86.6178861788618, + "grad_norm": 1.1724908351898193, + "learning_rate": 0.0001, + "loss": 61.6788, + "step": 5370 + }, + { + "epoch": 86.78048780487805, + "grad_norm": 1.3108958005905151, + "learning_rate": 0.0001, + "loss": 60.1461, + "step": 5380 + }, + { + "epoch": 86.9430894308943, + "grad_norm": 1.482298731803894, + "learning_rate": 0.0001, + "loss": 61.0457, + "step": 5390 + }, + { + "epoch": 87.09756097560975, + "grad_norm": 1.1471034288406372, + "learning_rate": 0.0001, + "loss": 57.6835, + "step": 5400 + }, + { + "epoch": 87.26016260162602, + "grad_norm": 0.9349374175071716, + "learning_rate": 0.0001, + "loss": 62.7018, + "step": 5410 + }, + { + "epoch": 87.42276422764228, + "grad_norm": 0.9917856454849243, + "learning_rate": 0.0001, + "loss": 60.2201, + "step": 5420 + }, + { + "epoch": 87.58536585365853, + "grad_norm": 1.0112310647964478, + "learning_rate": 0.0001, + "loss": 62.1185, + "step": 5430 + }, + { + "epoch": 87.7479674796748, + "grad_norm": 1.288894534111023, + "learning_rate": 0.0001, + "loss": 61.4348, + "step": 5440 + }, + { + "epoch": 87.91056910569105, + "grad_norm": 0.8577322363853455, + "learning_rate": 0.0001, + "loss": 60.7867, + "step": 5450 + }, + { + "epoch": 88.0650406504065, + "grad_norm": 0.8162860870361328, + "learning_rate": 0.0001, + "loss": 59.5224, + "step": 5460 + }, + { + "epoch": 88.22764227642277, + "grad_norm": 0.9491421580314636, + "learning_rate": 0.0001, + "loss": 60.4936, + "step": 5470 + }, + { + "epoch": 88.39024390243902, + "grad_norm": 4.164788246154785, + "learning_rate": 0.0001, + "loss": 61.8397, + "step": 5480 + }, + { + "epoch": 88.55284552845528, + "grad_norm": 0.9954016804695129, + "learning_rate": 0.0001, + "loss": 61.286, + "step": 5490 + }, + { + "epoch": 88.71544715447155, + "grad_norm": 1.4991120100021362, + "learning_rate": 0.0001, + "loss": 60.6157, + "step": 5500 + }, + { + "epoch": 88.8780487804878, + "grad_norm": 1.3678480386734009, + "learning_rate": 0.0001, + "loss": 62.5187, + "step": 5510 + }, + { + "epoch": 89.03252032520325, + "grad_norm": 1.057661771774292, + "learning_rate": 0.0001, + "loss": 57.5134, + "step": 5520 + }, + { + "epoch": 89.1951219512195, + "grad_norm": 0.8331280946731567, + "learning_rate": 0.0001, + "loss": 62.007, + "step": 5530 + }, + { + "epoch": 89.35772357723577, + "grad_norm": 0.9842398762702942, + "learning_rate": 0.0001, + "loss": 60.6178, + "step": 5540 + }, + { + "epoch": 89.52032520325203, + "grad_norm": 1.0510483980178833, + "learning_rate": 0.0001, + "loss": 59.6382, + "step": 5550 + }, + { + "epoch": 89.6829268292683, + "grad_norm": 1.013115406036377, + "learning_rate": 0.0001, + "loss": 61.015, + "step": 5560 + }, + { + "epoch": 89.84552845528455, + "grad_norm": 1.1595944166183472, + "learning_rate": 0.0001, + "loss": 62.5374, + "step": 5570 + }, + { + "epoch": 90.0, + "grad_norm": 1.1506270170211792, + "learning_rate": 0.0001, + "loss": 61.0063, + "step": 5580 + }, + { + "epoch": 90.16260162601625, + "grad_norm": 0.9940153360366821, + "learning_rate": 0.0001, + "loss": 62.0944, + "step": 5590 + }, + { + "epoch": 90.32520325203252, + "grad_norm": 1.1302493810653687, + "learning_rate": 0.0001, + "loss": 61.616, + "step": 5600 + }, + { + "epoch": 90.48780487804878, + "grad_norm": 1.2370656728744507, + "learning_rate": 0.0001, + "loss": 61.291, + "step": 5610 + }, + { + "epoch": 90.65040650406505, + "grad_norm": 1.228763461112976, + "learning_rate": 0.0001, + "loss": 60.0629, + "step": 5620 + }, + { + "epoch": 90.8130081300813, + "grad_norm": 1.2230069637298584, + "learning_rate": 0.0001, + "loss": 61.5391, + "step": 5630 + }, + { + "epoch": 90.97560975609755, + "grad_norm": 0.9849008917808533, + "learning_rate": 0.0001, + "loss": 63.1633, + "step": 5640 + }, + { + "epoch": 91.130081300813, + "grad_norm": 1.408632755279541, + "learning_rate": 0.0001, + "loss": 58.4195, + "step": 5650 + }, + { + "epoch": 91.29268292682927, + "grad_norm": 0.929027795791626, + "learning_rate": 0.0001, + "loss": 60.9862, + "step": 5660 + }, + { + "epoch": 91.45528455284553, + "grad_norm": 0.9228377342224121, + "learning_rate": 0.0001, + "loss": 60.8337, + "step": 5670 + }, + { + "epoch": 91.6178861788618, + "grad_norm": 1.0354220867156982, + "learning_rate": 0.0001, + "loss": 59.4667, + "step": 5680 + }, + { + "epoch": 91.78048780487805, + "grad_norm": 1.0338927507400513, + "learning_rate": 0.0001, + "loss": 62.0064, + "step": 5690 + }, + { + "epoch": 91.9430894308943, + "grad_norm": 1.101967215538025, + "learning_rate": 0.0001, + "loss": 61.6317, + "step": 5700 + }, + { + "epoch": 92.09756097560975, + "grad_norm": 1.370689034461975, + "learning_rate": 0.0001, + "loss": 57.2157, + "step": 5710 + }, + { + "epoch": 92.26016260162602, + "grad_norm": 0.9904115796089172, + "learning_rate": 0.0001, + "loss": 61.2786, + "step": 5720 + }, + { + "epoch": 92.42276422764228, + "grad_norm": 1.0545448064804077, + "learning_rate": 0.0001, + "loss": 63.2384, + "step": 5730 + }, + { + "epoch": 92.58536585365853, + "grad_norm": 1.0657731294631958, + "learning_rate": 0.0001, + "loss": 60.582, + "step": 5740 + }, + { + "epoch": 92.7479674796748, + "grad_norm": 0.9333183765411377, + "learning_rate": 0.0001, + "loss": 62.4627, + "step": 5750 + }, + { + "epoch": 92.91056910569105, + "grad_norm": 0.9810492396354675, + "learning_rate": 0.0001, + "loss": 60.0055, + "step": 5760 + }, + { + "epoch": 93.0650406504065, + "grad_norm": 1.6973884105682373, + "learning_rate": 0.0001, + "loss": 57.5381, + "step": 5770 + }, + { + "epoch": 93.22764227642277, + "grad_norm": 1.0075498819351196, + "learning_rate": 0.0001, + "loss": 61.2272, + "step": 5780 + }, + { + "epoch": 93.39024390243902, + "grad_norm": 0.980633556842804, + "learning_rate": 0.0001, + "loss": 60.4794, + "step": 5790 + }, + { + "epoch": 93.55284552845528, + "grad_norm": 1.128214955329895, + "learning_rate": 0.0001, + "loss": 60.4825, + "step": 5800 + }, + { + "epoch": 93.71544715447155, + "grad_norm": 1.302827000617981, + "learning_rate": 0.0001, + "loss": 62.4397, + "step": 5810 + }, + { + "epoch": 93.8780487804878, + "grad_norm": 0.826600193977356, + "learning_rate": 0.0001, + "loss": 61.1254, + "step": 5820 + }, + { + "epoch": 94.03252032520325, + "grad_norm": 11.418835639953613, + "learning_rate": 0.0001, + "loss": 58.5383, + "step": 5830 + }, + { + "epoch": 94.1951219512195, + "grad_norm": 1.0686477422714233, + "learning_rate": 0.0001, + "loss": 60.5696, + "step": 5840 + }, + { + "epoch": 94.35772357723577, + "grad_norm": 1.1232597827911377, + "learning_rate": 0.0001, + "loss": 62.4849, + "step": 5850 + }, + { + "epoch": 94.52032520325203, + "grad_norm": 1.1051944494247437, + "learning_rate": 0.0001, + "loss": 60.5387, + "step": 5860 + }, + { + "epoch": 94.6829268292683, + "grad_norm": 0.9954739809036255, + "learning_rate": 0.0001, + "loss": 61.3899, + "step": 5870 + }, + { + "epoch": 94.84552845528455, + "grad_norm": 1.9687999486923218, + "learning_rate": 0.0001, + "loss": 60.7711, + "step": 5880 + }, + { + "epoch": 95.0, + "grad_norm": 1.3172541856765747, + "learning_rate": 0.0001, + "loss": 58.5394, + "step": 5890 + }, + { + "epoch": 95.16260162601625, + "grad_norm": 1.2251427173614502, + "learning_rate": 0.0001, + "loss": 61.8319, + "step": 5900 + }, + { + "epoch": 95.32520325203252, + "grad_norm": 1.1701544523239136, + "learning_rate": 0.0001, + "loss": 60.1928, + "step": 5910 + }, + { + "epoch": 95.48780487804878, + "grad_norm": 1.209375262260437, + "learning_rate": 0.0001, + "loss": 61.1347, + "step": 5920 + }, + { + "epoch": 95.65040650406505, + "grad_norm": 0.9307044148445129, + "learning_rate": 0.0001, + "loss": 62.6676, + "step": 5930 + }, + { + "epoch": 95.8130081300813, + "grad_norm": 0.9679144024848938, + "learning_rate": 0.0001, + "loss": 60.7623, + "step": 5940 + }, + { + "epoch": 95.97560975609755, + "grad_norm": 1.398711085319519, + "learning_rate": 0.0001, + "loss": 59.0407, + "step": 5950 + }, + { + "epoch": 96.130081300813, + "grad_norm": 1.9118247032165527, + "learning_rate": 0.0001, + "loss": 59.9417, + "step": 5960 + }, + { + "epoch": 96.29268292682927, + "grad_norm": 1.036737084388733, + "learning_rate": 0.0001, + "loss": 60.6891, + "step": 5970 + }, + { + "epoch": 96.45528455284553, + "grad_norm": 1.5916132926940918, + "learning_rate": 0.0001, + "loss": 59.44, + "step": 5980 + }, + { + "epoch": 96.6178861788618, + "grad_norm": 1.526591181755066, + "learning_rate": 0.0001, + "loss": 58.9081, + "step": 5990 + }, + { + "epoch": 96.78048780487805, + "grad_norm": 0.9556435942649841, + "learning_rate": 0.0001, + "loss": 61.5743, + "step": 6000 + }, + { + "epoch": 96.9430894308943, + "grad_norm": 1.0041887760162354, + "learning_rate": 0.0001, + "loss": 60.0724, + "step": 6010 + }, + { + "epoch": 97.09756097560975, + "grad_norm": 0.9792081117630005, + "learning_rate": 0.0001, + "loss": 60.0613, + "step": 6020 + }, + { + "epoch": 97.26016260162602, + "grad_norm": 1.029634714126587, + "learning_rate": 0.0001, + "loss": 60.3526, + "step": 6030 + }, + { + "epoch": 97.42276422764228, + "grad_norm": 0.9812865257263184, + "learning_rate": 0.0001, + "loss": 60.9818, + "step": 6040 + }, + { + "epoch": 97.58536585365853, + "grad_norm": 0.8826327323913574, + "learning_rate": 0.0001, + "loss": 61.6375, + "step": 6050 + }, + { + "epoch": 97.7479674796748, + "grad_norm": 1.2197599411010742, + "learning_rate": 0.0001, + "loss": 60.2698, + "step": 6060 + }, + { + "epoch": 97.91056910569105, + "grad_norm": 0.9068922400474548, + "learning_rate": 0.0001, + "loss": 58.8698, + "step": 6070 + }, + { + "epoch": 98.0650406504065, + "grad_norm": 0.9008221626281738, + "learning_rate": 0.0001, + "loss": 56.4687, + "step": 6080 + }, + { + "epoch": 98.22764227642277, + "grad_norm": 1.8165878057479858, + "learning_rate": 0.0001, + "loss": 61.4213, + "step": 6090 + }, + { + "epoch": 98.39024390243902, + "grad_norm": 0.9541304111480713, + "learning_rate": 0.0001, + "loss": 62.8381, + "step": 6100 + }, + { + "epoch": 98.55284552845528, + "grad_norm": 1.293872594833374, + "learning_rate": 0.0001, + "loss": 61.3292, + "step": 6110 + }, + { + "epoch": 98.71544715447155, + "grad_norm": 0.9569422006607056, + "learning_rate": 0.0001, + "loss": 61.436, + "step": 6120 + }, + { + "epoch": 98.8780487804878, + "grad_norm": 1.160468578338623, + "learning_rate": 0.0001, + "loss": 60.8419, + "step": 6130 + }, + { + "epoch": 99.03252032520325, + "grad_norm": 1.3102303743362427, + "learning_rate": 0.0001, + "loss": 59.0064, + "step": 6140 + }, + { + "epoch": 99.1951219512195, + "grad_norm": 0.9553463459014893, + "learning_rate": 0.0001, + "loss": 59.925, + "step": 6150 + }, + { + "epoch": 99.35772357723577, + "grad_norm": 1.1993688344955444, + "learning_rate": 0.0001, + "loss": 62.3683, + "step": 6160 + }, + { + "epoch": 99.52032520325203, + "grad_norm": 1.0867350101470947, + "learning_rate": 0.0001, + "loss": 61.8848, + "step": 6170 + }, + { + "epoch": 99.6829268292683, + "grad_norm": 0.9924171566963196, + "learning_rate": 0.0001, + "loss": 60.9243, + "step": 6180 + }, + { + "epoch": 99.84552845528455, + "grad_norm": 0.9548614025115967, + "learning_rate": 0.0001, + "loss": 61.0234, + "step": 6190 + }, + { + "epoch": 100.0, + "grad_norm": 1.0070925951004028, + "learning_rate": 0.0001, + "loss": 56.6554, + "step": 6200 + }, + { + "epoch": 100.16260162601625, + "grad_norm": 1.202168583869934, + "learning_rate": 0.0001, + "loss": 61.3235, + "step": 6210 + }, + { + "epoch": 100.32520325203252, + "grad_norm": 0.9931091070175171, + "learning_rate": 0.0001, + "loss": 59.651, + "step": 6220 + }, + { + "epoch": 100.48780487804878, + "grad_norm": 0.9679157137870789, + "learning_rate": 0.0001, + "loss": 61.5785, + "step": 6230 + }, + { + "epoch": 100.65040650406505, + "grad_norm": 1.049842119216919, + "learning_rate": 0.0001, + "loss": 58.9646, + "step": 6240 + }, + { + "epoch": 100.8130081300813, + "grad_norm": 1.0772044658660889, + "learning_rate": 0.0001, + "loss": 62.3309, + "step": 6250 + }, + { + "epoch": 100.97560975609755, + "grad_norm": 0.9494168162345886, + "learning_rate": 0.0001, + "loss": 61.8663, + "step": 6260 + }, + { + "epoch": 101.130081300813, + "grad_norm": 1.0491962432861328, + "learning_rate": 0.0001, + "loss": 56.5872, + "step": 6270 + }, + { + "epoch": 101.29268292682927, + "grad_norm": 1.118611216545105, + "learning_rate": 0.0001, + "loss": 58.1338, + "step": 6280 + }, + { + "epoch": 101.45528455284553, + "grad_norm": 1.0419825315475464, + "learning_rate": 0.0001, + "loss": 59.6661, + "step": 6290 + }, + { + "epoch": 101.6178861788618, + "grad_norm": 1.2351056337356567, + "learning_rate": 0.0001, + "loss": 61.1697, + "step": 6300 + }, + { + "epoch": 101.78048780487805, + "grad_norm": 1.0556049346923828, + "learning_rate": 0.0001, + "loss": 60.463, + "step": 6310 + }, + { + "epoch": 101.9430894308943, + "grad_norm": 1.090761661529541, + "learning_rate": 0.0001, + "loss": 61.983, + "step": 6320 + }, + { + "epoch": 102.09756097560975, + "grad_norm": 1.4305297136306763, + "learning_rate": 0.0001, + "loss": 55.559, + "step": 6330 + }, + { + "epoch": 102.26016260162602, + "grad_norm": 1.1773794889450073, + "learning_rate": 0.0001, + "loss": 59.5931, + "step": 6340 + }, + { + "epoch": 102.42276422764228, + "grad_norm": 1.0028730630874634, + "learning_rate": 0.0001, + "loss": 60.2714, + "step": 6350 + }, + { + "epoch": 102.58536585365853, + "grad_norm": 0.9494019150733948, + "learning_rate": 0.0001, + "loss": 59.7757, + "step": 6360 + }, + { + "epoch": 102.7479674796748, + "grad_norm": 1.0415751934051514, + "learning_rate": 0.0001, + "loss": 60.7123, + "step": 6370 + }, + { + "epoch": 102.91056910569105, + "grad_norm": 1.2036449909210205, + "learning_rate": 0.0001, + "loss": 60.1201, + "step": 6380 + }, + { + "epoch": 103.0650406504065, + "grad_norm": 0.8801053166389465, + "learning_rate": 0.0001, + "loss": 56.4943, + "step": 6390 + }, + { + "epoch": 103.22764227642277, + "grad_norm": 0.9018735289573669, + "learning_rate": 0.0001, + "loss": 62.4522, + "step": 6400 + }, + { + "epoch": 103.39024390243902, + "grad_norm": 0.971685528755188, + "learning_rate": 0.0001, + "loss": 62.2367, + "step": 6410 + }, + { + "epoch": 103.55284552845528, + "grad_norm": 0.8531002402305603, + "learning_rate": 0.0001, + "loss": 59.5436, + "step": 6420 + }, + { + "epoch": 103.71544715447155, + "grad_norm": 2.123033285140991, + "learning_rate": 0.0001, + "loss": 59.0421, + "step": 6430 + }, + { + "epoch": 103.8780487804878, + "grad_norm": 1.209804654121399, + "learning_rate": 0.0001, + "loss": 60.7791, + "step": 6440 + }, + { + "epoch": 104.03252032520325, + "grad_norm": 0.8219161033630371, + "learning_rate": 0.0001, + "loss": 55.7522, + "step": 6450 + }, + { + "epoch": 104.1951219512195, + "grad_norm": 1.8217495679855347, + "learning_rate": 0.0001, + "loss": 60.6916, + "step": 6460 + }, + { + "epoch": 104.35772357723577, + "grad_norm": 1.060073971748352, + "learning_rate": 0.0001, + "loss": 59.971, + "step": 6470 + }, + { + "epoch": 104.52032520325203, + "grad_norm": 0.9892823696136475, + "learning_rate": 0.0001, + "loss": 61.467, + "step": 6480 + }, + { + "epoch": 104.6829268292683, + "grad_norm": 1.128934621810913, + "learning_rate": 0.0001, + "loss": 62.0451, + "step": 6490 + }, + { + "epoch": 104.84552845528455, + "grad_norm": 0.9627387523651123, + "learning_rate": 0.0001, + "loss": 61.532, + "step": 6500 + }, + { + "epoch": 105.0, + "grad_norm": 1.5344375371932983, + "learning_rate": 0.0001, + "loss": 56.4323, + "step": 6510 + }, + { + "epoch": 105.16260162601625, + "grad_norm": 0.9524087309837341, + "learning_rate": 0.0001, + "loss": 59.6628, + "step": 6520 + }, + { + "epoch": 105.32520325203252, + "grad_norm": 1.0469943284988403, + "learning_rate": 0.0001, + "loss": 59.0605, + "step": 6530 + }, + { + "epoch": 105.48780487804878, + "grad_norm": 2.7462990283966064, + "learning_rate": 0.0001, + "loss": 61.1429, + "step": 6540 + }, + { + "epoch": 105.65040650406505, + "grad_norm": 1.3140382766723633, + "learning_rate": 0.0001, + "loss": 58.2711, + "step": 6550 + }, + { + "epoch": 105.8130081300813, + "grad_norm": 1.1528750658035278, + "learning_rate": 0.0001, + "loss": 59.9937, + "step": 6560 + }, + { + "epoch": 105.97560975609755, + "grad_norm": 1.4449951648712158, + "learning_rate": 0.0001, + "loss": 59.1526, + "step": 6570 + }, + { + "epoch": 106.130081300813, + "grad_norm": 1.1350126266479492, + "learning_rate": 0.0001, + "loss": 55.1142, + "step": 6580 + }, + { + "epoch": 106.29268292682927, + "grad_norm": 0.9332845211029053, + "learning_rate": 0.0001, + "loss": 59.8049, + "step": 6590 + }, + { + "epoch": 106.45528455284553, + "grad_norm": 0.9678266644477844, + "learning_rate": 0.0001, + "loss": 60.9627, + "step": 6600 + }, + { + "epoch": 106.6178861788618, + "grad_norm": 1.1794427633285522, + "learning_rate": 0.0001, + "loss": 59.5372, + "step": 6610 + }, + { + "epoch": 106.78048780487805, + "grad_norm": 1.0404181480407715, + "learning_rate": 0.0001, + "loss": 59.2715, + "step": 6620 + }, + { + "epoch": 106.9430894308943, + "grad_norm": 0.8667073845863342, + "learning_rate": 0.0001, + "loss": 60.2625, + "step": 6630 + }, + { + "epoch": 107.09756097560975, + "grad_norm": 0.9940314888954163, + "learning_rate": 0.0001, + "loss": 55.6906, + "step": 6640 + }, + { + "epoch": 107.26016260162602, + "grad_norm": 1.3880919218063354, + "learning_rate": 0.0001, + "loss": 59.8103, + "step": 6650 + }, + { + "epoch": 107.42276422764228, + "grad_norm": 0.900931179523468, + "learning_rate": 0.0001, + "loss": 59.1929, + "step": 6660 + }, + { + "epoch": 107.58536585365853, + "grad_norm": 0.9374089241027832, + "learning_rate": 0.0001, + "loss": 60.1047, + "step": 6670 + }, + { + "epoch": 107.7479674796748, + "grad_norm": 1.0847140550613403, + "learning_rate": 0.0001, + "loss": 58.6943, + "step": 6680 + }, + { + "epoch": 107.91056910569105, + "grad_norm": 1.086506962776184, + "learning_rate": 0.0001, + "loss": 60.0341, + "step": 6690 + }, + { + "epoch": 108.0650406504065, + "grad_norm": 0.8984777331352234, + "learning_rate": 0.0001, + "loss": 57.0422, + "step": 6700 + }, + { + "epoch": 108.22764227642277, + "grad_norm": 1.187198519706726, + "learning_rate": 0.0001, + "loss": 60.7935, + "step": 6710 + }, + { + "epoch": 108.39024390243902, + "grad_norm": 0.8393696546554565, + "learning_rate": 0.0001, + "loss": 59.7896, + "step": 6720 + }, + { + "epoch": 108.55284552845528, + "grad_norm": 1.0898207426071167, + "learning_rate": 0.0001, + "loss": 60.2579, + "step": 6730 + }, + { + "epoch": 108.71544715447155, + "grad_norm": 1.016926646232605, + "learning_rate": 0.0001, + "loss": 60.9018, + "step": 6740 + }, + { + "epoch": 108.8780487804878, + "grad_norm": 1.3991293907165527, + "learning_rate": 0.0001, + "loss": 60.889, + "step": 6750 + }, + { + "epoch": 109.03252032520325, + "grad_norm": 0.9950843453407288, + "learning_rate": 0.0001, + "loss": 55.1098, + "step": 6760 + }, + { + "epoch": 109.1951219512195, + "grad_norm": 1.0843164920806885, + "learning_rate": 0.0001, + "loss": 60.3208, + "step": 6770 + }, + { + "epoch": 109.35772357723577, + "grad_norm": 0.9067014455795288, + "learning_rate": 0.0001, + "loss": 59.1254, + "step": 6780 + }, + { + "epoch": 109.52032520325203, + "grad_norm": 1.2166807651519775, + "learning_rate": 0.0001, + "loss": 59.9978, + "step": 6790 + }, + { + "epoch": 109.6829268292683, + "grad_norm": 1.256101131439209, + "learning_rate": 0.0001, + "loss": 60.2544, + "step": 6800 + }, + { + "epoch": 109.84552845528455, + "grad_norm": 1.312761664390564, + "learning_rate": 0.0001, + "loss": 59.1564, + "step": 6810 + }, + { + "epoch": 110.0, + "grad_norm": 0.9668724536895752, + "learning_rate": 0.0001, + "loss": 58.067, + "step": 6820 + }, + { + "epoch": 110.16260162601625, + "grad_norm": 1.1048885583877563, + "learning_rate": 0.0001, + "loss": 59.1506, + "step": 6830 + }, + { + "epoch": 110.32520325203252, + "grad_norm": 1.498469352722168, + "learning_rate": 0.0001, + "loss": 59.1366, + "step": 6840 + }, + { + "epoch": 110.48780487804878, + "grad_norm": 1.1747376918792725, + "learning_rate": 0.0001, + "loss": 59.8449, + "step": 6850 + }, + { + "epoch": 110.65040650406505, + "grad_norm": 1.1041772365570068, + "learning_rate": 0.0001, + "loss": 61.0879, + "step": 6860 + }, + { + "epoch": 110.8130081300813, + "grad_norm": 0.9452431797981262, + "learning_rate": 0.0001, + "loss": 60.8911, + "step": 6870 + }, + { + "epoch": 110.97560975609755, + "grad_norm": 1.0108182430267334, + "learning_rate": 0.0001, + "loss": 58.7924, + "step": 6880 + }, + { + "epoch": 111.130081300813, + "grad_norm": 0.9914487600326538, + "learning_rate": 0.0001, + "loss": 57.4749, + "step": 6890 + }, + { + "epoch": 111.29268292682927, + "grad_norm": 1.0404850244522095, + "learning_rate": 0.0001, + "loss": 59.5329, + "step": 6900 + }, + { + "epoch": 111.45528455284553, + "grad_norm": 1.0363966226577759, + "learning_rate": 0.0001, + "loss": 58.8972, + "step": 6910 + }, + { + "epoch": 111.6178861788618, + "grad_norm": 1.0189651250839233, + "learning_rate": 0.0001, + "loss": 59.3613, + "step": 6920 + }, + { + "epoch": 111.78048780487805, + "grad_norm": 0.9696914553642273, + "learning_rate": 0.0001, + "loss": 59.8001, + "step": 6930 + }, + { + "epoch": 111.9430894308943, + "grad_norm": 1.406821370124817, + "learning_rate": 0.0001, + "loss": 58.6405, + "step": 6940 + }, + { + "epoch": 112.09756097560975, + "grad_norm": 1.0969383716583252, + "learning_rate": 0.0001, + "loss": 57.4458, + "step": 6950 + }, + { + "epoch": 112.26016260162602, + "grad_norm": 1.117593765258789, + "learning_rate": 0.0001, + "loss": 62.599, + "step": 6960 + }, + { + "epoch": 112.42276422764228, + "grad_norm": 0.9327741861343384, + "learning_rate": 0.0001, + "loss": 60.632, + "step": 6970 + }, + { + "epoch": 112.58536585365853, + "grad_norm": 1.1198201179504395, + "learning_rate": 0.0001, + "loss": 59.6119, + "step": 6980 + }, + { + "epoch": 112.7479674796748, + "grad_norm": 0.8709354400634766, + "learning_rate": 0.0001, + "loss": 60.831, + "step": 6990 + }, + { + "epoch": 112.91056910569105, + "grad_norm": 0.9589444994926453, + "learning_rate": 0.0001, + "loss": 58.0002, + "step": 7000 + }, + { + "epoch": 113.0650406504065, + "grad_norm": 1.0971899032592773, + "learning_rate": 0.0001, + "loss": 56.8572, + "step": 7010 + }, + { + "epoch": 113.22764227642277, + "grad_norm": 0.9764513969421387, + "learning_rate": 0.0001, + "loss": 59.3038, + "step": 7020 + }, + { + "epoch": 113.39024390243902, + "grad_norm": 0.9050093293190002, + "learning_rate": 0.0001, + "loss": 59.9146, + "step": 7030 + }, + { + "epoch": 113.55284552845528, + "grad_norm": 0.9506774544715881, + "learning_rate": 0.0001, + "loss": 59.7523, + "step": 7040 + }, + { + "epoch": 113.71544715447155, + "grad_norm": 0.9678346514701843, + "learning_rate": 0.0001, + "loss": 61.3103, + "step": 7050 + }, + { + "epoch": 113.8780487804878, + "grad_norm": 0.9859515428543091, + "learning_rate": 0.0001, + "loss": 56.3866, + "step": 7060 + }, + { + "epoch": 114.03252032520325, + "grad_norm": 1.2696164846420288, + "learning_rate": 0.0001, + "loss": 58.16, + "step": 7070 + }, + { + "epoch": 114.1951219512195, + "grad_norm": 0.9323938488960266, + "learning_rate": 0.0001, + "loss": 58.9483, + "step": 7080 + }, + { + "epoch": 114.35772357723577, + "grad_norm": 1.4123785495758057, + "learning_rate": 0.0001, + "loss": 60.4666, + "step": 7090 + }, + { + "epoch": 114.52032520325203, + "grad_norm": 1.009020447731018, + "learning_rate": 0.0001, + "loss": 58.7152, + "step": 7100 + }, + { + "epoch": 114.6829268292683, + "grad_norm": 1.219383955001831, + "learning_rate": 0.0001, + "loss": 59.1494, + "step": 7110 + }, + { + "epoch": 114.84552845528455, + "grad_norm": 0.9579033255577087, + "learning_rate": 0.0001, + "loss": 58.9879, + "step": 7120 + }, + { + "epoch": 115.0, + "grad_norm": 0.899074912071228, + "learning_rate": 0.0001, + "loss": 55.2031, + "step": 7130 + }, + { + "epoch": 115.16260162601625, + "grad_norm": 0.9924830794334412, + "learning_rate": 0.0001, + "loss": 58.583, + "step": 7140 + }, + { + "epoch": 115.32520325203252, + "grad_norm": 1.0280295610427856, + "learning_rate": 0.0001, + "loss": 60.0948, + "step": 7150 + }, + { + "epoch": 115.48780487804878, + "grad_norm": 0.8108797669410706, + "learning_rate": 0.0001, + "loss": 60.9433, + "step": 7160 + }, + { + "epoch": 115.65040650406505, + "grad_norm": 1.2597944736480713, + "learning_rate": 0.0001, + "loss": 59.1905, + "step": 7170 + }, + { + "epoch": 115.8130081300813, + "grad_norm": 1.2565767765045166, + "learning_rate": 0.0001, + "loss": 59.3545, + "step": 7180 + }, + { + "epoch": 115.97560975609755, + "grad_norm": 1.2012240886688232, + "learning_rate": 0.0001, + "loss": 58.348, + "step": 7190 + }, + { + "epoch": 116.130081300813, + "grad_norm": 1.2883665561676025, + "learning_rate": 0.0001, + "loss": 57.462, + "step": 7200 + }, + { + "epoch": 116.29268292682927, + "grad_norm": 0.904152512550354, + "learning_rate": 0.0001, + "loss": 58.958, + "step": 7210 + }, + { + "epoch": 116.45528455284553, + "grad_norm": 1.1928468942642212, + "learning_rate": 0.0001, + "loss": 58.6106, + "step": 7220 + }, + { + "epoch": 116.6178861788618, + "grad_norm": 1.0655683279037476, + "learning_rate": 0.0001, + "loss": 58.4948, + "step": 7230 + }, + { + "epoch": 116.78048780487805, + "grad_norm": 1.0527434349060059, + "learning_rate": 0.0001, + "loss": 60.1771, + "step": 7240 + }, + { + "epoch": 116.9430894308943, + "grad_norm": 1.1439871788024902, + "learning_rate": 0.0001, + "loss": 58.6737, + "step": 7250 + }, + { + "epoch": 117.09756097560975, + "grad_norm": 1.506571650505066, + "learning_rate": 0.0001, + "loss": 58.0901, + "step": 7260 + }, + { + "epoch": 117.26016260162602, + "grad_norm": 1.1419955492019653, + "learning_rate": 0.0001, + "loss": 59.0705, + "step": 7270 + }, + { + "epoch": 117.42276422764228, + "grad_norm": 1.1211196184158325, + "learning_rate": 0.0001, + "loss": 57.801, + "step": 7280 + }, + { + "epoch": 117.58536585365853, + "grad_norm": 1.1424155235290527, + "learning_rate": 0.0001, + "loss": 59.7449, + "step": 7290 + }, + { + "epoch": 117.7479674796748, + "grad_norm": 1.0593281984329224, + "learning_rate": 0.0001, + "loss": 60.1789, + "step": 7300 + }, + { + "epoch": 117.91056910569105, + "grad_norm": 1.3680177927017212, + "learning_rate": 0.0001, + "loss": 58.4225, + "step": 7310 + }, + { + "epoch": 118.0650406504065, + "grad_norm": 0.8391700983047485, + "learning_rate": 0.0001, + "loss": 56.2102, + "step": 7320 + }, + { + "epoch": 118.22764227642277, + "grad_norm": 1.3166592121124268, + "learning_rate": 0.0001, + "loss": 61.0399, + "step": 7330 + }, + { + "epoch": 118.39024390243902, + "grad_norm": 1.929949402809143, + "learning_rate": 0.0001, + "loss": 59.6514, + "step": 7340 + }, + { + "epoch": 118.55284552845528, + "grad_norm": 0.9464254975318909, + "learning_rate": 0.0001, + "loss": 58.6488, + "step": 7350 + }, + { + "epoch": 118.71544715447155, + "grad_norm": 1.1066193580627441, + "learning_rate": 0.0001, + "loss": 61.5959, + "step": 7360 + }, + { + "epoch": 118.8780487804878, + "grad_norm": 1.216125726699829, + "learning_rate": 0.0001, + "loss": 58.8231, + "step": 7370 + }, + { + "epoch": 119.03252032520325, + "grad_norm": 1.1182583570480347, + "learning_rate": 0.0001, + "loss": 56.9496, + "step": 7380 + }, + { + "epoch": 119.1951219512195, + "grad_norm": 1.105729341506958, + "learning_rate": 0.0001, + "loss": 58.4291, + "step": 7390 + }, + { + "epoch": 119.35772357723577, + "grad_norm": 1.017879843711853, + "learning_rate": 0.0001, + "loss": 59.4963, + "step": 7400 + }, + { + "epoch": 119.52032520325203, + "grad_norm": 0.7882587909698486, + "learning_rate": 0.0001, + "loss": 57.7243, + "step": 7410 + }, + { + "epoch": 119.6829268292683, + "grad_norm": 0.8356695175170898, + "learning_rate": 0.0001, + "loss": 60.4962, + "step": 7420 + }, + { + "epoch": 119.84552845528455, + "grad_norm": 1.1358702182769775, + "learning_rate": 0.0001, + "loss": 60.7173, + "step": 7430 + }, + { + "epoch": 120.0, + "grad_norm": 1.116322636604309, + "learning_rate": 0.0001, + "loss": 53.8387, + "step": 7440 + }, + { + "epoch": 120.16260162601625, + "grad_norm": 0.8522642254829407, + "learning_rate": 0.0001, + "loss": 57.9095, + "step": 7450 + }, + { + "epoch": 120.32520325203252, + "grad_norm": 0.890731692314148, + "learning_rate": 0.0001, + "loss": 57.2296, + "step": 7460 + }, + { + "epoch": 120.48780487804878, + "grad_norm": 1.6366336345672607, + "learning_rate": 0.0001, + "loss": 58.9417, + "step": 7470 + }, + { + "epoch": 120.65040650406505, + "grad_norm": 0.9774733781814575, + "learning_rate": 0.0001, + "loss": 58.8446, + "step": 7480 + }, + { + "epoch": 120.8130081300813, + "grad_norm": 1.6188803911209106, + "learning_rate": 0.0001, + "loss": 59.9794, + "step": 7490 + }, + { + "epoch": 120.97560975609755, + "grad_norm": 1.146599292755127, + "learning_rate": 0.0001, + "loss": 59.8794, + "step": 7500 + }, + { + "epoch": 121.130081300813, + "grad_norm": 0.9472863078117371, + "learning_rate": 0.0001, + "loss": 55.2454, + "step": 7510 + }, + { + "epoch": 121.29268292682927, + "grad_norm": 0.8604205250740051, + "learning_rate": 0.0001, + "loss": 57.6314, + "step": 7520 + }, + { + "epoch": 121.45528455284553, + "grad_norm": 1.0755794048309326, + "learning_rate": 0.0001, + "loss": 60.909, + "step": 7530 + }, + { + "epoch": 121.6178861788618, + "grad_norm": 0.7967071533203125, + "learning_rate": 0.0001, + "loss": 60.1755, + "step": 7540 + }, + { + "epoch": 121.78048780487805, + "grad_norm": 1.037655234336853, + "learning_rate": 0.0001, + "loss": 59.283, + "step": 7550 + }, + { + "epoch": 121.9430894308943, + "grad_norm": 1.070142388343811, + "learning_rate": 0.0001, + "loss": 57.0941, + "step": 7560 + }, + { + "epoch": 122.09756097560975, + "grad_norm": 1.4201022386550903, + "learning_rate": 0.0001, + "loss": 56.2891, + "step": 7570 + }, + { + "epoch": 122.26016260162602, + "grad_norm": 1.0909277200698853, + "learning_rate": 0.0001, + "loss": 57.5213, + "step": 7580 + }, + { + "epoch": 122.42276422764228, + "grad_norm": 0.7573383450508118, + "learning_rate": 0.0001, + "loss": 59.0847, + "step": 7590 + }, + { + "epoch": 122.58536585365853, + "grad_norm": 0.8931725025177002, + "learning_rate": 0.0001, + "loss": 58.9706, + "step": 7600 + }, + { + "epoch": 122.7479674796748, + "grad_norm": 0.9459935426712036, + "learning_rate": 0.0001, + "loss": 58.6267, + "step": 7610 + }, + { + "epoch": 122.91056910569105, + "grad_norm": 0.9463319778442383, + "learning_rate": 0.0001, + "loss": 60.8536, + "step": 7620 + }, + { + "epoch": 123.0650406504065, + "grad_norm": 0.9286382794380188, + "learning_rate": 0.0001, + "loss": 55.2711, + "step": 7630 + }, + { + "epoch": 123.22764227642277, + "grad_norm": 0.9317494630813599, + "learning_rate": 0.0001, + "loss": 60.1578, + "step": 7640 + }, + { + "epoch": 123.39024390243902, + "grad_norm": 0.9834237098693848, + "learning_rate": 0.0001, + "loss": 58.8121, + "step": 7650 + }, + { + "epoch": 123.55284552845528, + "grad_norm": 0.9992889761924744, + "learning_rate": 0.0001, + "loss": 59.8061, + "step": 7660 + }, + { + "epoch": 123.71544715447155, + "grad_norm": 1.0354453325271606, + "learning_rate": 0.0001, + "loss": 59.5853, + "step": 7670 + }, + { + "epoch": 123.8780487804878, + "grad_norm": 1.0609121322631836, + "learning_rate": 0.0001, + "loss": 59.6556, + "step": 7680 + }, + { + "epoch": 124.03252032520325, + "grad_norm": 0.9742912650108337, + "learning_rate": 0.0001, + "loss": 57.2294, + "step": 7690 + }, + { + "epoch": 124.1951219512195, + "grad_norm": 0.9758710265159607, + "learning_rate": 0.0001, + "loss": 59.8581, + "step": 7700 + }, + { + "epoch": 124.35772357723577, + "grad_norm": 0.9492778182029724, + "learning_rate": 0.0001, + "loss": 60.0624, + "step": 7710 + }, + { + "epoch": 124.52032520325203, + "grad_norm": 0.8311835527420044, + "learning_rate": 0.0001, + "loss": 60.0849, + "step": 7720 + }, + { + "epoch": 124.6829268292683, + "grad_norm": 0.8652944564819336, + "learning_rate": 0.0001, + "loss": 58.2321, + "step": 7730 + }, + { + "epoch": 124.84552845528455, + "grad_norm": 1.2980835437774658, + "learning_rate": 0.0001, + "loss": 57.5315, + "step": 7740 + }, + { + "epoch": 125.0, + "grad_norm": 1.0550237894058228, + "learning_rate": 0.0001, + "loss": 56.9615, + "step": 7750 + }, + { + "epoch": 125.16260162601625, + "grad_norm": 1.1248353719711304, + "learning_rate": 0.0001, + "loss": 58.4853, + "step": 7760 + }, + { + "epoch": 125.32520325203252, + "grad_norm": 1.338152289390564, + "learning_rate": 0.0001, + "loss": 57.8604, + "step": 7770 + }, + { + "epoch": 125.48780487804878, + "grad_norm": 0.8162221908569336, + "learning_rate": 0.0001, + "loss": 56.6396, + "step": 7780 + }, + { + "epoch": 125.65040650406505, + "grad_norm": 1.0431848764419556, + "learning_rate": 0.0001, + "loss": 58.9896, + "step": 7790 + }, + { + "epoch": 125.8130081300813, + "grad_norm": 0.848052978515625, + "learning_rate": 0.0001, + "loss": 57.7459, + "step": 7800 + }, + { + "epoch": 125.97560975609755, + "grad_norm": 0.8441061973571777, + "learning_rate": 0.0001, + "loss": 57.9488, + "step": 7810 + }, + { + "epoch": 126.130081300813, + "grad_norm": 0.774566650390625, + "learning_rate": 0.0001, + "loss": 55.9949, + "step": 7820 + }, + { + "epoch": 126.29268292682927, + "grad_norm": 1.178572416305542, + "learning_rate": 0.0001, + "loss": 58.3554, + "step": 7830 + }, + { + "epoch": 126.45528455284553, + "grad_norm": 0.9363889098167419, + "learning_rate": 0.0001, + "loss": 60.6901, + "step": 7840 + }, + { + "epoch": 126.6178861788618, + "grad_norm": 1.0010961294174194, + "learning_rate": 0.0001, + "loss": 57.0078, + "step": 7850 + }, + { + "epoch": 126.78048780487805, + "grad_norm": 0.9573602080345154, + "learning_rate": 0.0001, + "loss": 58.0567, + "step": 7860 + }, + { + "epoch": 126.9430894308943, + "grad_norm": 1.1903672218322754, + "learning_rate": 0.0001, + "loss": 58.3699, + "step": 7870 + }, + { + "epoch": 127.09756097560975, + "grad_norm": 1.2158106565475464, + "learning_rate": 0.0001, + "loss": 57.1154, + "step": 7880 + }, + { + "epoch": 127.26016260162602, + "grad_norm": 1.198789119720459, + "learning_rate": 0.0001, + "loss": 59.1004, + "step": 7890 + }, + { + "epoch": 127.42276422764228, + "grad_norm": 0.8893532752990723, + "learning_rate": 0.0001, + "loss": 59.6481, + "step": 7900 + }, + { + "epoch": 127.58536585365853, + "grad_norm": 0.9834772348403931, + "learning_rate": 0.0001, + "loss": 57.4394, + "step": 7910 + }, + { + "epoch": 127.7479674796748, + "grad_norm": 1.1957768201828003, + "learning_rate": 0.0001, + "loss": 59.1622, + "step": 7920 + }, + { + "epoch": 127.91056910569105, + "grad_norm": 0.822515070438385, + "learning_rate": 0.0001, + "loss": 58.4967, + "step": 7930 + }, + { + "epoch": 128.0650406504065, + "grad_norm": 1.0374864339828491, + "learning_rate": 0.0001, + "loss": 53.9391, + "step": 7940 + }, + { + "epoch": 128.22764227642276, + "grad_norm": 1.1767654418945312, + "learning_rate": 0.0001, + "loss": 59.4305, + "step": 7950 + }, + { + "epoch": 128.390243902439, + "grad_norm": 1.0048503875732422, + "learning_rate": 0.0001, + "loss": 59.7954, + "step": 7960 + }, + { + "epoch": 128.5528455284553, + "grad_norm": 0.9039503931999207, + "learning_rate": 0.0001, + "loss": 58.6945, + "step": 7970 + }, + { + "epoch": 128.71544715447155, + "grad_norm": 1.047418236732483, + "learning_rate": 0.0001, + "loss": 58.6324, + "step": 7980 + }, + { + "epoch": 128.8780487804878, + "grad_norm": 1.7829573154449463, + "learning_rate": 0.0001, + "loss": 57.3699, + "step": 7990 + }, + { + "epoch": 129.03252032520325, + "grad_norm": 0.8874698281288147, + "learning_rate": 0.0001, + "loss": 57.8226, + "step": 8000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 323, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.717029130252111e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c7f229ebe1034caaf3e92006c8c7528c21af93d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517586849d044483051c7a59cec7c6ce389951878a7357ef837edb1cbc9983c8 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathpros1sf/checkpoint-8000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/latest b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/latest new file mode 100644 index 0000000000000000000000000000000000000000..25c776ee3abcad1c4d1e16e8275e4e00984a237c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/latest @@ -0,0 +1 @@ +global_step10000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4b621cae0031cfa066fb660d7e72c4dcdfe4509c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbdc2ac2a07f63d04c2a544069294c9b6bf740ddb7dfd846e2eeaff98fad8dd5 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..69e44b0021848d45662a9c3131dace17f19d00a0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ad07ed13c09e650e685ab0f70dada21aa67028ea506bf8a97a79c9854dd646b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..7d90bddbc9f4f25ec3d3052228fa579de29a728b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:797755cbc6c741d58510dd03e0b221f01b711524b664e83796733a46404b7570 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..64b09935369c50da62f1ddbea8d590e9b4408e29 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff45d8a517863c0155d7af1878f2b0eade905bca3b8a9cfea1726d98cfeeedfe +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..372cbd682c88f712a6946210ea2363ce106f3230 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1efa593a19cbbf7783bf259adeebb10eea0f803d4710a11bda46467f95b5357 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..fc70ff77b5888c06b343f6d21c7546da1e635de2 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cdde70749fef428fc2f46f407bdfa019a2bcd253af2206c874c2a94ff9894f5 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..cffbf71af334b85c61e27b05cf6db64e78407c9b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd8790898e94a4a5ab3272cf3af49a83db436e9fba671eea23293d92eaf99a82 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..c1330aa8d7b97a6cbd9e5be9c4143d90bc71d578 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:888d145701c5b8e6e4642df19859bde977ed26368475c99d3639211a3b1a91ee +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..3e6be54ae51a19eee36378dadaaec514cabd57e0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daaac6b1f6b3e21fcd2400cb841091392883c80d3b0c0a37e94da2f8bff1286b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c3b38e2c4b51d5a99b693a86d505062af0a9098 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:034d71d0f0787f0c3ec923c15d690debaef12acfa289bd37c34adb31a082231b +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bd4f0aa1caab341c6c3c4a25da636135d8292cec --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/trainer_state.json @@ -0,0 +1,7034 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.614763529198038, + "eval_steps": 2000, + "global_step": 10000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0016148240851012292, + "grad_norm": 43.90101623535156, + "learning_rate": 0.0001, + "loss": 307.2084, + "step": 10 + }, + { + "epoch": 0.0032296481702024584, + "grad_norm": 19.106313705444336, + "learning_rate": 0.0001, + "loss": 320.2794, + "step": 20 + }, + { + "epoch": 0.004844472255303688, + "grad_norm": 5.771847248077393, + "learning_rate": 0.0001, + "loss": 247.4977, + "step": 30 + }, + { + "epoch": 0.006459296340404917, + "grad_norm": 2.630659580230713, + "learning_rate": 0.0001, + "loss": 229.9593, + "step": 40 + }, + { + "epoch": 0.008074120425506147, + "grad_norm": 2.487499475479126, + "learning_rate": 0.0001, + "loss": 228.65, + "step": 50 + }, + { + "epoch": 0.009688944510607376, + "grad_norm": 2.078263521194458, + "learning_rate": 0.0001, + "loss": 227.648, + "step": 60 + }, + { + "epoch": 0.011303768595708604, + "grad_norm": 3.105417490005493, + "learning_rate": 0.0001, + "loss": 227.9345, + "step": 70 + }, + { + "epoch": 0.012918592680809833, + "grad_norm": 2.2145848274230957, + "learning_rate": 0.0001, + "loss": 227.3662, + "step": 80 + }, + { + "epoch": 0.014533416765911064, + "grad_norm": 1.9644615650177002, + "learning_rate": 0.0001, + "loss": 226.9145, + "step": 90 + }, + { + "epoch": 0.016148240851012293, + "grad_norm": 1.867382287979126, + "learning_rate": 0.0001, + "loss": 227.5703, + "step": 100 + }, + { + "epoch": 0.017763064936113522, + "grad_norm": 1.4613710641860962, + "learning_rate": 0.0001, + "loss": 227.1107, + "step": 110 + }, + { + "epoch": 0.01937788902121475, + "grad_norm": 2.8953869342803955, + "learning_rate": 0.0001, + "loss": 226.8185, + "step": 120 + }, + { + "epoch": 0.02099271310631598, + "grad_norm": 1.8446482419967651, + "learning_rate": 0.0001, + "loss": 226.9624, + "step": 130 + }, + { + "epoch": 0.02260753719141721, + "grad_norm": 1.3048362731933594, + "learning_rate": 0.0001, + "loss": 226.8404, + "step": 140 + }, + { + "epoch": 0.024222361276518438, + "grad_norm": 1.7100074291229248, + "learning_rate": 0.0001, + "loss": 226.1071, + "step": 150 + }, + { + "epoch": 0.025837185361619667, + "grad_norm": 1.7875330448150635, + "learning_rate": 0.0001, + "loss": 226.0764, + "step": 160 + }, + { + "epoch": 0.0274520094467209, + "grad_norm": 1.7503769397735596, + "learning_rate": 0.0001, + "loss": 226.0772, + "step": 170 + }, + { + "epoch": 0.029066833531822128, + "grad_norm": 2.7083983421325684, + "learning_rate": 0.0001, + "loss": 224.7667, + "step": 180 + }, + { + "epoch": 0.030681657616923357, + "grad_norm": 1.8192371129989624, + "learning_rate": 0.0001, + "loss": 223.6494, + "step": 190 + }, + { + "epoch": 0.032296481702024586, + "grad_norm": 3.210358142852783, + "learning_rate": 0.0001, + "loss": 223.3545, + "step": 200 + }, + { + "epoch": 0.03391130578712581, + "grad_norm": 5.819669246673584, + "learning_rate": 0.0001, + "loss": 222.1732, + "step": 210 + }, + { + "epoch": 0.035526129872227044, + "grad_norm": 4.185793399810791, + "learning_rate": 0.0001, + "loss": 221.6866, + "step": 220 + }, + { + "epoch": 0.03714095395732828, + "grad_norm": 2.7514216899871826, + "learning_rate": 0.0001, + "loss": 221.0769, + "step": 230 + }, + { + "epoch": 0.0387557780424295, + "grad_norm": 1.5478988885879517, + "learning_rate": 0.0001, + "loss": 219.7232, + "step": 240 + }, + { + "epoch": 0.040370602127530734, + "grad_norm": 1.4958223104476929, + "learning_rate": 0.0001, + "loss": 218.5454, + "step": 250 + }, + { + "epoch": 0.04198542621263196, + "grad_norm": 6.36635684967041, + "learning_rate": 0.0001, + "loss": 218.5061, + "step": 260 + }, + { + "epoch": 0.04360025029773319, + "grad_norm": 3.029601812362671, + "learning_rate": 0.0001, + "loss": 216.3466, + "step": 270 + }, + { + "epoch": 0.04521507438283442, + "grad_norm": 2.6436100006103516, + "learning_rate": 0.0001, + "loss": 214.0003, + "step": 280 + }, + { + "epoch": 0.04682989846793565, + "grad_norm": 5.329655647277832, + "learning_rate": 0.0001, + "loss": 212.9169, + "step": 290 + }, + { + "epoch": 0.048444722553036876, + "grad_norm": 4.930875301361084, + "learning_rate": 0.0001, + "loss": 209.7737, + "step": 300 + }, + { + "epoch": 0.05005954663813811, + "grad_norm": 18.902585983276367, + "learning_rate": 0.0001, + "loss": 209.4267, + "step": 310 + }, + { + "epoch": 0.051674370723239334, + "grad_norm": 5.364975929260254, + "learning_rate": 0.0001, + "loss": 206.49, + "step": 320 + }, + { + "epoch": 0.053289194808340566, + "grad_norm": 3.1421146392822266, + "learning_rate": 0.0001, + "loss": 204.8132, + "step": 330 + }, + { + "epoch": 0.0549040188934418, + "grad_norm": 3.399320602416992, + "learning_rate": 0.0001, + "loss": 202.4785, + "step": 340 + }, + { + "epoch": 0.056518842978543024, + "grad_norm": 5.137377738952637, + "learning_rate": 0.0001, + "loss": 199.5143, + "step": 350 + }, + { + "epoch": 0.058133667063644257, + "grad_norm": 5.4967522621154785, + "learning_rate": 0.0001, + "loss": 196.2809, + "step": 360 + }, + { + "epoch": 0.05974849114874548, + "grad_norm": 6.435070037841797, + "learning_rate": 0.0001, + "loss": 196.8831, + "step": 370 + }, + { + "epoch": 0.061363315233846714, + "grad_norm": 5.7334370613098145, + "learning_rate": 0.0001, + "loss": 193.141, + "step": 380 + }, + { + "epoch": 0.06297813931894794, + "grad_norm": 3.6522979736328125, + "learning_rate": 0.0001, + "loss": 191.5819, + "step": 390 + }, + { + "epoch": 0.06459296340404917, + "grad_norm": 2.614051103591919, + "learning_rate": 0.0001, + "loss": 190.046, + "step": 400 + }, + { + "epoch": 0.0662077874891504, + "grad_norm": 4.845924377441406, + "learning_rate": 0.0001, + "loss": 188.7121, + "step": 410 + }, + { + "epoch": 0.06782261157425162, + "grad_norm": 4.319681644439697, + "learning_rate": 0.0001, + "loss": 185.3697, + "step": 420 + }, + { + "epoch": 0.06943743565935286, + "grad_norm": 2.7018792629241943, + "learning_rate": 0.0001, + "loss": 183.5171, + "step": 430 + }, + { + "epoch": 0.07105225974445409, + "grad_norm": 2.6407113075256348, + "learning_rate": 0.0001, + "loss": 181.245, + "step": 440 + }, + { + "epoch": 0.07266708382955532, + "grad_norm": 3.314251184463501, + "learning_rate": 0.0001, + "loss": 179.3874, + "step": 450 + }, + { + "epoch": 0.07428190791465655, + "grad_norm": 2.635136365890503, + "learning_rate": 0.0001, + "loss": 177.8257, + "step": 460 + }, + { + "epoch": 0.07589673199975777, + "grad_norm": 5.227738857269287, + "learning_rate": 0.0001, + "loss": 176.4699, + "step": 470 + }, + { + "epoch": 0.077511556084859, + "grad_norm": 4.998415470123291, + "learning_rate": 0.0001, + "loss": 175.7601, + "step": 480 + }, + { + "epoch": 0.07912638016996024, + "grad_norm": 2.946892261505127, + "learning_rate": 0.0001, + "loss": 173.936, + "step": 490 + }, + { + "epoch": 0.08074120425506147, + "grad_norm": 4.20358943939209, + "learning_rate": 0.0001, + "loss": 172.065, + "step": 500 + }, + { + "epoch": 0.08235602834016269, + "grad_norm": 3.125765562057495, + "learning_rate": 0.0001, + "loss": 171.3727, + "step": 510 + }, + { + "epoch": 0.08397085242526392, + "grad_norm": 2.4870612621307373, + "learning_rate": 0.0001, + "loss": 169.9, + "step": 520 + }, + { + "epoch": 0.08558567651036515, + "grad_norm": 3.917384147644043, + "learning_rate": 0.0001, + "loss": 168.5049, + "step": 530 + }, + { + "epoch": 0.08720050059546638, + "grad_norm": 2.302142858505249, + "learning_rate": 0.0001, + "loss": 167.2027, + "step": 540 + }, + { + "epoch": 0.08881532468056762, + "grad_norm": 4.794841289520264, + "learning_rate": 0.0001, + "loss": 166.0887, + "step": 550 + }, + { + "epoch": 0.09043014876566884, + "grad_norm": 2.200291395187378, + "learning_rate": 0.0001, + "loss": 164.5882, + "step": 560 + }, + { + "epoch": 0.09204497285077007, + "grad_norm": 1.9572436809539795, + "learning_rate": 0.0001, + "loss": 163.3007, + "step": 570 + }, + { + "epoch": 0.0936597969358713, + "grad_norm": 3.443633556365967, + "learning_rate": 0.0001, + "loss": 160.6805, + "step": 580 + }, + { + "epoch": 0.09527462102097253, + "grad_norm": 2.6079020500183105, + "learning_rate": 0.0001, + "loss": 160.3372, + "step": 590 + }, + { + "epoch": 0.09688944510607375, + "grad_norm": 2.4401192665100098, + "learning_rate": 0.0001, + "loss": 160.0956, + "step": 600 + }, + { + "epoch": 0.09850426919117498, + "grad_norm": 2.686101198196411, + "learning_rate": 0.0001, + "loss": 157.6138, + "step": 610 + }, + { + "epoch": 0.10011909327627622, + "grad_norm": 2.594151496887207, + "learning_rate": 0.0001, + "loss": 156.0726, + "step": 620 + }, + { + "epoch": 0.10173391736137745, + "grad_norm": 2.509141683578491, + "learning_rate": 0.0001, + "loss": 155.9923, + "step": 630 + }, + { + "epoch": 0.10334874144647867, + "grad_norm": 2.4077377319335938, + "learning_rate": 0.0001, + "loss": 155.214, + "step": 640 + }, + { + "epoch": 0.1049635655315799, + "grad_norm": 4.083552837371826, + "learning_rate": 0.0001, + "loss": 152.8257, + "step": 650 + }, + { + "epoch": 0.10657838961668113, + "grad_norm": 2.7188713550567627, + "learning_rate": 0.0001, + "loss": 151.875, + "step": 660 + }, + { + "epoch": 0.10819321370178236, + "grad_norm": 2.41287899017334, + "learning_rate": 0.0001, + "loss": 151.935, + "step": 670 + }, + { + "epoch": 0.1098080377868836, + "grad_norm": 3.995708465576172, + "learning_rate": 0.0001, + "loss": 151.0069, + "step": 680 + }, + { + "epoch": 0.11142286187198482, + "grad_norm": 3.217705488204956, + "learning_rate": 0.0001, + "loss": 149.5922, + "step": 690 + }, + { + "epoch": 0.11303768595708605, + "grad_norm": 3.4349842071533203, + "learning_rate": 0.0001, + "loss": 148.0044, + "step": 700 + }, + { + "epoch": 0.11465251004218728, + "grad_norm": 2.106865167617798, + "learning_rate": 0.0001, + "loss": 147.8404, + "step": 710 + }, + { + "epoch": 0.11626733412728851, + "grad_norm": 1.8729510307312012, + "learning_rate": 0.0001, + "loss": 145.1299, + "step": 720 + }, + { + "epoch": 0.11788215821238973, + "grad_norm": 4.14620304107666, + "learning_rate": 0.0001, + "loss": 144.3928, + "step": 730 + }, + { + "epoch": 0.11949698229749096, + "grad_norm": 2.202505350112915, + "learning_rate": 0.0001, + "loss": 144.8357, + "step": 740 + }, + { + "epoch": 0.1211118063825922, + "grad_norm": 2.201470375061035, + "learning_rate": 0.0001, + "loss": 143.2202, + "step": 750 + }, + { + "epoch": 0.12272663046769343, + "grad_norm": 1.5977853536605835, + "learning_rate": 0.0001, + "loss": 143.3537, + "step": 760 + }, + { + "epoch": 0.12434145455279466, + "grad_norm": 2.4403927326202393, + "learning_rate": 0.0001, + "loss": 141.2914, + "step": 770 + }, + { + "epoch": 0.12595627863789588, + "grad_norm": 1.6895846128463745, + "learning_rate": 0.0001, + "loss": 139.9639, + "step": 780 + }, + { + "epoch": 0.12757110272299713, + "grad_norm": 3.4370665550231934, + "learning_rate": 0.0001, + "loss": 140.715, + "step": 790 + }, + { + "epoch": 0.12918592680809834, + "grad_norm": 2.1449077129364014, + "learning_rate": 0.0001, + "loss": 139.2094, + "step": 800 + }, + { + "epoch": 0.13080075089319956, + "grad_norm": 2.4578912258148193, + "learning_rate": 0.0001, + "loss": 137.6724, + "step": 810 + }, + { + "epoch": 0.1324155749783008, + "grad_norm": 1.8994814157485962, + "learning_rate": 0.0001, + "loss": 136.9551, + "step": 820 + }, + { + "epoch": 0.13403039906340203, + "grad_norm": 2.6531031131744385, + "learning_rate": 0.0001, + "loss": 136.5812, + "step": 830 + }, + { + "epoch": 0.13564522314850325, + "grad_norm": 3.8680083751678467, + "learning_rate": 0.0001, + "loss": 134.3655, + "step": 840 + }, + { + "epoch": 0.1372600472336045, + "grad_norm": 1.9594208002090454, + "learning_rate": 0.0001, + "loss": 135.4958, + "step": 850 + }, + { + "epoch": 0.1388748713187057, + "grad_norm": 1.5475000143051147, + "learning_rate": 0.0001, + "loss": 135.2142, + "step": 860 + }, + { + "epoch": 0.14048969540380696, + "grad_norm": 1.6569288969039917, + "learning_rate": 0.0001, + "loss": 133.4162, + "step": 870 + }, + { + "epoch": 0.14210451948890818, + "grad_norm": 1.6724224090576172, + "learning_rate": 0.0001, + "loss": 133.7519, + "step": 880 + }, + { + "epoch": 0.1437193435740094, + "grad_norm": 1.7299500703811646, + "learning_rate": 0.0001, + "loss": 131.471, + "step": 890 + }, + { + "epoch": 0.14533416765911064, + "grad_norm": 1.9444470405578613, + "learning_rate": 0.0001, + "loss": 128.5966, + "step": 900 + }, + { + "epoch": 0.14694899174421186, + "grad_norm": 2.404723644256592, + "learning_rate": 0.0001, + "loss": 129.9824, + "step": 910 + }, + { + "epoch": 0.1485638158293131, + "grad_norm": 2.1493897438049316, + "learning_rate": 0.0001, + "loss": 128.0219, + "step": 920 + }, + { + "epoch": 0.15017863991441432, + "grad_norm": 2.8861117362976074, + "learning_rate": 0.0001, + "loss": 128.5297, + "step": 930 + }, + { + "epoch": 0.15179346399951554, + "grad_norm": 2.546415090560913, + "learning_rate": 0.0001, + "loss": 126.6817, + "step": 940 + }, + { + "epoch": 0.1534082880846168, + "grad_norm": 1.727663278579712, + "learning_rate": 0.0001, + "loss": 125.0413, + "step": 950 + }, + { + "epoch": 0.155023112169718, + "grad_norm": 3.7156920433044434, + "learning_rate": 0.0001, + "loss": 124.8085, + "step": 960 + }, + { + "epoch": 0.15663793625481925, + "grad_norm": 3.0576517581939697, + "learning_rate": 0.0001, + "loss": 125.9107, + "step": 970 + }, + { + "epoch": 0.15825276033992047, + "grad_norm": 1.9951574802398682, + "learning_rate": 0.0001, + "loss": 123.5359, + "step": 980 + }, + { + "epoch": 0.1598675844250217, + "grad_norm": 1.4620885848999023, + "learning_rate": 0.0001, + "loss": 121.9363, + "step": 990 + }, + { + "epoch": 0.16148240851012294, + "grad_norm": 1.7640169858932495, + "learning_rate": 0.0001, + "loss": 121.1764, + "step": 1000 + }, + { + "epoch": 0.16309723259522416, + "grad_norm": 2.921750545501709, + "learning_rate": 0.0001, + "loss": 119.6931, + "step": 1010 + }, + { + "epoch": 0.16471205668032537, + "grad_norm": 1.5623588562011719, + "learning_rate": 0.0001, + "loss": 119.097, + "step": 1020 + }, + { + "epoch": 0.16632688076542662, + "grad_norm": 2.2386627197265625, + "learning_rate": 0.0001, + "loss": 120.9224, + "step": 1030 + }, + { + "epoch": 0.16794170485052784, + "grad_norm": 1.3213791847229004, + "learning_rate": 0.0001, + "loss": 119.5649, + "step": 1040 + }, + { + "epoch": 0.16955652893562909, + "grad_norm": 1.5255440473556519, + "learning_rate": 0.0001, + "loss": 118.1524, + "step": 1050 + }, + { + "epoch": 0.1711713530207303, + "grad_norm": 1.6038514375686646, + "learning_rate": 0.0001, + "loss": 118.6307, + "step": 1060 + }, + { + "epoch": 0.17278617710583152, + "grad_norm": 2.356739044189453, + "learning_rate": 0.0001, + "loss": 116.8472, + "step": 1070 + }, + { + "epoch": 0.17440100119093277, + "grad_norm": 2.5326743125915527, + "learning_rate": 0.0001, + "loss": 115.6625, + "step": 1080 + }, + { + "epoch": 0.176015825276034, + "grad_norm": 1.2823396921157837, + "learning_rate": 0.0001, + "loss": 115.3312, + "step": 1090 + }, + { + "epoch": 0.17763064936113523, + "grad_norm": 1.981788158416748, + "learning_rate": 0.0001, + "loss": 116.2257, + "step": 1100 + }, + { + "epoch": 0.17924547344623645, + "grad_norm": 1.540440559387207, + "learning_rate": 0.0001, + "loss": 114.5031, + "step": 1110 + }, + { + "epoch": 0.18086029753133767, + "grad_norm": 1.7667800188064575, + "learning_rate": 0.0001, + "loss": 113.7663, + "step": 1120 + }, + { + "epoch": 0.18247512161643892, + "grad_norm": 2.2793102264404297, + "learning_rate": 0.0001, + "loss": 115.219, + "step": 1130 + }, + { + "epoch": 0.18408994570154014, + "grad_norm": 2.132046699523926, + "learning_rate": 0.0001, + "loss": 112.8482, + "step": 1140 + }, + { + "epoch": 0.18570476978664135, + "grad_norm": 1.3153988122940063, + "learning_rate": 0.0001, + "loss": 111.4546, + "step": 1150 + }, + { + "epoch": 0.1873195938717426, + "grad_norm": 1.5246292352676392, + "learning_rate": 0.0001, + "loss": 112.4348, + "step": 1160 + }, + { + "epoch": 0.18893441795684382, + "grad_norm": 1.365096092224121, + "learning_rate": 0.0001, + "loss": 109.755, + "step": 1170 + }, + { + "epoch": 0.19054924204194507, + "grad_norm": 1.2777953147888184, + "learning_rate": 0.0001, + "loss": 111.0318, + "step": 1180 + }, + { + "epoch": 0.19216406612704628, + "grad_norm": 1.8672829866409302, + "learning_rate": 0.0001, + "loss": 109.1648, + "step": 1190 + }, + { + "epoch": 0.1937788902121475, + "grad_norm": 1.709987759590149, + "learning_rate": 0.0001, + "loss": 108.1616, + "step": 1200 + }, + { + "epoch": 0.19539371429724875, + "grad_norm": 1.176277995109558, + "learning_rate": 0.0001, + "loss": 109.3591, + "step": 1210 + }, + { + "epoch": 0.19700853838234997, + "grad_norm": 1.4181180000305176, + "learning_rate": 0.0001, + "loss": 110.0216, + "step": 1220 + }, + { + "epoch": 0.19862336246745121, + "grad_norm": 1.3246116638183594, + "learning_rate": 0.0001, + "loss": 107.9769, + "step": 1230 + }, + { + "epoch": 0.20023818655255243, + "grad_norm": 1.2501498460769653, + "learning_rate": 0.0001, + "loss": 108.2601, + "step": 1240 + }, + { + "epoch": 0.20185301063765365, + "grad_norm": 1.2277404069900513, + "learning_rate": 0.0001, + "loss": 107.1258, + "step": 1250 + }, + { + "epoch": 0.2034678347227549, + "grad_norm": 1.8064894676208496, + "learning_rate": 0.0001, + "loss": 105.6278, + "step": 1260 + }, + { + "epoch": 0.20508265880785612, + "grad_norm": 1.708897352218628, + "learning_rate": 0.0001, + "loss": 106.9269, + "step": 1270 + }, + { + "epoch": 0.20669748289295733, + "grad_norm": 1.4507478475570679, + "learning_rate": 0.0001, + "loss": 106.9236, + "step": 1280 + }, + { + "epoch": 0.20831230697805858, + "grad_norm": 1.4904310703277588, + "learning_rate": 0.0001, + "loss": 106.0653, + "step": 1290 + }, + { + "epoch": 0.2099271310631598, + "grad_norm": 1.2637165784835815, + "learning_rate": 0.0001, + "loss": 104.8665, + "step": 1300 + }, + { + "epoch": 0.21154195514826105, + "grad_norm": 1.4085133075714111, + "learning_rate": 0.0001, + "loss": 104.8891, + "step": 1310 + }, + { + "epoch": 0.21315677923336226, + "grad_norm": 1.5710629224777222, + "learning_rate": 0.0001, + "loss": 104.3009, + "step": 1320 + }, + { + "epoch": 0.21477160331846348, + "grad_norm": 1.3634663820266724, + "learning_rate": 0.0001, + "loss": 104.4545, + "step": 1330 + }, + { + "epoch": 0.21638642740356473, + "grad_norm": 1.5565859079360962, + "learning_rate": 0.0001, + "loss": 103.6205, + "step": 1340 + }, + { + "epoch": 0.21800125148866595, + "grad_norm": 1.4491888284683228, + "learning_rate": 0.0001, + "loss": 104.0892, + "step": 1350 + }, + { + "epoch": 0.2196160755737672, + "grad_norm": 1.7609063386917114, + "learning_rate": 0.0001, + "loss": 104.5177, + "step": 1360 + }, + { + "epoch": 0.2212308996588684, + "grad_norm": 1.6723753213882446, + "learning_rate": 0.0001, + "loss": 104.5552, + "step": 1370 + }, + { + "epoch": 0.22284572374396963, + "grad_norm": 1.3781665563583374, + "learning_rate": 0.0001, + "loss": 102.1022, + "step": 1380 + }, + { + "epoch": 0.22446054782907088, + "grad_norm": 1.5397366285324097, + "learning_rate": 0.0001, + "loss": 102.5882, + "step": 1390 + }, + { + "epoch": 0.2260753719141721, + "grad_norm": 1.4567879438400269, + "learning_rate": 0.0001, + "loss": 101.3781, + "step": 1400 + }, + { + "epoch": 0.22769019599927334, + "grad_norm": 1.2882301807403564, + "learning_rate": 0.0001, + "loss": 100.9378, + "step": 1410 + }, + { + "epoch": 0.22930502008437456, + "grad_norm": 1.4510424137115479, + "learning_rate": 0.0001, + "loss": 101.5383, + "step": 1420 + }, + { + "epoch": 0.23091984416947578, + "grad_norm": 2.8331449031829834, + "learning_rate": 0.0001, + "loss": 100.2773, + "step": 1430 + }, + { + "epoch": 0.23253466825457703, + "grad_norm": 1.363616943359375, + "learning_rate": 0.0001, + "loss": 101.646, + "step": 1440 + }, + { + "epoch": 0.23414949233967824, + "grad_norm": 1.2184548377990723, + "learning_rate": 0.0001, + "loss": 101.1133, + "step": 1450 + }, + { + "epoch": 0.23576431642477946, + "grad_norm": 2.235656976699829, + "learning_rate": 0.0001, + "loss": 100.2044, + "step": 1460 + }, + { + "epoch": 0.2373791405098807, + "grad_norm": 1.2334418296813965, + "learning_rate": 0.0001, + "loss": 99.4031, + "step": 1470 + }, + { + "epoch": 0.23899396459498193, + "grad_norm": 1.2394684553146362, + "learning_rate": 0.0001, + "loss": 101.1247, + "step": 1480 + }, + { + "epoch": 0.24060878868008317, + "grad_norm": 1.5218186378479004, + "learning_rate": 0.0001, + "loss": 100.4413, + "step": 1490 + }, + { + "epoch": 0.2422236127651844, + "grad_norm": 1.2112364768981934, + "learning_rate": 0.0001, + "loss": 99.8722, + "step": 1500 + }, + { + "epoch": 0.2438384368502856, + "grad_norm": 1.2277610301971436, + "learning_rate": 0.0001, + "loss": 99.0276, + "step": 1510 + }, + { + "epoch": 0.24545326093538686, + "grad_norm": 1.1005053520202637, + "learning_rate": 0.0001, + "loss": 98.6764, + "step": 1520 + }, + { + "epoch": 0.24706808502048808, + "grad_norm": 1.0754456520080566, + "learning_rate": 0.0001, + "loss": 97.8968, + "step": 1530 + }, + { + "epoch": 0.24868290910558932, + "grad_norm": 1.0695950984954834, + "learning_rate": 0.0001, + "loss": 98.4032, + "step": 1540 + }, + { + "epoch": 0.25029773319069054, + "grad_norm": 1.5098627805709839, + "learning_rate": 0.0001, + "loss": 97.9391, + "step": 1550 + }, + { + "epoch": 0.25191255727579176, + "grad_norm": 1.006681203842163, + "learning_rate": 0.0001, + "loss": 96.5525, + "step": 1560 + }, + { + "epoch": 0.253527381360893, + "grad_norm": 1.3987332582473755, + "learning_rate": 0.0001, + "loss": 98.383, + "step": 1570 + }, + { + "epoch": 0.25514220544599425, + "grad_norm": 1.290887713432312, + "learning_rate": 0.0001, + "loss": 99.0901, + "step": 1580 + }, + { + "epoch": 0.25675702953109547, + "grad_norm": 0.991177499294281, + "learning_rate": 0.0001, + "loss": 97.7093, + "step": 1590 + }, + { + "epoch": 0.2583718536161967, + "grad_norm": 1.147647738456726, + "learning_rate": 0.0001, + "loss": 96.9181, + "step": 1600 + }, + { + "epoch": 0.2599866777012979, + "grad_norm": 1.1756514310836792, + "learning_rate": 0.0001, + "loss": 98.3935, + "step": 1610 + }, + { + "epoch": 0.2616015017863991, + "grad_norm": 1.1708602905273438, + "learning_rate": 0.0001, + "loss": 96.715, + "step": 1620 + }, + { + "epoch": 0.2632163258715004, + "grad_norm": 1.4577715396881104, + "learning_rate": 0.0001, + "loss": 97.2971, + "step": 1630 + }, + { + "epoch": 0.2648311499566016, + "grad_norm": 1.1172800064086914, + "learning_rate": 0.0001, + "loss": 97.6494, + "step": 1640 + }, + { + "epoch": 0.26644597404170284, + "grad_norm": 1.1491258144378662, + "learning_rate": 0.0001, + "loss": 97.5503, + "step": 1650 + }, + { + "epoch": 0.26806079812680406, + "grad_norm": 1.2188555002212524, + "learning_rate": 0.0001, + "loss": 95.3423, + "step": 1660 + }, + { + "epoch": 0.2696756222119053, + "grad_norm": 1.8595614433288574, + "learning_rate": 0.0001, + "loss": 96.0112, + "step": 1670 + }, + { + "epoch": 0.2712904462970065, + "grad_norm": 1.1887950897216797, + "learning_rate": 0.0001, + "loss": 95.4939, + "step": 1680 + }, + { + "epoch": 0.27290527038210777, + "grad_norm": 1.1777839660644531, + "learning_rate": 0.0001, + "loss": 95.9619, + "step": 1690 + }, + { + "epoch": 0.274520094467209, + "grad_norm": 1.1241233348846436, + "learning_rate": 0.0001, + "loss": 96.2834, + "step": 1700 + }, + { + "epoch": 0.2761349185523102, + "grad_norm": 1.0900121927261353, + "learning_rate": 0.0001, + "loss": 94.7518, + "step": 1710 + }, + { + "epoch": 0.2777497426374114, + "grad_norm": 1.0615205764770508, + "learning_rate": 0.0001, + "loss": 95.8032, + "step": 1720 + }, + { + "epoch": 0.27936456672251264, + "grad_norm": 1.2626168727874756, + "learning_rate": 0.0001, + "loss": 93.9454, + "step": 1730 + }, + { + "epoch": 0.2809793908076139, + "grad_norm": 0.8869941830635071, + "learning_rate": 0.0001, + "loss": 95.4085, + "step": 1740 + }, + { + "epoch": 0.28259421489271513, + "grad_norm": 1.0350773334503174, + "learning_rate": 0.0001, + "loss": 95.7954, + "step": 1750 + }, + { + "epoch": 0.28420903897781635, + "grad_norm": 1.364768147468567, + "learning_rate": 0.0001, + "loss": 93.6447, + "step": 1760 + }, + { + "epoch": 0.28582386306291757, + "grad_norm": 1.0115883350372314, + "learning_rate": 0.0001, + "loss": 93.7905, + "step": 1770 + }, + { + "epoch": 0.2874386871480188, + "grad_norm": 0.9393435120582581, + "learning_rate": 0.0001, + "loss": 94.3108, + "step": 1780 + }, + { + "epoch": 0.28905351123312006, + "grad_norm": 1.0769627094268799, + "learning_rate": 0.0001, + "loss": 92.566, + "step": 1790 + }, + { + "epoch": 0.2906683353182213, + "grad_norm": 2.526201009750366, + "learning_rate": 0.0001, + "loss": 92.8161, + "step": 1800 + }, + { + "epoch": 0.2922831594033225, + "grad_norm": 1.3066468238830566, + "learning_rate": 0.0001, + "loss": 92.0459, + "step": 1810 + }, + { + "epoch": 0.2938979834884237, + "grad_norm": 1.121713638305664, + "learning_rate": 0.0001, + "loss": 93.5032, + "step": 1820 + }, + { + "epoch": 0.29551280757352494, + "grad_norm": 1.3744316101074219, + "learning_rate": 0.0001, + "loss": 94.2623, + "step": 1830 + }, + { + "epoch": 0.2971276316586262, + "grad_norm": 1.1561540365219116, + "learning_rate": 0.0001, + "loss": 94.1611, + "step": 1840 + }, + { + "epoch": 0.29874245574372743, + "grad_norm": 1.3947464227676392, + "learning_rate": 0.0001, + "loss": 93.5962, + "step": 1850 + }, + { + "epoch": 0.30035727982882865, + "grad_norm": 1.0011566877365112, + "learning_rate": 0.0001, + "loss": 92.0199, + "step": 1860 + }, + { + "epoch": 0.30197210391392987, + "grad_norm": 0.9978747963905334, + "learning_rate": 0.0001, + "loss": 91.0945, + "step": 1870 + }, + { + "epoch": 0.3035869279990311, + "grad_norm": 0.92522132396698, + "learning_rate": 0.0001, + "loss": 93.274, + "step": 1880 + }, + { + "epoch": 0.30520175208413236, + "grad_norm": 1.0089120864868164, + "learning_rate": 0.0001, + "loss": 91.9308, + "step": 1890 + }, + { + "epoch": 0.3068165761692336, + "grad_norm": 1.0535134077072144, + "learning_rate": 0.0001, + "loss": 92.563, + "step": 1900 + }, + { + "epoch": 0.3084314002543348, + "grad_norm": 0.9782275557518005, + "learning_rate": 0.0001, + "loss": 90.3486, + "step": 1910 + }, + { + "epoch": 0.310046224339436, + "grad_norm": 1.1433162689208984, + "learning_rate": 0.0001, + "loss": 90.4977, + "step": 1920 + }, + { + "epoch": 0.31166104842453723, + "grad_norm": 0.98392653465271, + "learning_rate": 0.0001, + "loss": 92.6529, + "step": 1930 + }, + { + "epoch": 0.3132758725096385, + "grad_norm": 0.8538148403167725, + "learning_rate": 0.0001, + "loss": 90.6859, + "step": 1940 + }, + { + "epoch": 0.3148906965947397, + "grad_norm": 1.019405484199524, + "learning_rate": 0.0001, + "loss": 91.0028, + "step": 1950 + }, + { + "epoch": 0.31650552067984095, + "grad_norm": 0.9828067421913147, + "learning_rate": 0.0001, + "loss": 89.9826, + "step": 1960 + }, + { + "epoch": 0.31812034476494216, + "grad_norm": 0.9725450277328491, + "learning_rate": 0.0001, + "loss": 91.7342, + "step": 1970 + }, + { + "epoch": 0.3197351688500434, + "grad_norm": 1.0139408111572266, + "learning_rate": 0.0001, + "loss": 90.5362, + "step": 1980 + }, + { + "epoch": 0.3213499929351446, + "grad_norm": 1.2493853569030762, + "learning_rate": 0.0001, + "loss": 92.7222, + "step": 1990 + }, + { + "epoch": 0.3229648170202459, + "grad_norm": 1.0152915716171265, + "learning_rate": 0.0001, + "loss": 92.2271, + "step": 2000 + }, + { + "epoch": 0.3245796411053471, + "grad_norm": 0.9745819568634033, + "learning_rate": 0.0001, + "loss": 88.7034, + "step": 2010 + }, + { + "epoch": 0.3261944651904483, + "grad_norm": 1.009624719619751, + "learning_rate": 0.0001, + "loss": 90.7428, + "step": 2020 + }, + { + "epoch": 0.32780928927554953, + "grad_norm": 0.910858690738678, + "learning_rate": 0.0001, + "loss": 89.8199, + "step": 2030 + }, + { + "epoch": 0.32942411336065075, + "grad_norm": 1.1745878458023071, + "learning_rate": 0.0001, + "loss": 90.8267, + "step": 2040 + }, + { + "epoch": 0.331038937445752, + "grad_norm": 0.9409273266792297, + "learning_rate": 0.0001, + "loss": 90.2937, + "step": 2050 + }, + { + "epoch": 0.33265376153085324, + "grad_norm": 1.4152942895889282, + "learning_rate": 0.0001, + "loss": 90.3965, + "step": 2060 + }, + { + "epoch": 0.33426858561595446, + "grad_norm": 0.9529693722724915, + "learning_rate": 0.0001, + "loss": 90.4739, + "step": 2070 + }, + { + "epoch": 0.3358834097010557, + "grad_norm": 0.8531096577644348, + "learning_rate": 0.0001, + "loss": 90.9301, + "step": 2080 + }, + { + "epoch": 0.3374982337861569, + "grad_norm": 0.9314167499542236, + "learning_rate": 0.0001, + "loss": 90.4235, + "step": 2090 + }, + { + "epoch": 0.33911305787125817, + "grad_norm": 0.9253250360488892, + "learning_rate": 0.0001, + "loss": 88.1893, + "step": 2100 + }, + { + "epoch": 0.3407278819563594, + "grad_norm": 1.773053526878357, + "learning_rate": 0.0001, + "loss": 88.3417, + "step": 2110 + }, + { + "epoch": 0.3423427060414606, + "grad_norm": 0.9432951211929321, + "learning_rate": 0.0001, + "loss": 89.7141, + "step": 2120 + }, + { + "epoch": 0.34395753012656183, + "grad_norm": 3.8847527503967285, + "learning_rate": 0.0001, + "loss": 89.1009, + "step": 2130 + }, + { + "epoch": 0.34557235421166305, + "grad_norm": 1.0309821367263794, + "learning_rate": 0.0001, + "loss": 89.2965, + "step": 2140 + }, + { + "epoch": 0.3471871782967643, + "grad_norm": 0.9590495824813843, + "learning_rate": 0.0001, + "loss": 89.1734, + "step": 2150 + }, + { + "epoch": 0.34880200238186554, + "grad_norm": 0.9943560361862183, + "learning_rate": 0.0001, + "loss": 85.425, + "step": 2160 + }, + { + "epoch": 0.35041682646696676, + "grad_norm": 1.3322339057922363, + "learning_rate": 0.0001, + "loss": 89.7791, + "step": 2170 + }, + { + "epoch": 0.352031650552068, + "grad_norm": 1.0212253332138062, + "learning_rate": 0.0001, + "loss": 89.3771, + "step": 2180 + }, + { + "epoch": 0.3536464746371692, + "grad_norm": 1.0446089506149292, + "learning_rate": 0.0001, + "loss": 88.6894, + "step": 2190 + }, + { + "epoch": 0.35526129872227047, + "grad_norm": 1.05767822265625, + "learning_rate": 0.0001, + "loss": 89.5341, + "step": 2200 + }, + { + "epoch": 0.3568761228073717, + "grad_norm": 1.0373681783676147, + "learning_rate": 0.0001, + "loss": 89.7173, + "step": 2210 + }, + { + "epoch": 0.3584909468924729, + "grad_norm": 0.9098039269447327, + "learning_rate": 0.0001, + "loss": 89.2116, + "step": 2220 + }, + { + "epoch": 0.3601057709775741, + "grad_norm": 1.028130292892456, + "learning_rate": 0.0001, + "loss": 88.6627, + "step": 2230 + }, + { + "epoch": 0.36172059506267534, + "grad_norm": 1.0295171737670898, + "learning_rate": 0.0001, + "loss": 89.608, + "step": 2240 + }, + { + "epoch": 0.36333541914777656, + "grad_norm": 0.9720722436904907, + "learning_rate": 0.0001, + "loss": 88.3624, + "step": 2250 + }, + { + "epoch": 0.36495024323287784, + "grad_norm": 0.839654803276062, + "learning_rate": 0.0001, + "loss": 87.2983, + "step": 2260 + }, + { + "epoch": 0.36656506731797905, + "grad_norm": 1.1600219011306763, + "learning_rate": 0.0001, + "loss": 86.6004, + "step": 2270 + }, + { + "epoch": 0.3681798914030803, + "grad_norm": 1.0513924360275269, + "learning_rate": 0.0001, + "loss": 86.6062, + "step": 2280 + }, + { + "epoch": 0.3697947154881815, + "grad_norm": 0.9728474617004395, + "learning_rate": 0.0001, + "loss": 86.1856, + "step": 2290 + }, + { + "epoch": 0.3714095395732827, + "grad_norm": 1.0196386575698853, + "learning_rate": 0.0001, + "loss": 87.6088, + "step": 2300 + }, + { + "epoch": 0.373024363658384, + "grad_norm": 1.0439444780349731, + "learning_rate": 0.0001, + "loss": 85.2374, + "step": 2310 + }, + { + "epoch": 0.3746391877434852, + "grad_norm": 1.0020275115966797, + "learning_rate": 0.0001, + "loss": 86.0596, + "step": 2320 + }, + { + "epoch": 0.3762540118285864, + "grad_norm": 1.0107691287994385, + "learning_rate": 0.0001, + "loss": 86.9513, + "step": 2330 + }, + { + "epoch": 0.37786883591368764, + "grad_norm": 1.680018663406372, + "learning_rate": 0.0001, + "loss": 87.8885, + "step": 2340 + }, + { + "epoch": 0.37948365999878886, + "grad_norm": 0.9181709885597229, + "learning_rate": 0.0001, + "loss": 86.7662, + "step": 2350 + }, + { + "epoch": 0.38109848408389013, + "grad_norm": 0.9598278403282166, + "learning_rate": 0.0001, + "loss": 88.3775, + "step": 2360 + }, + { + "epoch": 0.38271330816899135, + "grad_norm": 1.0707905292510986, + "learning_rate": 0.0001, + "loss": 86.0997, + "step": 2370 + }, + { + "epoch": 0.38432813225409257, + "grad_norm": 1.125797152519226, + "learning_rate": 0.0001, + "loss": 85.8117, + "step": 2380 + }, + { + "epoch": 0.3859429563391938, + "grad_norm": 1.0251439809799194, + "learning_rate": 0.0001, + "loss": 86.0806, + "step": 2390 + }, + { + "epoch": 0.387557780424295, + "grad_norm": 1.0193125009536743, + "learning_rate": 0.0001, + "loss": 87.264, + "step": 2400 + }, + { + "epoch": 0.3891726045093963, + "grad_norm": 0.9309751391410828, + "learning_rate": 0.0001, + "loss": 85.9187, + "step": 2410 + }, + { + "epoch": 0.3907874285944975, + "grad_norm": 1.1025906801223755, + "learning_rate": 0.0001, + "loss": 86.4365, + "step": 2420 + }, + { + "epoch": 0.3924022526795987, + "grad_norm": 0.9880627393722534, + "learning_rate": 0.0001, + "loss": 86.2303, + "step": 2430 + }, + { + "epoch": 0.39401707676469994, + "grad_norm": 0.9520611763000488, + "learning_rate": 0.0001, + "loss": 84.9035, + "step": 2440 + }, + { + "epoch": 0.39563190084980115, + "grad_norm": 0.8641896843910217, + "learning_rate": 0.0001, + "loss": 85.4754, + "step": 2450 + }, + { + "epoch": 0.39724672493490243, + "grad_norm": 1.1162904500961304, + "learning_rate": 0.0001, + "loss": 85.6767, + "step": 2460 + }, + { + "epoch": 0.39886154902000365, + "grad_norm": 0.9073636531829834, + "learning_rate": 0.0001, + "loss": 85.4198, + "step": 2470 + }, + { + "epoch": 0.40047637310510487, + "grad_norm": 1.1135166883468628, + "learning_rate": 0.0001, + "loss": 86.0884, + "step": 2480 + }, + { + "epoch": 0.4020911971902061, + "grad_norm": 0.9273980259895325, + "learning_rate": 0.0001, + "loss": 85.5473, + "step": 2490 + }, + { + "epoch": 0.4037060212753073, + "grad_norm": 0.8653831481933594, + "learning_rate": 0.0001, + "loss": 87.279, + "step": 2500 + }, + { + "epoch": 0.4053208453604086, + "grad_norm": 0.8535159826278687, + "learning_rate": 0.0001, + "loss": 86.7072, + "step": 2510 + }, + { + "epoch": 0.4069356694455098, + "grad_norm": 0.8864684700965881, + "learning_rate": 0.0001, + "loss": 86.441, + "step": 2520 + }, + { + "epoch": 0.408550493530611, + "grad_norm": 0.8042080998420715, + "learning_rate": 0.0001, + "loss": 83.1706, + "step": 2530 + }, + { + "epoch": 0.41016531761571223, + "grad_norm": 1.0486373901367188, + "learning_rate": 0.0001, + "loss": 83.54, + "step": 2540 + }, + { + "epoch": 0.41178014170081345, + "grad_norm": 0.8087959289550781, + "learning_rate": 0.0001, + "loss": 85.7615, + "step": 2550 + }, + { + "epoch": 0.41339496578591467, + "grad_norm": 1.5032018423080444, + "learning_rate": 0.0001, + "loss": 85.7368, + "step": 2560 + }, + { + "epoch": 0.41500978987101594, + "grad_norm": 1.0786182880401611, + "learning_rate": 0.0001, + "loss": 86.3484, + "step": 2570 + }, + { + "epoch": 0.41662461395611716, + "grad_norm": 0.8744294047355652, + "learning_rate": 0.0001, + "loss": 83.5945, + "step": 2580 + }, + { + "epoch": 0.4182394380412184, + "grad_norm": 0.7840721607208252, + "learning_rate": 0.0001, + "loss": 86.3831, + "step": 2590 + }, + { + "epoch": 0.4198542621263196, + "grad_norm": 0.7977836728096008, + "learning_rate": 0.0001, + "loss": 83.6314, + "step": 2600 + }, + { + "epoch": 0.4214690862114208, + "grad_norm": 0.892484188079834, + "learning_rate": 0.0001, + "loss": 86.4223, + "step": 2610 + }, + { + "epoch": 0.4230839102965221, + "grad_norm": 0.912995457649231, + "learning_rate": 0.0001, + "loss": 83.8971, + "step": 2620 + }, + { + "epoch": 0.4246987343816233, + "grad_norm": 0.9566861987113953, + "learning_rate": 0.0001, + "loss": 85.291, + "step": 2630 + }, + { + "epoch": 0.42631355846672453, + "grad_norm": 0.8324154615402222, + "learning_rate": 0.0001, + "loss": 84.6233, + "step": 2640 + }, + { + "epoch": 0.42792838255182575, + "grad_norm": 0.9490360617637634, + "learning_rate": 0.0001, + "loss": 81.799, + "step": 2650 + }, + { + "epoch": 0.42954320663692697, + "grad_norm": 0.9744566679000854, + "learning_rate": 0.0001, + "loss": 85.4837, + "step": 2660 + }, + { + "epoch": 0.43115803072202824, + "grad_norm": 0.941614031791687, + "learning_rate": 0.0001, + "loss": 82.838, + "step": 2670 + }, + { + "epoch": 0.43277285480712946, + "grad_norm": 1.338364601135254, + "learning_rate": 0.0001, + "loss": 84.4224, + "step": 2680 + }, + { + "epoch": 0.4343876788922307, + "grad_norm": 0.8643816113471985, + "learning_rate": 0.0001, + "loss": 84.3959, + "step": 2690 + }, + { + "epoch": 0.4360025029773319, + "grad_norm": 1.0231473445892334, + "learning_rate": 0.0001, + "loss": 82.3144, + "step": 2700 + }, + { + "epoch": 0.4376173270624331, + "grad_norm": 0.9089650511741638, + "learning_rate": 0.0001, + "loss": 83.0326, + "step": 2710 + }, + { + "epoch": 0.4392321511475344, + "grad_norm": 0.893628716468811, + "learning_rate": 0.0001, + "loss": 83.0981, + "step": 2720 + }, + { + "epoch": 0.4408469752326356, + "grad_norm": 0.869076132774353, + "learning_rate": 0.0001, + "loss": 84.2571, + "step": 2730 + }, + { + "epoch": 0.4424617993177368, + "grad_norm": 0.7964235544204712, + "learning_rate": 0.0001, + "loss": 83.3329, + "step": 2740 + }, + { + "epoch": 0.44407662340283804, + "grad_norm": 3.1645138263702393, + "learning_rate": 0.0001, + "loss": 83.7232, + "step": 2750 + }, + { + "epoch": 0.44569144748793926, + "grad_norm": 0.934644877910614, + "learning_rate": 0.0001, + "loss": 85.0688, + "step": 2760 + }, + { + "epoch": 0.44730627157304054, + "grad_norm": 0.8120711445808411, + "learning_rate": 0.0001, + "loss": 82.7706, + "step": 2770 + }, + { + "epoch": 0.44892109565814176, + "grad_norm": 0.8161435127258301, + "learning_rate": 0.0001, + "loss": 84.4826, + "step": 2780 + }, + { + "epoch": 0.450535919743243, + "grad_norm": 0.8659493327140808, + "learning_rate": 0.0001, + "loss": 82.6222, + "step": 2790 + }, + { + "epoch": 0.4521507438283442, + "grad_norm": 0.9306702613830566, + "learning_rate": 0.0001, + "loss": 83.5897, + "step": 2800 + }, + { + "epoch": 0.4537655679134454, + "grad_norm": 2.2544431686401367, + "learning_rate": 0.0001, + "loss": 82.9603, + "step": 2810 + }, + { + "epoch": 0.4553803919985467, + "grad_norm": 0.8868395686149597, + "learning_rate": 0.0001, + "loss": 82.6674, + "step": 2820 + }, + { + "epoch": 0.4569952160836479, + "grad_norm": 0.9336455464363098, + "learning_rate": 0.0001, + "loss": 84.6974, + "step": 2830 + }, + { + "epoch": 0.4586100401687491, + "grad_norm": 0.8814165592193604, + "learning_rate": 0.0001, + "loss": 83.9329, + "step": 2840 + }, + { + "epoch": 0.46022486425385034, + "grad_norm": 0.8125660419464111, + "learning_rate": 0.0001, + "loss": 84.6168, + "step": 2850 + }, + { + "epoch": 0.46183968833895156, + "grad_norm": 0.9447190165519714, + "learning_rate": 0.0001, + "loss": 82.4098, + "step": 2860 + }, + { + "epoch": 0.4634545124240528, + "grad_norm": 0.8080969452857971, + "learning_rate": 0.0001, + "loss": 83.4076, + "step": 2870 + }, + { + "epoch": 0.46506933650915405, + "grad_norm": 1.0586694478988647, + "learning_rate": 0.0001, + "loss": 83.3295, + "step": 2880 + }, + { + "epoch": 0.46668416059425527, + "grad_norm": 0.8207664489746094, + "learning_rate": 0.0001, + "loss": 84.5056, + "step": 2890 + }, + { + "epoch": 0.4682989846793565, + "grad_norm": 1.0647571086883545, + "learning_rate": 0.0001, + "loss": 83.0766, + "step": 2900 + }, + { + "epoch": 0.4699138087644577, + "grad_norm": 0.9416915774345398, + "learning_rate": 0.0001, + "loss": 83.8166, + "step": 2910 + }, + { + "epoch": 0.4715286328495589, + "grad_norm": 0.7479362487792969, + "learning_rate": 0.0001, + "loss": 82.7507, + "step": 2920 + }, + { + "epoch": 0.4731434569346602, + "grad_norm": 1.0775514841079712, + "learning_rate": 0.0001, + "loss": 84.4641, + "step": 2930 + }, + { + "epoch": 0.4747582810197614, + "grad_norm": 0.8736841082572937, + "learning_rate": 0.0001, + "loss": 82.0451, + "step": 2940 + }, + { + "epoch": 0.47637310510486264, + "grad_norm": 0.9500495791435242, + "learning_rate": 0.0001, + "loss": 82.6425, + "step": 2950 + }, + { + "epoch": 0.47798792918996386, + "grad_norm": 0.8551210165023804, + "learning_rate": 0.0001, + "loss": 81.1971, + "step": 2960 + }, + { + "epoch": 0.4796027532750651, + "grad_norm": 0.8149814605712891, + "learning_rate": 0.0001, + "loss": 82.7919, + "step": 2970 + }, + { + "epoch": 0.48121757736016635, + "grad_norm": 0.9012249112129211, + "learning_rate": 0.0001, + "loss": 81.975, + "step": 2980 + }, + { + "epoch": 0.48283240144526757, + "grad_norm": 1.3784620761871338, + "learning_rate": 0.0001, + "loss": 81.541, + "step": 2990 + }, + { + "epoch": 0.4844472255303688, + "grad_norm": 0.7921801209449768, + "learning_rate": 0.0001, + "loss": 83.6215, + "step": 3000 + }, + { + "epoch": 0.48606204961547, + "grad_norm": 0.9051580429077148, + "learning_rate": 0.0001, + "loss": 82.4048, + "step": 3010 + }, + { + "epoch": 0.4876768737005712, + "grad_norm": 1.0717192888259888, + "learning_rate": 0.0001, + "loss": 82.4079, + "step": 3020 + }, + { + "epoch": 0.4892916977856725, + "grad_norm": 0.8971781730651855, + "learning_rate": 0.0001, + "loss": 83.3668, + "step": 3030 + }, + { + "epoch": 0.4909065218707737, + "grad_norm": 0.9635320901870728, + "learning_rate": 0.0001, + "loss": 82.8244, + "step": 3040 + }, + { + "epoch": 0.49252134595587493, + "grad_norm": 1.1009949445724487, + "learning_rate": 0.0001, + "loss": 82.2174, + "step": 3050 + }, + { + "epoch": 0.49413617004097615, + "grad_norm": 0.9805473685264587, + "learning_rate": 0.0001, + "loss": 83.7897, + "step": 3060 + }, + { + "epoch": 0.49575099412607737, + "grad_norm": 1.2107361555099487, + "learning_rate": 0.0001, + "loss": 82.5108, + "step": 3070 + }, + { + "epoch": 0.49736581821117865, + "grad_norm": 0.9171829223632812, + "learning_rate": 0.0001, + "loss": 80.9126, + "step": 3080 + }, + { + "epoch": 0.49898064229627986, + "grad_norm": 0.8093295097351074, + "learning_rate": 0.0001, + "loss": 83.8147, + "step": 3090 + }, + { + "epoch": 0.5005954663813811, + "grad_norm": 1.2166653871536255, + "learning_rate": 0.0001, + "loss": 82.0288, + "step": 3100 + }, + { + "epoch": 0.5022102904664824, + "grad_norm": 1.0471502542495728, + "learning_rate": 0.0001, + "loss": 82.0706, + "step": 3110 + }, + { + "epoch": 0.5038251145515835, + "grad_norm": 1.007645845413208, + "learning_rate": 0.0001, + "loss": 81.7387, + "step": 3120 + }, + { + "epoch": 0.5054399386366848, + "grad_norm": 0.7252638339996338, + "learning_rate": 0.0001, + "loss": 82.9107, + "step": 3130 + }, + { + "epoch": 0.507054762721786, + "grad_norm": 0.8565683364868164, + "learning_rate": 0.0001, + "loss": 80.2935, + "step": 3140 + }, + { + "epoch": 0.5086695868068872, + "grad_norm": 0.9905404448509216, + "learning_rate": 0.0001, + "loss": 82.5201, + "step": 3150 + }, + { + "epoch": 0.5102844108919885, + "grad_norm": 0.887428343296051, + "learning_rate": 0.0001, + "loss": 80.8528, + "step": 3160 + }, + { + "epoch": 0.5118992349770897, + "grad_norm": 1.062942624092102, + "learning_rate": 0.0001, + "loss": 80.7035, + "step": 3170 + }, + { + "epoch": 0.5135140590621909, + "grad_norm": 1.203599214553833, + "learning_rate": 0.0001, + "loss": 81.2521, + "step": 3180 + }, + { + "epoch": 0.5151288831472921, + "grad_norm": 0.8452383875846863, + "learning_rate": 0.0001, + "loss": 81.6079, + "step": 3190 + }, + { + "epoch": 0.5167437072323934, + "grad_norm": 1.2530587911605835, + "learning_rate": 0.0001, + "loss": 82.2148, + "step": 3200 + }, + { + "epoch": 0.5183585313174947, + "grad_norm": 0.9351627826690674, + "learning_rate": 0.0001, + "loss": 82.7857, + "step": 3210 + }, + { + "epoch": 0.5199733554025958, + "grad_norm": 1.0128077268600464, + "learning_rate": 0.0001, + "loss": 81.0532, + "step": 3220 + }, + { + "epoch": 0.5215881794876971, + "grad_norm": 0.9316917061805725, + "learning_rate": 0.0001, + "loss": 80.7443, + "step": 3230 + }, + { + "epoch": 0.5232030035727983, + "grad_norm": 1.502705693244934, + "learning_rate": 0.0001, + "loss": 80.0608, + "step": 3240 + }, + { + "epoch": 0.5248178276578995, + "grad_norm": 0.8166837692260742, + "learning_rate": 0.0001, + "loss": 81.2013, + "step": 3250 + }, + { + "epoch": 0.5264326517430008, + "grad_norm": 0.8210042119026184, + "learning_rate": 0.0001, + "loss": 80.2491, + "step": 3260 + }, + { + "epoch": 0.528047475828102, + "grad_norm": 0.7854771018028259, + "learning_rate": 0.0001, + "loss": 81.1017, + "step": 3270 + }, + { + "epoch": 0.5296622999132032, + "grad_norm": 0.8530565500259399, + "learning_rate": 0.0001, + "loss": 80.8123, + "step": 3280 + }, + { + "epoch": 0.5312771239983044, + "grad_norm": 0.9446498155593872, + "learning_rate": 0.0001, + "loss": 80.0029, + "step": 3290 + }, + { + "epoch": 0.5328919480834057, + "grad_norm": 0.9646537899971008, + "learning_rate": 0.0001, + "loss": 81.5413, + "step": 3300 + }, + { + "epoch": 0.5345067721685068, + "grad_norm": 0.9262019395828247, + "learning_rate": 0.0001, + "loss": 80.9399, + "step": 3310 + }, + { + "epoch": 0.5361215962536081, + "grad_norm": 0.8515486121177673, + "learning_rate": 0.0001, + "loss": 80.9421, + "step": 3320 + }, + { + "epoch": 0.5377364203387094, + "grad_norm": 1.679321050643921, + "learning_rate": 0.0001, + "loss": 82.4851, + "step": 3330 + }, + { + "epoch": 0.5393512444238105, + "grad_norm": 0.8289812207221985, + "learning_rate": 0.0001, + "loss": 80.5664, + "step": 3340 + }, + { + "epoch": 0.5409660685089118, + "grad_norm": 0.7463789582252502, + "learning_rate": 0.0001, + "loss": 81.5193, + "step": 3350 + }, + { + "epoch": 0.542580892594013, + "grad_norm": 0.8332446217536926, + "learning_rate": 0.0001, + "loss": 80.8091, + "step": 3360 + }, + { + "epoch": 0.5441957166791143, + "grad_norm": 0.7612847089767456, + "learning_rate": 0.0001, + "loss": 79.8893, + "step": 3370 + }, + { + "epoch": 0.5458105407642155, + "grad_norm": 1.0557948350906372, + "learning_rate": 0.0001, + "loss": 81.4305, + "step": 3380 + }, + { + "epoch": 0.5474253648493167, + "grad_norm": 0.7975788116455078, + "learning_rate": 0.0001, + "loss": 80.0105, + "step": 3390 + }, + { + "epoch": 0.549040188934418, + "grad_norm": 0.9049227237701416, + "learning_rate": 0.0001, + "loss": 81.2689, + "step": 3400 + }, + { + "epoch": 0.5506550130195191, + "grad_norm": 1.1195895671844482, + "learning_rate": 0.0001, + "loss": 80.7163, + "step": 3410 + }, + { + "epoch": 0.5522698371046204, + "grad_norm": 0.7506540417671204, + "learning_rate": 0.0001, + "loss": 79.0855, + "step": 3420 + }, + { + "epoch": 0.5538846611897217, + "grad_norm": 0.73989337682724, + "learning_rate": 0.0001, + "loss": 81.1865, + "step": 3430 + }, + { + "epoch": 0.5554994852748228, + "grad_norm": 0.8418401479721069, + "learning_rate": 0.0001, + "loss": 79.5714, + "step": 3440 + }, + { + "epoch": 0.5571143093599241, + "grad_norm": 0.9807226061820984, + "learning_rate": 0.0001, + "loss": 78.5699, + "step": 3450 + }, + { + "epoch": 0.5587291334450253, + "grad_norm": 1.2890545129776, + "learning_rate": 0.0001, + "loss": 81.0128, + "step": 3460 + }, + { + "epoch": 0.5603439575301266, + "grad_norm": 0.8593361973762512, + "learning_rate": 0.0001, + "loss": 80.1638, + "step": 3470 + }, + { + "epoch": 0.5619587816152278, + "grad_norm": 0.7766354084014893, + "learning_rate": 0.0001, + "loss": 81.5597, + "step": 3480 + }, + { + "epoch": 0.563573605700329, + "grad_norm": 0.7785626649856567, + "learning_rate": 0.0001, + "loss": 79.3862, + "step": 3490 + }, + { + "epoch": 0.5651884297854303, + "grad_norm": 0.8429279327392578, + "learning_rate": 0.0001, + "loss": 76.6294, + "step": 3500 + }, + { + "epoch": 0.5668032538705314, + "grad_norm": 0.8141765594482422, + "learning_rate": 0.0001, + "loss": 80.2584, + "step": 3510 + }, + { + "epoch": 0.5684180779556327, + "grad_norm": 0.7065490484237671, + "learning_rate": 0.0001, + "loss": 79.3626, + "step": 3520 + }, + { + "epoch": 0.570032902040734, + "grad_norm": 0.7002031803131104, + "learning_rate": 0.0001, + "loss": 79.283, + "step": 3530 + }, + { + "epoch": 0.5716477261258351, + "grad_norm": 0.8715317845344543, + "learning_rate": 0.0001, + "loss": 80.3145, + "step": 3540 + }, + { + "epoch": 0.5732625502109364, + "grad_norm": 0.9584882855415344, + "learning_rate": 0.0001, + "loss": 78.7443, + "step": 3550 + }, + { + "epoch": 0.5748773742960376, + "grad_norm": 0.7908946871757507, + "learning_rate": 0.0001, + "loss": 79.1906, + "step": 3560 + }, + { + "epoch": 0.5764921983811389, + "grad_norm": 0.8416746258735657, + "learning_rate": 0.0001, + "loss": 77.8172, + "step": 3570 + }, + { + "epoch": 0.5781070224662401, + "grad_norm": 0.823667049407959, + "learning_rate": 0.0001, + "loss": 81.864, + "step": 3580 + }, + { + "epoch": 0.5797218465513413, + "grad_norm": 1.4282617568969727, + "learning_rate": 0.0001, + "loss": 79.9052, + "step": 3590 + }, + { + "epoch": 0.5813366706364426, + "grad_norm": 0.8747813701629639, + "learning_rate": 0.0001, + "loss": 78.8234, + "step": 3600 + }, + { + "epoch": 0.5829514947215437, + "grad_norm": 0.8415511846542358, + "learning_rate": 0.0001, + "loss": 78.8171, + "step": 3610 + }, + { + "epoch": 0.584566318806645, + "grad_norm": 0.9014492034912109, + "learning_rate": 0.0001, + "loss": 80.5011, + "step": 3620 + }, + { + "epoch": 0.5861811428917463, + "grad_norm": 0.7535204887390137, + "learning_rate": 0.0001, + "loss": 79.4895, + "step": 3630 + }, + { + "epoch": 0.5877959669768474, + "grad_norm": 0.8196508884429932, + "learning_rate": 0.0001, + "loss": 79.7101, + "step": 3640 + }, + { + "epoch": 0.5894107910619487, + "grad_norm": 1.5975492000579834, + "learning_rate": 0.0001, + "loss": 79.0687, + "step": 3650 + }, + { + "epoch": 0.5910256151470499, + "grad_norm": 0.9251692295074463, + "learning_rate": 0.0001, + "loss": 80.2377, + "step": 3660 + }, + { + "epoch": 0.5926404392321512, + "grad_norm": 0.7143557071685791, + "learning_rate": 0.0001, + "loss": 78.0671, + "step": 3670 + }, + { + "epoch": 0.5942552633172524, + "grad_norm": 0.7876290678977966, + "learning_rate": 0.0001, + "loss": 79.1455, + "step": 3680 + }, + { + "epoch": 0.5958700874023536, + "grad_norm": 1.5361895561218262, + "learning_rate": 0.0001, + "loss": 78.1253, + "step": 3690 + }, + { + "epoch": 0.5974849114874549, + "grad_norm": 0.7637040019035339, + "learning_rate": 0.0001, + "loss": 77.4658, + "step": 3700 + }, + { + "epoch": 0.599099735572556, + "grad_norm": 0.8166250586509705, + "learning_rate": 0.0001, + "loss": 79.5806, + "step": 3710 + }, + { + "epoch": 0.6007145596576573, + "grad_norm": 1.2767380475997925, + "learning_rate": 0.0001, + "loss": 79.1894, + "step": 3720 + }, + { + "epoch": 0.6023293837427586, + "grad_norm": 1.3048640489578247, + "learning_rate": 0.0001, + "loss": 78.0593, + "step": 3730 + }, + { + "epoch": 0.6039442078278597, + "grad_norm": 0.7945445775985718, + "learning_rate": 0.0001, + "loss": 79.7623, + "step": 3740 + }, + { + "epoch": 0.605559031912961, + "grad_norm": 0.7366836667060852, + "learning_rate": 0.0001, + "loss": 77.9571, + "step": 3750 + }, + { + "epoch": 0.6071738559980622, + "grad_norm": 0.9126115441322327, + "learning_rate": 0.0001, + "loss": 79.9502, + "step": 3760 + }, + { + "epoch": 0.6087886800831634, + "grad_norm": 0.800355851650238, + "learning_rate": 0.0001, + "loss": 78.7244, + "step": 3770 + }, + { + "epoch": 0.6104035041682647, + "grad_norm": 0.8290980458259583, + "learning_rate": 0.0001, + "loss": 78.7724, + "step": 3780 + }, + { + "epoch": 0.6120183282533659, + "grad_norm": 0.8512316942214966, + "learning_rate": 0.0001, + "loss": 78.6277, + "step": 3790 + }, + { + "epoch": 0.6136331523384672, + "grad_norm": 0.8909353613853455, + "learning_rate": 0.0001, + "loss": 78.5956, + "step": 3800 + }, + { + "epoch": 0.6152479764235683, + "grad_norm": 0.8328800797462463, + "learning_rate": 0.0001, + "loss": 78.1323, + "step": 3810 + }, + { + "epoch": 0.6168628005086696, + "grad_norm": 0.7950423359870911, + "learning_rate": 0.0001, + "loss": 78.7529, + "step": 3820 + }, + { + "epoch": 0.6184776245937709, + "grad_norm": 0.9988545179367065, + "learning_rate": 0.0001, + "loss": 80.7727, + "step": 3830 + }, + { + "epoch": 0.620092448678872, + "grad_norm": 0.7243085503578186, + "learning_rate": 0.0001, + "loss": 76.7616, + "step": 3840 + }, + { + "epoch": 0.6217072727639733, + "grad_norm": 0.7674855589866638, + "learning_rate": 0.0001, + "loss": 79.1938, + "step": 3850 + }, + { + "epoch": 0.6233220968490745, + "grad_norm": 0.8979483842849731, + "learning_rate": 0.0001, + "loss": 79.1522, + "step": 3860 + }, + { + "epoch": 0.6249369209341757, + "grad_norm": 0.7759745717048645, + "learning_rate": 0.0001, + "loss": 77.7001, + "step": 3870 + }, + { + "epoch": 0.626551745019277, + "grad_norm": 0.9940371513366699, + "learning_rate": 0.0001, + "loss": 79.8712, + "step": 3880 + }, + { + "epoch": 0.6281665691043782, + "grad_norm": 0.8103354573249817, + "learning_rate": 0.0001, + "loss": 77.5229, + "step": 3890 + }, + { + "epoch": 0.6297813931894795, + "grad_norm": 0.9188411831855774, + "learning_rate": 0.0001, + "loss": 78.2933, + "step": 3900 + }, + { + "epoch": 0.6313962172745806, + "grad_norm": 0.823017954826355, + "learning_rate": 0.0001, + "loss": 77.8692, + "step": 3910 + }, + { + "epoch": 0.6330110413596819, + "grad_norm": 0.8833358287811279, + "learning_rate": 0.0001, + "loss": 78.4465, + "step": 3920 + }, + { + "epoch": 0.634625865444783, + "grad_norm": 0.6880685687065125, + "learning_rate": 0.0001, + "loss": 78.8881, + "step": 3930 + }, + { + "epoch": 0.6362406895298843, + "grad_norm": 0.778455913066864, + "learning_rate": 0.0001, + "loss": 76.0277, + "step": 3940 + }, + { + "epoch": 0.6378555136149856, + "grad_norm": 0.8132772445678711, + "learning_rate": 0.0001, + "loss": 79.1284, + "step": 3950 + }, + { + "epoch": 0.6394703377000868, + "grad_norm": 0.7703788876533508, + "learning_rate": 0.0001, + "loss": 76.9013, + "step": 3960 + }, + { + "epoch": 0.641085161785188, + "grad_norm": 0.7876151204109192, + "learning_rate": 0.0001, + "loss": 77.1354, + "step": 3970 + }, + { + "epoch": 0.6426999858702892, + "grad_norm": 0.8027064800262451, + "learning_rate": 0.0001, + "loss": 77.4936, + "step": 3980 + }, + { + "epoch": 0.6443148099553905, + "grad_norm": 0.8261818885803223, + "learning_rate": 0.0001, + "loss": 76.7634, + "step": 3990 + }, + { + "epoch": 0.6459296340404918, + "grad_norm": 0.7024852633476257, + "learning_rate": 0.0001, + "loss": 79.951, + "step": 4000 + }, + { + "epoch": 0.6475444581255929, + "grad_norm": 1.015307068824768, + "learning_rate": 0.0001, + "loss": 76.0354, + "step": 4010 + }, + { + "epoch": 0.6491592822106942, + "grad_norm": 0.715346097946167, + "learning_rate": 0.0001, + "loss": 78.9021, + "step": 4020 + }, + { + "epoch": 0.6507741062957954, + "grad_norm": 0.8994929790496826, + "learning_rate": 0.0001, + "loss": 77.0587, + "step": 4030 + }, + { + "epoch": 0.6523889303808966, + "grad_norm": 0.7683436274528503, + "learning_rate": 0.0001, + "loss": 78.6943, + "step": 4040 + }, + { + "epoch": 0.6540037544659979, + "grad_norm": 0.7273952960968018, + "learning_rate": 0.0001, + "loss": 77.8396, + "step": 4050 + }, + { + "epoch": 0.6556185785510991, + "grad_norm": 1.0676878690719604, + "learning_rate": 0.0001, + "loss": 79.3449, + "step": 4060 + }, + { + "epoch": 0.6572334026362003, + "grad_norm": 0.6916168928146362, + "learning_rate": 0.0001, + "loss": 76.5462, + "step": 4070 + }, + { + "epoch": 0.6588482267213015, + "grad_norm": 0.6466719508171082, + "learning_rate": 0.0001, + "loss": 77.9354, + "step": 4080 + }, + { + "epoch": 0.6604630508064028, + "grad_norm": 0.7406665086746216, + "learning_rate": 0.0001, + "loss": 76.0135, + "step": 4090 + }, + { + "epoch": 0.662077874891504, + "grad_norm": 0.7104353904724121, + "learning_rate": 0.0001, + "loss": 78.3727, + "step": 4100 + }, + { + "epoch": 0.6636926989766052, + "grad_norm": 0.8328452110290527, + "learning_rate": 0.0001, + "loss": 77.5354, + "step": 4110 + }, + { + "epoch": 0.6653075230617065, + "grad_norm": 0.7012260556221008, + "learning_rate": 0.0001, + "loss": 78.7406, + "step": 4120 + }, + { + "epoch": 0.6669223471468076, + "grad_norm": 1.0436632633209229, + "learning_rate": 0.0001, + "loss": 78.0063, + "step": 4130 + }, + { + "epoch": 0.6685371712319089, + "grad_norm": 0.8471246361732483, + "learning_rate": 0.0001, + "loss": 77.8871, + "step": 4140 + }, + { + "epoch": 0.6701519953170102, + "grad_norm": 0.7378994822502136, + "learning_rate": 0.0001, + "loss": 76.9109, + "step": 4150 + }, + { + "epoch": 0.6717668194021114, + "grad_norm": 0.7778745889663696, + "learning_rate": 0.0001, + "loss": 78.3374, + "step": 4160 + }, + { + "epoch": 0.6733816434872126, + "grad_norm": 0.7841216325759888, + "learning_rate": 0.0001, + "loss": 78.1657, + "step": 4170 + }, + { + "epoch": 0.6749964675723138, + "grad_norm": 1.0027995109558105, + "learning_rate": 0.0001, + "loss": 76.6232, + "step": 4180 + }, + { + "epoch": 0.6766112916574151, + "grad_norm": 1.0034840106964111, + "learning_rate": 0.0001, + "loss": 79.5785, + "step": 4190 + }, + { + "epoch": 0.6782261157425163, + "grad_norm": 0.8399350643157959, + "learning_rate": 0.0001, + "loss": 76.8101, + "step": 4200 + }, + { + "epoch": 0.6798409398276175, + "grad_norm": 0.8864424824714661, + "learning_rate": 0.0001, + "loss": 77.8671, + "step": 4210 + }, + { + "epoch": 0.6814557639127188, + "grad_norm": 0.8322188258171082, + "learning_rate": 0.0001, + "loss": 77.6597, + "step": 4220 + }, + { + "epoch": 0.68307058799782, + "grad_norm": 0.8962367177009583, + "learning_rate": 0.0001, + "loss": 77.7324, + "step": 4230 + }, + { + "epoch": 0.6846854120829212, + "grad_norm": 0.9727205038070679, + "learning_rate": 0.0001, + "loss": 77.8531, + "step": 4240 + }, + { + "epoch": 0.6863002361680225, + "grad_norm": 0.956584095954895, + "learning_rate": 0.0001, + "loss": 78.709, + "step": 4250 + }, + { + "epoch": 0.6879150602531237, + "grad_norm": 0.8531600832939148, + "learning_rate": 0.0001, + "loss": 78.7755, + "step": 4260 + }, + { + "epoch": 0.6895298843382249, + "grad_norm": 0.8059844374656677, + "learning_rate": 0.0001, + "loss": 77.1636, + "step": 4270 + }, + { + "epoch": 0.6911447084233261, + "grad_norm": 0.8373456597328186, + "learning_rate": 0.0001, + "loss": 78.6561, + "step": 4280 + }, + { + "epoch": 0.6927595325084274, + "grad_norm": 0.7819933891296387, + "learning_rate": 0.0001, + "loss": 77.6061, + "step": 4290 + }, + { + "epoch": 0.6943743565935286, + "grad_norm": 0.8829817771911621, + "learning_rate": 0.0001, + "loss": 77.3387, + "step": 4300 + }, + { + "epoch": 0.6959891806786298, + "grad_norm": 0.791772723197937, + "learning_rate": 0.0001, + "loss": 77.4368, + "step": 4310 + }, + { + "epoch": 0.6976040047637311, + "grad_norm": 0.9949231147766113, + "learning_rate": 0.0001, + "loss": 73.4359, + "step": 4320 + }, + { + "epoch": 0.6992188288488322, + "grad_norm": 0.8676188588142395, + "learning_rate": 0.0001, + "loss": 78.4836, + "step": 4330 + }, + { + "epoch": 0.7008336529339335, + "grad_norm": 1.1704007387161255, + "learning_rate": 0.0001, + "loss": 76.5091, + "step": 4340 + }, + { + "epoch": 0.7024484770190348, + "grad_norm": 0.8877903819084167, + "learning_rate": 0.0001, + "loss": 78.021, + "step": 4350 + }, + { + "epoch": 0.704063301104136, + "grad_norm": 1.381914496421814, + "learning_rate": 0.0001, + "loss": 77.8544, + "step": 4360 + }, + { + "epoch": 0.7056781251892372, + "grad_norm": 0.835023045539856, + "learning_rate": 0.0001, + "loss": 76.1552, + "step": 4370 + }, + { + "epoch": 0.7072929492743384, + "grad_norm": 0.9814149141311646, + "learning_rate": 0.0001, + "loss": 76.7646, + "step": 4380 + }, + { + "epoch": 0.7089077733594397, + "grad_norm": 0.8012816309928894, + "learning_rate": 0.0001, + "loss": 75.794, + "step": 4390 + }, + { + "epoch": 0.7105225974445409, + "grad_norm": 0.8340234160423279, + "learning_rate": 0.0001, + "loss": 77.4417, + "step": 4400 + }, + { + "epoch": 0.7121374215296421, + "grad_norm": 0.7257950305938721, + "learning_rate": 0.0001, + "loss": 77.1807, + "step": 4410 + }, + { + "epoch": 0.7137522456147434, + "grad_norm": 0.7342708706855774, + "learning_rate": 0.0001, + "loss": 75.5091, + "step": 4420 + }, + { + "epoch": 0.7153670696998445, + "grad_norm": 0.8298904895782471, + "learning_rate": 0.0001, + "loss": 76.6651, + "step": 4430 + }, + { + "epoch": 0.7169818937849458, + "grad_norm": 0.8564396500587463, + "learning_rate": 0.0001, + "loss": 76.0055, + "step": 4440 + }, + { + "epoch": 0.7185967178700471, + "grad_norm": 0.9145296812057495, + "learning_rate": 0.0001, + "loss": 76.0369, + "step": 4450 + }, + { + "epoch": 0.7202115419551482, + "grad_norm": 0.8074191212654114, + "learning_rate": 0.0001, + "loss": 77.4436, + "step": 4460 + }, + { + "epoch": 0.7218263660402495, + "grad_norm": 0.8212682008743286, + "learning_rate": 0.0001, + "loss": 76.3129, + "step": 4470 + }, + { + "epoch": 0.7234411901253507, + "grad_norm": 0.751323938369751, + "learning_rate": 0.0001, + "loss": 77.603, + "step": 4480 + }, + { + "epoch": 0.725056014210452, + "grad_norm": 0.9664170145988464, + "learning_rate": 0.0001, + "loss": 77.481, + "step": 4490 + }, + { + "epoch": 0.7266708382955531, + "grad_norm": 0.7369455695152283, + "learning_rate": 0.0001, + "loss": 76.3238, + "step": 4500 + }, + { + "epoch": 0.7282856623806544, + "grad_norm": 0.8448513746261597, + "learning_rate": 0.0001, + "loss": 76.5679, + "step": 4510 + }, + { + "epoch": 0.7299004864657557, + "grad_norm": 0.8605676293373108, + "learning_rate": 0.0001, + "loss": 75.7667, + "step": 4520 + }, + { + "epoch": 0.7315153105508568, + "grad_norm": 0.8034664392471313, + "learning_rate": 0.0001, + "loss": 76.2163, + "step": 4530 + }, + { + "epoch": 0.7331301346359581, + "grad_norm": 0.7846826910972595, + "learning_rate": 0.0001, + "loss": 73.5979, + "step": 4540 + }, + { + "epoch": 0.7347449587210593, + "grad_norm": 0.7422606348991394, + "learning_rate": 0.0001, + "loss": 76.2334, + "step": 4550 + }, + { + "epoch": 0.7363597828061605, + "grad_norm": 1.4478483200073242, + "learning_rate": 0.0001, + "loss": 77.6168, + "step": 4560 + }, + { + "epoch": 0.7379746068912618, + "grad_norm": 2.0957767963409424, + "learning_rate": 0.0001, + "loss": 76.3199, + "step": 4570 + }, + { + "epoch": 0.739589430976363, + "grad_norm": 0.6757177114486694, + "learning_rate": 0.0001, + "loss": 77.2923, + "step": 4580 + }, + { + "epoch": 0.7412042550614643, + "grad_norm": 0.9281920790672302, + "learning_rate": 0.0001, + "loss": 76.9963, + "step": 4590 + }, + { + "epoch": 0.7428190791465654, + "grad_norm": 0.8702669143676758, + "learning_rate": 0.0001, + "loss": 76.2553, + "step": 4600 + }, + { + "epoch": 0.7444339032316667, + "grad_norm": 0.7976512312889099, + "learning_rate": 0.0001, + "loss": 74.7622, + "step": 4610 + }, + { + "epoch": 0.746048727316768, + "grad_norm": 0.9116189479827881, + "learning_rate": 0.0001, + "loss": 76.3198, + "step": 4620 + }, + { + "epoch": 0.7476635514018691, + "grad_norm": 0.6239303946495056, + "learning_rate": 0.0001, + "loss": 75.7678, + "step": 4630 + }, + { + "epoch": 0.7492783754869704, + "grad_norm": 0.8380976915359497, + "learning_rate": 0.0001, + "loss": 76.8305, + "step": 4640 + }, + { + "epoch": 0.7508931995720716, + "grad_norm": 0.7394141554832458, + "learning_rate": 0.0001, + "loss": 76.5033, + "step": 4650 + }, + { + "epoch": 0.7525080236571728, + "grad_norm": 1.0694036483764648, + "learning_rate": 0.0001, + "loss": 75.0084, + "step": 4660 + }, + { + "epoch": 0.7541228477422741, + "grad_norm": 0.7530115246772766, + "learning_rate": 0.0001, + "loss": 74.4977, + "step": 4670 + }, + { + "epoch": 0.7557376718273753, + "grad_norm": 1.9561909437179565, + "learning_rate": 0.0001, + "loss": 74.7077, + "step": 4680 + }, + { + "epoch": 0.7573524959124766, + "grad_norm": 0.625341534614563, + "learning_rate": 0.0001, + "loss": 76.1238, + "step": 4690 + }, + { + "epoch": 0.7589673199975777, + "grad_norm": 0.9790500998497009, + "learning_rate": 0.0001, + "loss": 76.3997, + "step": 4700 + }, + { + "epoch": 0.760582144082679, + "grad_norm": 0.7087915539741516, + "learning_rate": 0.0001, + "loss": 75.6893, + "step": 4710 + }, + { + "epoch": 0.7621969681677803, + "grad_norm": 1.0995875597000122, + "learning_rate": 0.0001, + "loss": 74.8588, + "step": 4720 + }, + { + "epoch": 0.7638117922528814, + "grad_norm": 1.0309056043624878, + "learning_rate": 0.0001, + "loss": 73.5934, + "step": 4730 + }, + { + "epoch": 0.7654266163379827, + "grad_norm": 1.0132921934127808, + "learning_rate": 0.0001, + "loss": 73.9868, + "step": 4740 + }, + { + "epoch": 0.7670414404230839, + "grad_norm": 0.7138039469718933, + "learning_rate": 0.0001, + "loss": 76.4459, + "step": 4750 + }, + { + "epoch": 0.7686562645081851, + "grad_norm": 0.7055657505989075, + "learning_rate": 0.0001, + "loss": 75.1068, + "step": 4760 + }, + { + "epoch": 0.7702710885932864, + "grad_norm": 0.7398490905761719, + "learning_rate": 0.0001, + "loss": 75.3697, + "step": 4770 + }, + { + "epoch": 0.7718859126783876, + "grad_norm": 0.6836295127868652, + "learning_rate": 0.0001, + "loss": 77.4988, + "step": 4780 + }, + { + "epoch": 0.7735007367634888, + "grad_norm": 0.7552574276924133, + "learning_rate": 0.0001, + "loss": 75.149, + "step": 4790 + }, + { + "epoch": 0.77511556084859, + "grad_norm": 0.7930851578712463, + "learning_rate": 0.0001, + "loss": 73.8661, + "step": 4800 + }, + { + "epoch": 0.7767303849336913, + "grad_norm": 0.7549886107444763, + "learning_rate": 0.0001, + "loss": 75.042, + "step": 4810 + }, + { + "epoch": 0.7783452090187926, + "grad_norm": 0.9932214617729187, + "learning_rate": 0.0001, + "loss": 75.6619, + "step": 4820 + }, + { + "epoch": 0.7799600331038937, + "grad_norm": 0.7685471773147583, + "learning_rate": 0.0001, + "loss": 75.8404, + "step": 4830 + }, + { + "epoch": 0.781574857188995, + "grad_norm": 0.828737199306488, + "learning_rate": 0.0001, + "loss": 75.5839, + "step": 4840 + }, + { + "epoch": 0.7831896812740962, + "grad_norm": 0.8575420379638672, + "learning_rate": 0.0001, + "loss": 75.7889, + "step": 4850 + }, + { + "epoch": 0.7848045053591974, + "grad_norm": 0.7053834199905396, + "learning_rate": 0.0001, + "loss": 75.9372, + "step": 4860 + }, + { + "epoch": 0.7864193294442987, + "grad_norm": 0.7861505150794983, + "learning_rate": 0.0001, + "loss": 76.1532, + "step": 4870 + }, + { + "epoch": 0.7880341535293999, + "grad_norm": 0.8357386589050293, + "learning_rate": 0.0001, + "loss": 75.8536, + "step": 4880 + }, + { + "epoch": 0.7896489776145011, + "grad_norm": 3.448834180831909, + "learning_rate": 0.0001, + "loss": 74.1794, + "step": 4890 + }, + { + "epoch": 0.7912638016996023, + "grad_norm": 0.813522458076477, + "learning_rate": 0.0001, + "loss": 74.5202, + "step": 4900 + }, + { + "epoch": 0.7928786257847036, + "grad_norm": 0.840230405330658, + "learning_rate": 0.0001, + "loss": 74.6634, + "step": 4910 + }, + { + "epoch": 0.7944934498698049, + "grad_norm": 0.7855745553970337, + "learning_rate": 0.0001, + "loss": 73.5369, + "step": 4920 + }, + { + "epoch": 0.796108273954906, + "grad_norm": 0.7708212733268738, + "learning_rate": 0.0001, + "loss": 74.6225, + "step": 4930 + }, + { + "epoch": 0.7977230980400073, + "grad_norm": 0.7140830159187317, + "learning_rate": 0.0001, + "loss": 75.0585, + "step": 4940 + }, + { + "epoch": 0.7993379221251085, + "grad_norm": 0.7960936427116394, + "learning_rate": 0.0001, + "loss": 74.0574, + "step": 4950 + }, + { + "epoch": 0.8009527462102097, + "grad_norm": 0.7269165515899658, + "learning_rate": 0.0001, + "loss": 74.0379, + "step": 4960 + }, + { + "epoch": 0.802567570295311, + "grad_norm": 0.8505013585090637, + "learning_rate": 0.0001, + "loss": 74.7448, + "step": 4970 + }, + { + "epoch": 0.8041823943804122, + "grad_norm": 0.8935654759407043, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 4980 + }, + { + "epoch": 0.8057972184655134, + "grad_norm": 0.9447595477104187, + "learning_rate": 0.0001, + "loss": 75.9674, + "step": 4990 + }, + { + "epoch": 0.8074120425506146, + "grad_norm": 1.6739715337753296, + "learning_rate": 0.0001, + "loss": 73.9035, + "step": 5000 + }, + { + "epoch": 0.8090268666357159, + "grad_norm": 0.861902117729187, + "learning_rate": 0.0001, + "loss": 74.906, + "step": 5010 + }, + { + "epoch": 0.8106416907208172, + "grad_norm": 0.8328655362129211, + "learning_rate": 0.0001, + "loss": 74.1493, + "step": 5020 + }, + { + "epoch": 0.8122565148059183, + "grad_norm": 0.8193135261535645, + "learning_rate": 0.0001, + "loss": 74.6614, + "step": 5030 + }, + { + "epoch": 0.8138713388910196, + "grad_norm": 0.6954565048217773, + "learning_rate": 0.0001, + "loss": 73.7366, + "step": 5040 + }, + { + "epoch": 0.8154861629761208, + "grad_norm": 0.7381289005279541, + "learning_rate": 0.0001, + "loss": 73.5086, + "step": 5050 + }, + { + "epoch": 0.817100987061222, + "grad_norm": 1.090443730354309, + "learning_rate": 0.0001, + "loss": 75.0695, + "step": 5060 + }, + { + "epoch": 0.8187158111463233, + "grad_norm": 1.2544814348220825, + "learning_rate": 0.0001, + "loss": 72.89, + "step": 5070 + }, + { + "epoch": 0.8203306352314245, + "grad_norm": 0.632146418094635, + "learning_rate": 0.0001, + "loss": 74.8604, + "step": 5080 + }, + { + "epoch": 0.8219454593165257, + "grad_norm": 1.1038259267807007, + "learning_rate": 0.0001, + "loss": 74.7354, + "step": 5090 + }, + { + "epoch": 0.8235602834016269, + "grad_norm": 0.6886508464813232, + "learning_rate": 0.0001, + "loss": 75.4243, + "step": 5100 + }, + { + "epoch": 0.8251751074867282, + "grad_norm": 0.8367568850517273, + "learning_rate": 0.0001, + "loss": 75.2732, + "step": 5110 + }, + { + "epoch": 0.8267899315718293, + "grad_norm": 0.9302715063095093, + "learning_rate": 0.0001, + "loss": 75.7924, + "step": 5120 + }, + { + "epoch": 0.8284047556569306, + "grad_norm": 0.6871219873428345, + "learning_rate": 0.0001, + "loss": 74.2905, + "step": 5130 + }, + { + "epoch": 0.8300195797420319, + "grad_norm": 0.7319445013999939, + "learning_rate": 0.0001, + "loss": 73.7774, + "step": 5140 + }, + { + "epoch": 0.831634403827133, + "grad_norm": 0.7184522151947021, + "learning_rate": 0.0001, + "loss": 73.0554, + "step": 5150 + }, + { + "epoch": 0.8332492279122343, + "grad_norm": 0.8462343811988831, + "learning_rate": 0.0001, + "loss": 73.7615, + "step": 5160 + }, + { + "epoch": 0.8348640519973355, + "grad_norm": 0.9436241984367371, + "learning_rate": 0.0001, + "loss": 73.9464, + "step": 5170 + }, + { + "epoch": 0.8364788760824368, + "grad_norm": 0.9079630374908447, + "learning_rate": 0.0001, + "loss": 73.2727, + "step": 5180 + }, + { + "epoch": 0.838093700167538, + "grad_norm": 0.8280741572380066, + "learning_rate": 0.0001, + "loss": 73.9751, + "step": 5190 + }, + { + "epoch": 0.8397085242526392, + "grad_norm": 0.6475157141685486, + "learning_rate": 0.0001, + "loss": 74.7105, + "step": 5200 + }, + { + "epoch": 0.8413233483377405, + "grad_norm": 0.7090312838554382, + "learning_rate": 0.0001, + "loss": 73.6489, + "step": 5210 + }, + { + "epoch": 0.8429381724228416, + "grad_norm": 1.20967698097229, + "learning_rate": 0.0001, + "loss": 75.1335, + "step": 5220 + }, + { + "epoch": 0.8445529965079429, + "grad_norm": 2.053532600402832, + "learning_rate": 0.0001, + "loss": 75.3111, + "step": 5230 + }, + { + "epoch": 0.8461678205930442, + "grad_norm": 0.7563302516937256, + "learning_rate": 0.0001, + "loss": 72.9935, + "step": 5240 + }, + { + "epoch": 0.8477826446781453, + "grad_norm": 0.7801489233970642, + "learning_rate": 0.0001, + "loss": 74.7856, + "step": 5250 + }, + { + "epoch": 0.8493974687632466, + "grad_norm": 0.737870454788208, + "learning_rate": 0.0001, + "loss": 75.6281, + "step": 5260 + }, + { + "epoch": 0.8510122928483478, + "grad_norm": 0.704352617263794, + "learning_rate": 0.0001, + "loss": 75.4598, + "step": 5270 + }, + { + "epoch": 0.8526271169334491, + "grad_norm": 0.6452057361602783, + "learning_rate": 0.0001, + "loss": 74.0341, + "step": 5280 + }, + { + "epoch": 0.8542419410185503, + "grad_norm": 0.6863471865653992, + "learning_rate": 0.0001, + "loss": 74.3727, + "step": 5290 + }, + { + "epoch": 0.8558567651036515, + "grad_norm": 9.756170272827148, + "learning_rate": 0.0001, + "loss": 75.5231, + "step": 5300 + }, + { + "epoch": 0.8574715891887528, + "grad_norm": 0.8624544739723206, + "learning_rate": 0.0001, + "loss": 74.7821, + "step": 5310 + }, + { + "epoch": 0.8590864132738539, + "grad_norm": 0.6637155413627625, + "learning_rate": 0.0001, + "loss": 74.1153, + "step": 5320 + }, + { + "epoch": 0.8607012373589552, + "grad_norm": 1.0764445066452026, + "learning_rate": 0.0001, + "loss": 73.7208, + "step": 5330 + }, + { + "epoch": 0.8623160614440565, + "grad_norm": 0.8567569255828857, + "learning_rate": 0.0001, + "loss": 73.1399, + "step": 5340 + }, + { + "epoch": 0.8639308855291576, + "grad_norm": 0.7681576013565063, + "learning_rate": 0.0001, + "loss": 73.7577, + "step": 5350 + }, + { + "epoch": 0.8655457096142589, + "grad_norm": 0.9336384534835815, + "learning_rate": 0.0001, + "loss": 74.1022, + "step": 5360 + }, + { + "epoch": 0.8671605336993601, + "grad_norm": 0.8603724837303162, + "learning_rate": 0.0001, + "loss": 75.9579, + "step": 5370 + }, + { + "epoch": 0.8687753577844614, + "grad_norm": 0.77194744348526, + "learning_rate": 0.0001, + "loss": 72.4799, + "step": 5380 + }, + { + "epoch": 0.8703901818695626, + "grad_norm": 0.7480376958847046, + "learning_rate": 0.0001, + "loss": 76.1257, + "step": 5390 + }, + { + "epoch": 0.8720050059546638, + "grad_norm": 0.7841023206710815, + "learning_rate": 0.0001, + "loss": 73.1512, + "step": 5400 + }, + { + "epoch": 0.8736198300397651, + "grad_norm": 0.8741798996925354, + "learning_rate": 0.0001, + "loss": 74.2151, + "step": 5410 + }, + { + "epoch": 0.8752346541248662, + "grad_norm": 0.9762082099914551, + "learning_rate": 0.0001, + "loss": 76.7138, + "step": 5420 + }, + { + "epoch": 0.8768494782099675, + "grad_norm": 0.7670348286628723, + "learning_rate": 0.0001, + "loss": 72.9567, + "step": 5430 + }, + { + "epoch": 0.8784643022950688, + "grad_norm": 0.7449469566345215, + "learning_rate": 0.0001, + "loss": 74.8841, + "step": 5440 + }, + { + "epoch": 0.8800791263801699, + "grad_norm": 0.6984641551971436, + "learning_rate": 0.0001, + "loss": 73.8906, + "step": 5450 + }, + { + "epoch": 0.8816939504652712, + "grad_norm": 0.7854361534118652, + "learning_rate": 0.0001, + "loss": 74.626, + "step": 5460 + }, + { + "epoch": 0.8833087745503724, + "grad_norm": 0.9584240317344666, + "learning_rate": 0.0001, + "loss": 74.7048, + "step": 5470 + }, + { + "epoch": 0.8849235986354737, + "grad_norm": 0.6040284037590027, + "learning_rate": 0.0001, + "loss": 74.7194, + "step": 5480 + }, + { + "epoch": 0.8865384227205749, + "grad_norm": 0.7756215929985046, + "learning_rate": 0.0001, + "loss": 73.8178, + "step": 5490 + }, + { + "epoch": 0.8881532468056761, + "grad_norm": 0.9070586562156677, + "learning_rate": 0.0001, + "loss": 72.8309, + "step": 5500 + }, + { + "epoch": 0.8897680708907774, + "grad_norm": 0.8464986681938171, + "learning_rate": 0.0001, + "loss": 73.8847, + "step": 5510 + }, + { + "epoch": 0.8913828949758785, + "grad_norm": 0.7968605160713196, + "learning_rate": 0.0001, + "loss": 73.1691, + "step": 5520 + }, + { + "epoch": 0.8929977190609798, + "grad_norm": 0.9934583306312561, + "learning_rate": 0.0001, + "loss": 76.2662, + "step": 5530 + }, + { + "epoch": 0.8946125431460811, + "grad_norm": 0.7272152900695801, + "learning_rate": 0.0001, + "loss": 75.5295, + "step": 5540 + }, + { + "epoch": 0.8962273672311822, + "grad_norm": 0.857345461845398, + "learning_rate": 0.0001, + "loss": 75.3703, + "step": 5550 + }, + { + "epoch": 0.8978421913162835, + "grad_norm": 0.713918924331665, + "learning_rate": 0.0001, + "loss": 73.4563, + "step": 5560 + }, + { + "epoch": 0.8994570154013847, + "grad_norm": 0.759233832359314, + "learning_rate": 0.0001, + "loss": 73.2744, + "step": 5570 + }, + { + "epoch": 0.901071839486486, + "grad_norm": 0.7085754871368408, + "learning_rate": 0.0001, + "loss": 74.7792, + "step": 5580 + }, + { + "epoch": 0.9026866635715872, + "grad_norm": 0.7370456457138062, + "learning_rate": 0.0001, + "loss": 74.4216, + "step": 5590 + }, + { + "epoch": 0.9043014876566884, + "grad_norm": 0.779746949672699, + "learning_rate": 0.0001, + "loss": 75.9822, + "step": 5600 + }, + { + "epoch": 0.9059163117417897, + "grad_norm": 0.7915763854980469, + "learning_rate": 0.0001, + "loss": 74.3706, + "step": 5610 + }, + { + "epoch": 0.9075311358268908, + "grad_norm": 0.7886125445365906, + "learning_rate": 0.0001, + "loss": 72.9453, + "step": 5620 + }, + { + "epoch": 0.9091459599119921, + "grad_norm": 0.7804751992225647, + "learning_rate": 0.0001, + "loss": 73.514, + "step": 5630 + }, + { + "epoch": 0.9107607839970934, + "grad_norm": 0.7168806195259094, + "learning_rate": 0.0001, + "loss": 71.4159, + "step": 5640 + }, + { + "epoch": 0.9123756080821945, + "grad_norm": 0.7553626894950867, + "learning_rate": 0.0001, + "loss": 74.4353, + "step": 5650 + }, + { + "epoch": 0.9139904321672958, + "grad_norm": 0.6688916683197021, + "learning_rate": 0.0001, + "loss": 74.3786, + "step": 5660 + }, + { + "epoch": 0.915605256252397, + "grad_norm": 0.7425532341003418, + "learning_rate": 0.0001, + "loss": 73.3355, + "step": 5670 + }, + { + "epoch": 0.9172200803374982, + "grad_norm": 0.6473882794380188, + "learning_rate": 0.0001, + "loss": 74.2315, + "step": 5680 + }, + { + "epoch": 0.9188349044225994, + "grad_norm": 0.7222439050674438, + "learning_rate": 0.0001, + "loss": 74.2885, + "step": 5690 + }, + { + "epoch": 0.9204497285077007, + "grad_norm": 1.5716948509216309, + "learning_rate": 0.0001, + "loss": 74.6739, + "step": 5700 + }, + { + "epoch": 0.922064552592802, + "grad_norm": 0.8077759742736816, + "learning_rate": 0.0001, + "loss": 72.6617, + "step": 5710 + }, + { + "epoch": 0.9236793766779031, + "grad_norm": 0.6948631405830383, + "learning_rate": 0.0001, + "loss": 73.6247, + "step": 5720 + }, + { + "epoch": 0.9252942007630044, + "grad_norm": 0.8823757767677307, + "learning_rate": 0.0001, + "loss": 73.2536, + "step": 5730 + }, + { + "epoch": 0.9269090248481056, + "grad_norm": 0.7066764235496521, + "learning_rate": 0.0001, + "loss": 74.1748, + "step": 5740 + }, + { + "epoch": 0.9285238489332068, + "grad_norm": 0.6979793906211853, + "learning_rate": 0.0001, + "loss": 72.3189, + "step": 5750 + }, + { + "epoch": 0.9301386730183081, + "grad_norm": 0.6961462497711182, + "learning_rate": 0.0001, + "loss": 74.6196, + "step": 5760 + }, + { + "epoch": 0.9317534971034093, + "grad_norm": 0.7797656655311584, + "learning_rate": 0.0001, + "loss": 74.5472, + "step": 5770 + }, + { + "epoch": 0.9333683211885105, + "grad_norm": 1.0671838521957397, + "learning_rate": 0.0001, + "loss": 73.241, + "step": 5780 + }, + { + "epoch": 0.9349831452736117, + "grad_norm": 0.7566089034080505, + "learning_rate": 0.0001, + "loss": 73.6784, + "step": 5790 + }, + { + "epoch": 0.936597969358713, + "grad_norm": 0.8563273549079895, + "learning_rate": 0.0001, + "loss": 73.6871, + "step": 5800 + }, + { + "epoch": 0.9382127934438143, + "grad_norm": 0.7885987162590027, + "learning_rate": 0.0001, + "loss": 73.4447, + "step": 5810 + }, + { + "epoch": 0.9398276175289154, + "grad_norm": 0.8751112818717957, + "learning_rate": 0.0001, + "loss": 74.0531, + "step": 5820 + }, + { + "epoch": 0.9414424416140167, + "grad_norm": 0.6663336157798767, + "learning_rate": 0.0001, + "loss": 73.0557, + "step": 5830 + }, + { + "epoch": 0.9430572656991179, + "grad_norm": 0.7993099689483643, + "learning_rate": 0.0001, + "loss": 74.4467, + "step": 5840 + }, + { + "epoch": 0.9446720897842191, + "grad_norm": 0.6987188458442688, + "learning_rate": 0.0001, + "loss": 74.0327, + "step": 5850 + }, + { + "epoch": 0.9462869138693204, + "grad_norm": 0.8458583354949951, + "learning_rate": 0.0001, + "loss": 73.1757, + "step": 5860 + }, + { + "epoch": 0.9479017379544216, + "grad_norm": 0.6372436881065369, + "learning_rate": 0.0001, + "loss": 73.1067, + "step": 5870 + }, + { + "epoch": 0.9495165620395228, + "grad_norm": 0.7447753548622131, + "learning_rate": 0.0001, + "loss": 75.5461, + "step": 5880 + }, + { + "epoch": 0.951131386124624, + "grad_norm": 0.6633971929550171, + "learning_rate": 0.0001, + "loss": 72.9966, + "step": 5890 + }, + { + "epoch": 0.9527462102097253, + "grad_norm": 0.8576784729957581, + "learning_rate": 0.0001, + "loss": 71.5712, + "step": 5900 + }, + { + "epoch": 0.9543610342948265, + "grad_norm": 1.4696853160858154, + "learning_rate": 0.0001, + "loss": 74.7893, + "step": 5910 + }, + { + "epoch": 0.9559758583799277, + "grad_norm": 0.611210286617279, + "learning_rate": 0.0001, + "loss": 74.3426, + "step": 5920 + }, + { + "epoch": 0.957590682465029, + "grad_norm": 0.5947410464286804, + "learning_rate": 0.0001, + "loss": 71.2563, + "step": 5930 + }, + { + "epoch": 0.9592055065501301, + "grad_norm": 0.7169992923736572, + "learning_rate": 0.0001, + "loss": 71.6049, + "step": 5940 + }, + { + "epoch": 0.9608203306352314, + "grad_norm": 0.6773818731307983, + "learning_rate": 0.0001, + "loss": 75.0328, + "step": 5950 + }, + { + "epoch": 0.9624351547203327, + "grad_norm": 0.6714807748794556, + "learning_rate": 0.0001, + "loss": 71.5105, + "step": 5960 + }, + { + "epoch": 0.9640499788054339, + "grad_norm": 1.0064337253570557, + "learning_rate": 0.0001, + "loss": 75.8887, + "step": 5970 + }, + { + "epoch": 0.9656648028905351, + "grad_norm": 0.9802227020263672, + "learning_rate": 0.0001, + "loss": 73.5655, + "step": 5980 + }, + { + "epoch": 0.9672796269756363, + "grad_norm": 0.7097306251525879, + "learning_rate": 0.0001, + "loss": 73.298, + "step": 5990 + }, + { + "epoch": 0.9688944510607376, + "grad_norm": 0.6680361032485962, + "learning_rate": 0.0001, + "loss": 73.8873, + "step": 6000 + }, + { + "epoch": 0.9705092751458388, + "grad_norm": 1.1302506923675537, + "learning_rate": 0.0001, + "loss": 73.3328, + "step": 6010 + }, + { + "epoch": 0.97212409923094, + "grad_norm": 0.7163954377174377, + "learning_rate": 0.0001, + "loss": 70.5955, + "step": 6020 + }, + { + "epoch": 0.9737389233160413, + "grad_norm": 0.8182240128517151, + "learning_rate": 0.0001, + "loss": 72.6397, + "step": 6030 + }, + { + "epoch": 0.9753537474011424, + "grad_norm": 0.6752052903175354, + "learning_rate": 0.0001, + "loss": 73.1188, + "step": 6040 + }, + { + "epoch": 0.9769685714862437, + "grad_norm": 0.8829680681228638, + "learning_rate": 0.0001, + "loss": 75.7346, + "step": 6050 + }, + { + "epoch": 0.978583395571345, + "grad_norm": 0.9254767894744873, + "learning_rate": 0.0001, + "loss": 73.6221, + "step": 6060 + }, + { + "epoch": 0.9801982196564462, + "grad_norm": 0.7527495622634888, + "learning_rate": 0.0001, + "loss": 73.948, + "step": 6070 + }, + { + "epoch": 0.9818130437415474, + "grad_norm": 1.0135066509246826, + "learning_rate": 0.0001, + "loss": 72.8929, + "step": 6080 + }, + { + "epoch": 0.9834278678266486, + "grad_norm": 0.7093394994735718, + "learning_rate": 0.0001, + "loss": 73.5826, + "step": 6090 + }, + { + "epoch": 0.9850426919117499, + "grad_norm": 0.6651231646537781, + "learning_rate": 0.0001, + "loss": 73.167, + "step": 6100 + }, + { + "epoch": 0.9866575159968511, + "grad_norm": 0.720552921295166, + "learning_rate": 0.0001, + "loss": 74.8985, + "step": 6110 + }, + { + "epoch": 0.9882723400819523, + "grad_norm": 1.7821632623672485, + "learning_rate": 0.0001, + "loss": 74.1414, + "step": 6120 + }, + { + "epoch": 0.9898871641670536, + "grad_norm": 0.6405150890350342, + "learning_rate": 0.0001, + "loss": 73.5989, + "step": 6130 + }, + { + "epoch": 0.9915019882521547, + "grad_norm": 0.8207482695579529, + "learning_rate": 0.0001, + "loss": 73.0977, + "step": 6140 + }, + { + "epoch": 0.993116812337256, + "grad_norm": 0.7320757508277893, + "learning_rate": 0.0001, + "loss": 72.0241, + "step": 6150 + }, + { + "epoch": 0.9947316364223573, + "grad_norm": 0.9522950649261475, + "learning_rate": 0.0001, + "loss": 73.9831, + "step": 6160 + }, + { + "epoch": 0.9963464605074585, + "grad_norm": 0.7437159419059753, + "learning_rate": 0.0001, + "loss": 72.1054, + "step": 6170 + }, + { + "epoch": 0.9979612845925597, + "grad_norm": 0.7035623788833618, + "learning_rate": 0.0001, + "loss": 73.959, + "step": 6180 + }, + { + "epoch": 0.9995761086776609, + "grad_norm": 0.7559277415275574, + "learning_rate": 0.0001, + "loss": 71.4651, + "step": 6190 + }, + { + "epoch": 1.001130376859571, + "grad_norm": 0.852342426776886, + "learning_rate": 0.0001, + "loss": 71.8844, + "step": 6200 + }, + { + "epoch": 1.002745200944672, + "grad_norm": 0.8018911480903625, + "learning_rate": 0.0001, + "loss": 74.5093, + "step": 6210 + }, + { + "epoch": 1.0043600250297733, + "grad_norm": 0.6613730192184448, + "learning_rate": 0.0001, + "loss": 71.9953, + "step": 6220 + }, + { + "epoch": 1.0059748491148746, + "grad_norm": 0.9815977215766907, + "learning_rate": 0.0001, + "loss": 70.8632, + "step": 6230 + }, + { + "epoch": 1.0075896731999758, + "grad_norm": 0.781581699848175, + "learning_rate": 0.0001, + "loss": 72.3603, + "step": 6240 + }, + { + "epoch": 1.009204497285077, + "grad_norm": 0.8079054355621338, + "learning_rate": 0.0001, + "loss": 73.6153, + "step": 6250 + }, + { + "epoch": 1.0108193213701782, + "grad_norm": 0.8356053233146667, + "learning_rate": 0.0001, + "loss": 73.1562, + "step": 6260 + }, + { + "epoch": 1.0124341454552794, + "grad_norm": 0.7107778787612915, + "learning_rate": 0.0001, + "loss": 74.5276, + "step": 6270 + }, + { + "epoch": 1.0140489695403807, + "grad_norm": 0.8601720929145813, + "learning_rate": 0.0001, + "loss": 70.103, + "step": 6280 + }, + { + "epoch": 1.015663793625482, + "grad_norm": 0.675837516784668, + "learning_rate": 0.0001, + "loss": 72.6102, + "step": 6290 + }, + { + "epoch": 1.0172786177105833, + "grad_norm": 0.7160016298294067, + "learning_rate": 0.0001, + "loss": 72.2404, + "step": 6300 + }, + { + "epoch": 1.0188934417956843, + "grad_norm": 0.7286407351493835, + "learning_rate": 0.0001, + "loss": 72.7149, + "step": 6310 + }, + { + "epoch": 1.0205082658807856, + "grad_norm": 0.8183980584144592, + "learning_rate": 0.0001, + "loss": 73.5827, + "step": 6320 + }, + { + "epoch": 1.0221230899658869, + "grad_norm": 0.7300626039505005, + "learning_rate": 0.0001, + "loss": 72.3459, + "step": 6330 + }, + { + "epoch": 1.0237379140509881, + "grad_norm": 0.7222613096237183, + "learning_rate": 0.0001, + "loss": 71.881, + "step": 6340 + }, + { + "epoch": 1.0253527381360894, + "grad_norm": 0.7324674725532532, + "learning_rate": 0.0001, + "loss": 74.4781, + "step": 6350 + }, + { + "epoch": 1.0269675622211905, + "grad_norm": 0.9173555374145508, + "learning_rate": 0.0001, + "loss": 74.3222, + "step": 6360 + }, + { + "epoch": 1.0285823863062917, + "grad_norm": 0.8404964208602905, + "learning_rate": 0.0001, + "loss": 71.8959, + "step": 6370 + }, + { + "epoch": 1.030197210391393, + "grad_norm": 0.8735223412513733, + "learning_rate": 0.0001, + "loss": 72.7251, + "step": 6380 + }, + { + "epoch": 1.0318120344764943, + "grad_norm": 0.8467233777046204, + "learning_rate": 0.0001, + "loss": 72.5218, + "step": 6390 + }, + { + "epoch": 1.0334268585615956, + "grad_norm": 0.673565149307251, + "learning_rate": 0.0001, + "loss": 70.6392, + "step": 6400 + }, + { + "epoch": 1.0350416826466966, + "grad_norm": 0.6558571457862854, + "learning_rate": 0.0001, + "loss": 73.5305, + "step": 6410 + }, + { + "epoch": 1.0366565067317979, + "grad_norm": 0.7251600623130798, + "learning_rate": 0.0001, + "loss": 72.76, + "step": 6420 + }, + { + "epoch": 1.0382713308168992, + "grad_norm": 0.7458305358886719, + "learning_rate": 0.0001, + "loss": 73.6812, + "step": 6430 + }, + { + "epoch": 1.0398861549020004, + "grad_norm": 1.493802547454834, + "learning_rate": 0.0001, + "loss": 74.2455, + "step": 6440 + }, + { + "epoch": 1.0415009789871017, + "grad_norm": 0.894632875919342, + "learning_rate": 0.0001, + "loss": 74.2072, + "step": 6450 + }, + { + "epoch": 1.0431158030722028, + "grad_norm": 0.7142218947410583, + "learning_rate": 0.0001, + "loss": 71.4293, + "step": 6460 + }, + { + "epoch": 1.044730627157304, + "grad_norm": 0.702639639377594, + "learning_rate": 0.0001, + "loss": 71.5383, + "step": 6470 + }, + { + "epoch": 1.0463454512424053, + "grad_norm": 0.9378212094306946, + "learning_rate": 0.0001, + "loss": 72.8916, + "step": 6480 + }, + { + "epoch": 1.0479602753275066, + "grad_norm": 0.8298363089561462, + "learning_rate": 0.0001, + "loss": 72.3344, + "step": 6490 + }, + { + "epoch": 1.0495750994126078, + "grad_norm": 0.6617016792297363, + "learning_rate": 0.0001, + "loss": 71.0487, + "step": 6500 + }, + { + "epoch": 1.051189923497709, + "grad_norm": 0.638520359992981, + "learning_rate": 0.0001, + "loss": 72.2583, + "step": 6510 + }, + { + "epoch": 1.0528047475828102, + "grad_norm": 0.7101534008979797, + "learning_rate": 0.0001, + "loss": 70.8697, + "step": 6520 + }, + { + "epoch": 1.0544195716679114, + "grad_norm": 0.9425339698791504, + "learning_rate": 0.0001, + "loss": 71.0641, + "step": 6530 + }, + { + "epoch": 1.0560343957530127, + "grad_norm": 0.6572297811508179, + "learning_rate": 0.0001, + "loss": 72.6265, + "step": 6540 + }, + { + "epoch": 1.057649219838114, + "grad_norm": 0.745622456073761, + "learning_rate": 0.0001, + "loss": 73.0587, + "step": 6550 + }, + { + "epoch": 1.059264043923215, + "grad_norm": 0.7667096257209778, + "learning_rate": 0.0001, + "loss": 72.1779, + "step": 6560 + }, + { + "epoch": 1.0608788680083163, + "grad_norm": 0.7322410941123962, + "learning_rate": 0.0001, + "loss": 71.5503, + "step": 6570 + }, + { + "epoch": 1.0624936920934176, + "grad_norm": 0.5739616751670837, + "learning_rate": 0.0001, + "loss": 72.3846, + "step": 6580 + }, + { + "epoch": 1.0641085161785189, + "grad_norm": 0.7149553894996643, + "learning_rate": 0.0001, + "loss": 71.5459, + "step": 6590 + }, + { + "epoch": 1.0657233402636201, + "grad_norm": 0.6634597778320312, + "learning_rate": 0.0001, + "loss": 72.2528, + "step": 6600 + }, + { + "epoch": 1.0673381643487212, + "grad_norm": 0.7208803296089172, + "learning_rate": 0.0001, + "loss": 71.1115, + "step": 6610 + }, + { + "epoch": 1.0689529884338225, + "grad_norm": 0.9845868945121765, + "learning_rate": 0.0001, + "loss": 72.6418, + "step": 6620 + }, + { + "epoch": 1.0705678125189237, + "grad_norm": 0.9322167634963989, + "learning_rate": 0.0001, + "loss": 71.937, + "step": 6630 + }, + { + "epoch": 1.072182636604025, + "grad_norm": 0.6323093771934509, + "learning_rate": 0.0001, + "loss": 71.1537, + "step": 6640 + }, + { + "epoch": 1.0737974606891263, + "grad_norm": 0.6476802229881287, + "learning_rate": 0.0001, + "loss": 73.7811, + "step": 6650 + }, + { + "epoch": 1.0754122847742273, + "grad_norm": 0.7882822751998901, + "learning_rate": 0.0001, + "loss": 71.9548, + "step": 6660 + }, + { + "epoch": 1.0770271088593286, + "grad_norm": 0.7651168704032898, + "learning_rate": 0.0001, + "loss": 73.4066, + "step": 6670 + }, + { + "epoch": 1.07864193294443, + "grad_norm": 0.7536904215812683, + "learning_rate": 0.0001, + "loss": 72.1063, + "step": 6680 + }, + { + "epoch": 1.0802567570295312, + "grad_norm": 0.7908427119255066, + "learning_rate": 0.0001, + "loss": 72.6374, + "step": 6690 + }, + { + "epoch": 1.0818715811146324, + "grad_norm": 0.7796691060066223, + "learning_rate": 0.0001, + "loss": 72.2938, + "step": 6700 + }, + { + "epoch": 1.0834864051997335, + "grad_norm": 0.8512274622917175, + "learning_rate": 0.0001, + "loss": 71.556, + "step": 6710 + }, + { + "epoch": 1.0851012292848348, + "grad_norm": 0.7285851836204529, + "learning_rate": 0.0001, + "loss": 72.406, + "step": 6720 + }, + { + "epoch": 1.086716053369936, + "grad_norm": 0.6613008975982666, + "learning_rate": 0.0001, + "loss": 70.6509, + "step": 6730 + }, + { + "epoch": 1.0883308774550373, + "grad_norm": 0.7158271074295044, + "learning_rate": 0.0001, + "loss": 71.8434, + "step": 6740 + }, + { + "epoch": 1.0899457015401384, + "grad_norm": 0.8635967373847961, + "learning_rate": 0.0001, + "loss": 71.8279, + "step": 6750 + }, + { + "epoch": 1.0915605256252396, + "grad_norm": 0.7020159959793091, + "learning_rate": 0.0001, + "loss": 72.2417, + "step": 6760 + }, + { + "epoch": 1.093175349710341, + "grad_norm": 0.6389853954315186, + "learning_rate": 0.0001, + "loss": 72.1879, + "step": 6770 + }, + { + "epoch": 1.0947901737954422, + "grad_norm": 0.6460697650909424, + "learning_rate": 0.0001, + "loss": 72.3356, + "step": 6780 + }, + { + "epoch": 1.0964049978805435, + "grad_norm": 0.7152006030082703, + "learning_rate": 0.0001, + "loss": 71.3725, + "step": 6790 + }, + { + "epoch": 1.0980198219656445, + "grad_norm": 1.0050997734069824, + "learning_rate": 0.0001, + "loss": 72.5452, + "step": 6800 + }, + { + "epoch": 1.0996346460507458, + "grad_norm": 0.6967180371284485, + "learning_rate": 0.0001, + "loss": 72.0832, + "step": 6810 + }, + { + "epoch": 1.101249470135847, + "grad_norm": 0.7472837567329407, + "learning_rate": 0.0001, + "loss": 72.8122, + "step": 6820 + }, + { + "epoch": 1.1028642942209483, + "grad_norm": 0.8198382258415222, + "learning_rate": 0.0001, + "loss": 72.5116, + "step": 6830 + }, + { + "epoch": 1.1044791183060496, + "grad_norm": 0.8112207651138306, + "learning_rate": 0.0001, + "loss": 72.4218, + "step": 6840 + }, + { + "epoch": 1.1060939423911507, + "grad_norm": 0.8412114977836609, + "learning_rate": 0.0001, + "loss": 71.0198, + "step": 6850 + }, + { + "epoch": 1.107708766476252, + "grad_norm": 0.6730719804763794, + "learning_rate": 0.0001, + "loss": 71.2045, + "step": 6860 + }, + { + "epoch": 1.1093235905613532, + "grad_norm": 0.6895949244499207, + "learning_rate": 0.0001, + "loss": 72.3056, + "step": 6870 + }, + { + "epoch": 1.1109384146464545, + "grad_norm": 2.476609945297241, + "learning_rate": 0.0001, + "loss": 70.3865, + "step": 6880 + }, + { + "epoch": 1.1125532387315558, + "grad_norm": 0.7285001873970032, + "learning_rate": 0.0001, + "loss": 71.5761, + "step": 6890 + }, + { + "epoch": 1.1141680628166568, + "grad_norm": 0.7546679973602295, + "learning_rate": 0.0001, + "loss": 72.5979, + "step": 6900 + }, + { + "epoch": 1.115782886901758, + "grad_norm": 0.8974841237068176, + "learning_rate": 0.0001, + "loss": 74.5888, + "step": 6910 + }, + { + "epoch": 1.1173977109868594, + "grad_norm": 0.8785758018493652, + "learning_rate": 0.0001, + "loss": 71.662, + "step": 6920 + }, + { + "epoch": 1.1190125350719606, + "grad_norm": 0.6805328130722046, + "learning_rate": 0.0001, + "loss": 72.201, + "step": 6930 + }, + { + "epoch": 1.120627359157062, + "grad_norm": 0.8132315278053284, + "learning_rate": 0.0001, + "loss": 72.7473, + "step": 6940 + }, + { + "epoch": 1.122242183242163, + "grad_norm": 1.1625338792800903, + "learning_rate": 0.0001, + "loss": 70.5987, + "step": 6950 + }, + { + "epoch": 1.1238570073272642, + "grad_norm": 1.0984737873077393, + "learning_rate": 0.0001, + "loss": 72.7335, + "step": 6960 + }, + { + "epoch": 1.1254718314123655, + "grad_norm": 0.6798779368400574, + "learning_rate": 0.0001, + "loss": 72.1942, + "step": 6970 + }, + { + "epoch": 1.1270866554974668, + "grad_norm": 0.7546010613441467, + "learning_rate": 0.0001, + "loss": 70.8014, + "step": 6980 + }, + { + "epoch": 1.128701479582568, + "grad_norm": 1.0238053798675537, + "learning_rate": 0.0001, + "loss": 72.5617, + "step": 6990 + }, + { + "epoch": 1.130316303667669, + "grad_norm": 0.6535419821739197, + "learning_rate": 0.0001, + "loss": 70.2193, + "step": 7000 + }, + { + "epoch": 1.1319311277527704, + "grad_norm": 0.7354230284690857, + "learning_rate": 0.0001, + "loss": 72.0685, + "step": 7010 + }, + { + "epoch": 1.1335459518378717, + "grad_norm": 0.7848799824714661, + "learning_rate": 0.0001, + "loss": 72.2869, + "step": 7020 + }, + { + "epoch": 1.135160775922973, + "grad_norm": 0.6519262790679932, + "learning_rate": 0.0001, + "loss": 72.2023, + "step": 7030 + }, + { + "epoch": 1.1367756000080742, + "grad_norm": 0.8428511023521423, + "learning_rate": 0.0001, + "loss": 71.3396, + "step": 7040 + }, + { + "epoch": 1.1383904240931753, + "grad_norm": 0.8873165249824524, + "learning_rate": 0.0001, + "loss": 69.8284, + "step": 7050 + }, + { + "epoch": 1.1400052481782765, + "grad_norm": 0.9141374230384827, + "learning_rate": 0.0001, + "loss": 70.3666, + "step": 7060 + }, + { + "epoch": 1.1416200722633778, + "grad_norm": 0.6556552648544312, + "learning_rate": 0.0001, + "loss": 71.1316, + "step": 7070 + }, + { + "epoch": 1.143234896348479, + "grad_norm": 0.6736201047897339, + "learning_rate": 0.0001, + "loss": 70.2601, + "step": 7080 + }, + { + "epoch": 1.1448497204335804, + "grad_norm": 0.6965599656105042, + "learning_rate": 0.0001, + "loss": 71.9409, + "step": 7090 + }, + { + "epoch": 1.1464645445186814, + "grad_norm": 1.0600581169128418, + "learning_rate": 0.0001, + "loss": 72.7344, + "step": 7100 + }, + { + "epoch": 1.1480793686037827, + "grad_norm": 0.7501764297485352, + "learning_rate": 0.0001, + "loss": 72.3129, + "step": 7110 + }, + { + "epoch": 1.149694192688884, + "grad_norm": 0.7105675935745239, + "learning_rate": 0.0001, + "loss": 71.1356, + "step": 7120 + }, + { + "epoch": 1.1513090167739852, + "grad_norm": 0.7446026802062988, + "learning_rate": 0.0001, + "loss": 69.8208, + "step": 7130 + }, + { + "epoch": 1.1529238408590865, + "grad_norm": 0.6664865016937256, + "learning_rate": 0.0001, + "loss": 72.2466, + "step": 7140 + }, + { + "epoch": 1.1545386649441876, + "grad_norm": 1.7590374946594238, + "learning_rate": 0.0001, + "loss": 71.4987, + "step": 7150 + }, + { + "epoch": 1.1561534890292888, + "grad_norm": 0.7941271066665649, + "learning_rate": 0.0001, + "loss": 70.787, + "step": 7160 + }, + { + "epoch": 1.15776831311439, + "grad_norm": 0.7473573088645935, + "learning_rate": 0.0001, + "loss": 71.2516, + "step": 7170 + }, + { + "epoch": 1.1593831371994914, + "grad_norm": 0.7589637041091919, + "learning_rate": 0.0001, + "loss": 71.4781, + "step": 7180 + }, + { + "epoch": 1.1609979612845926, + "grad_norm": 0.8148510456085205, + "learning_rate": 0.0001, + "loss": 71.1491, + "step": 7190 + }, + { + "epoch": 1.1626127853696937, + "grad_norm": 0.6803365349769592, + "learning_rate": 0.0001, + "loss": 71.7937, + "step": 7200 + }, + { + "epoch": 1.164227609454795, + "grad_norm": 0.7314755320549011, + "learning_rate": 0.0001, + "loss": 70.6566, + "step": 7210 + }, + { + "epoch": 1.1658424335398963, + "grad_norm": 0.6494631171226501, + "learning_rate": 0.0001, + "loss": 70.5057, + "step": 7220 + }, + { + "epoch": 1.1674572576249975, + "grad_norm": 0.6076239943504333, + "learning_rate": 0.0001, + "loss": 72.2985, + "step": 7230 + }, + { + "epoch": 1.1690720817100988, + "grad_norm": 0.6650206446647644, + "learning_rate": 0.0001, + "loss": 71.0719, + "step": 7240 + }, + { + "epoch": 1.1706869057951999, + "grad_norm": 0.6926266551017761, + "learning_rate": 0.0001, + "loss": 70.3714, + "step": 7250 + }, + { + "epoch": 1.1723017298803011, + "grad_norm": 0.6534716486930847, + "learning_rate": 0.0001, + "loss": 71.3194, + "step": 7260 + }, + { + "epoch": 1.1739165539654024, + "grad_norm": 0.7194528579711914, + "learning_rate": 0.0001, + "loss": 72.3277, + "step": 7270 + }, + { + "epoch": 1.1755313780505037, + "grad_norm": 0.7496437430381775, + "learning_rate": 0.0001, + "loss": 71.1136, + "step": 7280 + }, + { + "epoch": 1.177146202135605, + "grad_norm": 0.690522313117981, + "learning_rate": 0.0001, + "loss": 70.065, + "step": 7290 + }, + { + "epoch": 1.178761026220706, + "grad_norm": 0.6966914534568787, + "learning_rate": 0.0001, + "loss": 73.1137, + "step": 7300 + }, + { + "epoch": 1.1803758503058073, + "grad_norm": 0.7304641604423523, + "learning_rate": 0.0001, + "loss": 71.2466, + "step": 7310 + }, + { + "epoch": 1.1819906743909085, + "grad_norm": 0.7564008235931396, + "learning_rate": 0.0001, + "loss": 71.4147, + "step": 7320 + }, + { + "epoch": 1.1836054984760098, + "grad_norm": 0.61886066198349, + "learning_rate": 0.0001, + "loss": 70.3297, + "step": 7330 + }, + { + "epoch": 1.185220322561111, + "grad_norm": 0.6281574964523315, + "learning_rate": 0.0001, + "loss": 70.9607, + "step": 7340 + }, + { + "epoch": 1.1868351466462121, + "grad_norm": 0.6583316326141357, + "learning_rate": 0.0001, + "loss": 70.3443, + "step": 7350 + }, + { + "epoch": 1.1884499707313134, + "grad_norm": 1.8159942626953125, + "learning_rate": 0.0001, + "loss": 71.4792, + "step": 7360 + }, + { + "epoch": 1.1900647948164147, + "grad_norm": 0.7253140807151794, + "learning_rate": 0.0001, + "loss": 72.2182, + "step": 7370 + }, + { + "epoch": 1.191679618901516, + "grad_norm": 0.9141016602516174, + "learning_rate": 0.0001, + "loss": 68.7443, + "step": 7380 + }, + { + "epoch": 1.193294442986617, + "grad_norm": 0.7392430901527405, + "learning_rate": 0.0001, + "loss": 69.1888, + "step": 7390 + }, + { + "epoch": 1.1949092670717183, + "grad_norm": 0.7713924646377563, + "learning_rate": 0.0001, + "loss": 69.1301, + "step": 7400 + }, + { + "epoch": 1.1965240911568196, + "grad_norm": 0.6000798344612122, + "learning_rate": 0.0001, + "loss": 71.2576, + "step": 7410 + }, + { + "epoch": 1.1981389152419208, + "grad_norm": 0.687111496925354, + "learning_rate": 0.0001, + "loss": 70.2919, + "step": 7420 + }, + { + "epoch": 1.1997537393270221, + "grad_norm": 1.1718226671218872, + "learning_rate": 0.0001, + "loss": 71.9498, + "step": 7430 + }, + { + "epoch": 1.2013685634121232, + "grad_norm": 0.7176038026809692, + "learning_rate": 0.0001, + "loss": 71.91, + "step": 7440 + }, + { + "epoch": 1.2029833874972244, + "grad_norm": 0.7705256938934326, + "learning_rate": 0.0001, + "loss": 70.1241, + "step": 7450 + }, + { + "epoch": 1.2045982115823257, + "grad_norm": 0.8004609942436218, + "learning_rate": 0.0001, + "loss": 69.5121, + "step": 7460 + }, + { + "epoch": 1.206213035667427, + "grad_norm": 0.6708329319953918, + "learning_rate": 0.0001, + "loss": 73.0173, + "step": 7470 + }, + { + "epoch": 1.2078278597525283, + "grad_norm": 0.8777983784675598, + "learning_rate": 0.0001, + "loss": 71.6843, + "step": 7480 + }, + { + "epoch": 1.2094426838376293, + "grad_norm": 0.7683864831924438, + "learning_rate": 0.0001, + "loss": 71.6914, + "step": 7490 + }, + { + "epoch": 1.2110575079227306, + "grad_norm": 0.8022210597991943, + "learning_rate": 0.0001, + "loss": 69.1724, + "step": 7500 + }, + { + "epoch": 1.2126723320078319, + "grad_norm": 0.7250640392303467, + "learning_rate": 0.0001, + "loss": 70.6568, + "step": 7510 + }, + { + "epoch": 1.2142871560929331, + "grad_norm": 0.5625214576721191, + "learning_rate": 0.0001, + "loss": 69.4849, + "step": 7520 + }, + { + "epoch": 1.2159019801780344, + "grad_norm": 0.7980468273162842, + "learning_rate": 0.0001, + "loss": 70.8164, + "step": 7530 + }, + { + "epoch": 1.2175168042631355, + "grad_norm": 0.6900951862335205, + "learning_rate": 0.0001, + "loss": 71.1719, + "step": 7540 + }, + { + "epoch": 1.2191316283482367, + "grad_norm": 0.6284971833229065, + "learning_rate": 0.0001, + "loss": 70.1751, + "step": 7550 + }, + { + "epoch": 1.220746452433338, + "grad_norm": 0.6641798615455627, + "learning_rate": 0.0001, + "loss": 71.9043, + "step": 7560 + }, + { + "epoch": 1.2223612765184393, + "grad_norm": 0.7439492344856262, + "learning_rate": 0.0001, + "loss": 70.7286, + "step": 7570 + }, + { + "epoch": 1.2239761006035406, + "grad_norm": 0.8018355369567871, + "learning_rate": 0.0001, + "loss": 71.638, + "step": 7580 + }, + { + "epoch": 1.2255909246886416, + "grad_norm": 0.7318522930145264, + "learning_rate": 0.0001, + "loss": 70.0867, + "step": 7590 + }, + { + "epoch": 1.2272057487737429, + "grad_norm": 0.6799285411834717, + "learning_rate": 0.0001, + "loss": 71.3427, + "step": 7600 + }, + { + "epoch": 1.2288205728588442, + "grad_norm": 0.6796864867210388, + "learning_rate": 0.0001, + "loss": 71.1698, + "step": 7610 + }, + { + "epoch": 1.2304353969439454, + "grad_norm": 0.7256141304969788, + "learning_rate": 0.0001, + "loss": 71.1051, + "step": 7620 + }, + { + "epoch": 1.2320502210290467, + "grad_norm": 0.8318199515342712, + "learning_rate": 0.0001, + "loss": 71.3179, + "step": 7630 + }, + { + "epoch": 1.2336650451141478, + "grad_norm": 0.6594699025154114, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 7640 + }, + { + "epoch": 1.235279869199249, + "grad_norm": 0.7900195717811584, + "learning_rate": 0.0001, + "loss": 72.0139, + "step": 7650 + }, + { + "epoch": 1.2368946932843503, + "grad_norm": 0.7845672965049744, + "learning_rate": 0.0001, + "loss": 70.808, + "step": 7660 + }, + { + "epoch": 1.2385095173694516, + "grad_norm": 0.6598045229911804, + "learning_rate": 0.0001, + "loss": 69.8953, + "step": 7670 + }, + { + "epoch": 1.2401243414545529, + "grad_norm": 1.3222978115081787, + "learning_rate": 0.0001, + "loss": 68.787, + "step": 7680 + }, + { + "epoch": 1.241739165539654, + "grad_norm": 0.6660508513450623, + "learning_rate": 0.0001, + "loss": 71.5831, + "step": 7690 + }, + { + "epoch": 1.2433539896247552, + "grad_norm": 0.6305698156356812, + "learning_rate": 0.0001, + "loss": 70.3232, + "step": 7700 + }, + { + "epoch": 1.2449688137098565, + "grad_norm": 1.2648323774337769, + "learning_rate": 0.0001, + "loss": 67.8901, + "step": 7710 + }, + { + "epoch": 1.2465836377949577, + "grad_norm": 0.7177619934082031, + "learning_rate": 0.0001, + "loss": 69.6582, + "step": 7720 + }, + { + "epoch": 1.248198461880059, + "grad_norm": 0.6716091632843018, + "learning_rate": 0.0001, + "loss": 70.2034, + "step": 7730 + }, + { + "epoch": 1.24981328596516, + "grad_norm": 0.6050158143043518, + "learning_rate": 0.0001, + "loss": 69.3942, + "step": 7740 + }, + { + "epoch": 1.2514281100502613, + "grad_norm": 0.7001014947891235, + "learning_rate": 0.0001, + "loss": 69.5971, + "step": 7750 + }, + { + "epoch": 1.2530429341353626, + "grad_norm": 0.7221280932426453, + "learning_rate": 0.0001, + "loss": 71.3651, + "step": 7760 + }, + { + "epoch": 1.2546577582204639, + "grad_norm": 0.9130340218544006, + "learning_rate": 0.0001, + "loss": 68.3038, + "step": 7770 + }, + { + "epoch": 1.2562725823055652, + "grad_norm": 0.7703199982643127, + "learning_rate": 0.0001, + "loss": 71.6127, + "step": 7780 + }, + { + "epoch": 1.2578874063906662, + "grad_norm": 0.6334695816040039, + "learning_rate": 0.0001, + "loss": 68.9074, + "step": 7790 + }, + { + "epoch": 1.2595022304757675, + "grad_norm": 0.7074242234230042, + "learning_rate": 0.0001, + "loss": 70.0569, + "step": 7800 + }, + { + "epoch": 1.2611170545608688, + "grad_norm": 0.6956025958061218, + "learning_rate": 0.0001, + "loss": 69.892, + "step": 7810 + }, + { + "epoch": 1.26273187864597, + "grad_norm": 0.7235708236694336, + "learning_rate": 0.0001, + "loss": 71.1777, + "step": 7820 + }, + { + "epoch": 1.2643467027310713, + "grad_norm": 0.6157007813453674, + "learning_rate": 0.0001, + "loss": 71.5091, + "step": 7830 + }, + { + "epoch": 1.2659615268161724, + "grad_norm": 0.5352533459663391, + "learning_rate": 0.0001, + "loss": 70.083, + "step": 7840 + }, + { + "epoch": 1.2675763509012736, + "grad_norm": 0.7171182632446289, + "learning_rate": 0.0001, + "loss": 68.4585, + "step": 7850 + }, + { + "epoch": 1.269191174986375, + "grad_norm": 0.7503113746643066, + "learning_rate": 0.0001, + "loss": 70.4206, + "step": 7860 + }, + { + "epoch": 1.2708059990714762, + "grad_norm": 0.6552693843841553, + "learning_rate": 0.0001, + "loss": 70.8455, + "step": 7870 + }, + { + "epoch": 1.2724208231565775, + "grad_norm": 0.7173179984092712, + "learning_rate": 0.0001, + "loss": 68.82, + "step": 7880 + }, + { + "epoch": 1.2740356472416785, + "grad_norm": 0.7352942824363708, + "learning_rate": 0.0001, + "loss": 70.4974, + "step": 7890 + }, + { + "epoch": 1.2756504713267798, + "grad_norm": 0.6120197176933289, + "learning_rate": 0.0001, + "loss": 72.1638, + "step": 7900 + }, + { + "epoch": 1.277265295411881, + "grad_norm": 0.6271925568580627, + "learning_rate": 0.0001, + "loss": 69.3267, + "step": 7910 + }, + { + "epoch": 1.2788801194969823, + "grad_norm": 0.752636194229126, + "learning_rate": 0.0001, + "loss": 70.6761, + "step": 7920 + }, + { + "epoch": 1.2804949435820836, + "grad_norm": 0.7307374477386475, + "learning_rate": 0.0001, + "loss": 70.2909, + "step": 7930 + }, + { + "epoch": 1.2821097676671847, + "grad_norm": 0.5977495908737183, + "learning_rate": 0.0001, + "loss": 72.2969, + "step": 7940 + }, + { + "epoch": 1.283724591752286, + "grad_norm": 0.7927457094192505, + "learning_rate": 0.0001, + "loss": 70.3784, + "step": 7950 + }, + { + "epoch": 1.2853394158373872, + "grad_norm": 0.7520087957382202, + "learning_rate": 0.0001, + "loss": 70.7122, + "step": 7960 + }, + { + "epoch": 1.2869542399224885, + "grad_norm": 0.6036237478256226, + "learning_rate": 0.0001, + "loss": 69.8718, + "step": 7970 + }, + { + "epoch": 1.2885690640075897, + "grad_norm": 0.6805875897407532, + "learning_rate": 0.0001, + "loss": 70.2391, + "step": 7980 + }, + { + "epoch": 1.2901838880926908, + "grad_norm": 0.774559497833252, + "learning_rate": 0.0001, + "loss": 71.1605, + "step": 7990 + }, + { + "epoch": 1.291798712177792, + "grad_norm": 0.6991063356399536, + "learning_rate": 0.0001, + "loss": 69.9779, + "step": 8000 + }, + { + "epoch": 1.2934135362628933, + "grad_norm": 0.5877949595451355, + "learning_rate": 0.0001, + "loss": 71.16, + "step": 8010 + }, + { + "epoch": 1.2950283603479946, + "grad_norm": 0.8840436339378357, + "learning_rate": 0.0001, + "loss": 70.9454, + "step": 8020 + }, + { + "epoch": 1.296643184433096, + "grad_norm": 0.7532126307487488, + "learning_rate": 0.0001, + "loss": 70.141, + "step": 8030 + }, + { + "epoch": 1.298258008518197, + "grad_norm": 0.6880044341087341, + "learning_rate": 0.0001, + "loss": 70.0746, + "step": 8040 + }, + { + "epoch": 1.2998728326032982, + "grad_norm": 0.6319366097450256, + "learning_rate": 0.0001, + "loss": 69.9396, + "step": 8050 + }, + { + "epoch": 1.3014876566883995, + "grad_norm": 0.6329445838928223, + "learning_rate": 0.0001, + "loss": 69.9043, + "step": 8060 + }, + { + "epoch": 1.3031024807735008, + "grad_norm": 0.8892332315444946, + "learning_rate": 0.0001, + "loss": 69.4187, + "step": 8070 + }, + { + "epoch": 1.304717304858602, + "grad_norm": 0.696500301361084, + "learning_rate": 0.0001, + "loss": 68.1885, + "step": 8080 + }, + { + "epoch": 1.306332128943703, + "grad_norm": 0.5933213233947754, + "learning_rate": 0.0001, + "loss": 68.8017, + "step": 8090 + }, + { + "epoch": 1.3079469530288044, + "grad_norm": 0.9274749159812927, + "learning_rate": 0.0001, + "loss": 71.5875, + "step": 8100 + }, + { + "epoch": 1.3095617771139056, + "grad_norm": 0.7540446519851685, + "learning_rate": 0.0001, + "loss": 71.1233, + "step": 8110 + }, + { + "epoch": 1.311176601199007, + "grad_norm": 0.6701658964157104, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8120 + }, + { + "epoch": 1.3127914252841082, + "grad_norm": 0.6096424460411072, + "learning_rate": 0.0001, + "loss": 69.9583, + "step": 8130 + }, + { + "epoch": 1.3144062493692092, + "grad_norm": 0.809524416923523, + "learning_rate": 0.0001, + "loss": 71.8149, + "step": 8140 + }, + { + "epoch": 1.3160210734543105, + "grad_norm": 0.7928242683410645, + "learning_rate": 0.0001, + "loss": 70.2084, + "step": 8150 + }, + { + "epoch": 1.3176358975394118, + "grad_norm": 0.7462850213050842, + "learning_rate": 0.0001, + "loss": 71.4789, + "step": 8160 + }, + { + "epoch": 1.319250721624513, + "grad_norm": 0.8166030049324036, + "learning_rate": 0.0001, + "loss": 69.3154, + "step": 8170 + }, + { + "epoch": 1.3208655457096143, + "grad_norm": 0.7002414464950562, + "learning_rate": 0.0001, + "loss": 70.8048, + "step": 8180 + }, + { + "epoch": 1.3224803697947154, + "grad_norm": 0.7653809189796448, + "learning_rate": 0.0001, + "loss": 70.0221, + "step": 8190 + }, + { + "epoch": 1.3240951938798167, + "grad_norm": 0.6777089238166809, + "learning_rate": 0.0001, + "loss": 70.614, + "step": 8200 + }, + { + "epoch": 1.325710017964918, + "grad_norm": 0.692739725112915, + "learning_rate": 0.0001, + "loss": 71.2311, + "step": 8210 + }, + { + "epoch": 1.3273248420500192, + "grad_norm": 0.7985519766807556, + "learning_rate": 0.0001, + "loss": 68.1982, + "step": 8220 + }, + { + "epoch": 1.3289396661351205, + "grad_norm": 0.7002689242362976, + "learning_rate": 0.0001, + "loss": 70.9181, + "step": 8230 + }, + { + "epoch": 1.3305544902202215, + "grad_norm": 0.7029191255569458, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 8240 + }, + { + "epoch": 1.3321693143053228, + "grad_norm": 0.7292912602424622, + "learning_rate": 0.0001, + "loss": 71.5386, + "step": 8250 + }, + { + "epoch": 1.333784138390424, + "grad_norm": 0.6141605973243713, + "learning_rate": 0.0001, + "loss": 72.3396, + "step": 8260 + }, + { + "epoch": 1.3353989624755254, + "grad_norm": 0.6486957669258118, + "learning_rate": 0.0001, + "loss": 69.7607, + "step": 8270 + }, + { + "epoch": 1.3370137865606266, + "grad_norm": 0.7813800573348999, + "learning_rate": 0.0001, + "loss": 68.8979, + "step": 8280 + }, + { + "epoch": 1.3386286106457277, + "grad_norm": 0.6805126070976257, + "learning_rate": 0.0001, + "loss": 70.2179, + "step": 8290 + }, + { + "epoch": 1.340243434730829, + "grad_norm": 0.7202772498130798, + "learning_rate": 0.0001, + "loss": 72.5008, + "step": 8300 + }, + { + "epoch": 1.3418582588159302, + "grad_norm": 0.6881810426712036, + "learning_rate": 0.0001, + "loss": 70.1987, + "step": 8310 + }, + { + "epoch": 1.3434730829010315, + "grad_norm": 0.6337536573410034, + "learning_rate": 0.0001, + "loss": 70.4073, + "step": 8320 + }, + { + "epoch": 1.3450879069861328, + "grad_norm": 0.7313241958618164, + "learning_rate": 0.0001, + "loss": 69.3598, + "step": 8330 + }, + { + "epoch": 1.3467027310712338, + "grad_norm": 0.635947585105896, + "learning_rate": 0.0001, + "loss": 69.5143, + "step": 8340 + }, + { + "epoch": 1.3483175551563351, + "grad_norm": 0.6219074726104736, + "learning_rate": 0.0001, + "loss": 69.9903, + "step": 8350 + }, + { + "epoch": 1.3499323792414364, + "grad_norm": 0.5928048491477966, + "learning_rate": 0.0001, + "loss": 70.4222, + "step": 8360 + }, + { + "epoch": 1.3515472033265377, + "grad_norm": 0.5927143096923828, + "learning_rate": 0.0001, + "loss": 69.8642, + "step": 8370 + }, + { + "epoch": 1.353162027411639, + "grad_norm": 0.7614993453025818, + "learning_rate": 0.0001, + "loss": 69.5787, + "step": 8380 + }, + { + "epoch": 1.35477685149674, + "grad_norm": 0.5488380193710327, + "learning_rate": 0.0001, + "loss": 69.0028, + "step": 8390 + }, + { + "epoch": 1.3563916755818413, + "grad_norm": 0.6219164133071899, + "learning_rate": 0.0001, + "loss": 69.4591, + "step": 8400 + }, + { + "epoch": 1.3580064996669425, + "grad_norm": 0.6940358877182007, + "learning_rate": 0.0001, + "loss": 71.3795, + "step": 8410 + }, + { + "epoch": 1.3596213237520438, + "grad_norm": 0.5919192433357239, + "learning_rate": 0.0001, + "loss": 71.2761, + "step": 8420 + }, + { + "epoch": 1.361236147837145, + "grad_norm": 1.0803378820419312, + "learning_rate": 0.0001, + "loss": 69.0046, + "step": 8430 + }, + { + "epoch": 1.3628509719222461, + "grad_norm": 0.7028629183769226, + "learning_rate": 0.0001, + "loss": 71.5996, + "step": 8440 + }, + { + "epoch": 1.3644657960073474, + "grad_norm": 0.6829854249954224, + "learning_rate": 0.0001, + "loss": 67.8831, + "step": 8450 + }, + { + "epoch": 1.3660806200924487, + "grad_norm": 0.6937337517738342, + "learning_rate": 0.0001, + "loss": 69.8321, + "step": 8460 + }, + { + "epoch": 1.36769544417755, + "grad_norm": 0.7725892066955566, + "learning_rate": 0.0001, + "loss": 71.0784, + "step": 8470 + }, + { + "epoch": 1.3693102682626512, + "grad_norm": 0.7171871066093445, + "learning_rate": 0.0001, + "loss": 69.0828, + "step": 8480 + }, + { + "epoch": 1.3709250923477523, + "grad_norm": 0.6605113744735718, + "learning_rate": 0.0001, + "loss": 70.9461, + "step": 8490 + }, + { + "epoch": 1.3725399164328536, + "grad_norm": 0.9005604982376099, + "learning_rate": 0.0001, + "loss": 71.474, + "step": 8500 + }, + { + "epoch": 1.3741547405179548, + "grad_norm": 0.7454419732093811, + "learning_rate": 0.0001, + "loss": 70.3113, + "step": 8510 + }, + { + "epoch": 1.375769564603056, + "grad_norm": 0.7574180960655212, + "learning_rate": 0.0001, + "loss": 70.9015, + "step": 8520 + }, + { + "epoch": 1.3773843886881574, + "grad_norm": 0.6525025367736816, + "learning_rate": 0.0001, + "loss": 72.3132, + "step": 8530 + }, + { + "epoch": 1.3789992127732584, + "grad_norm": 0.6548861861228943, + "learning_rate": 0.0001, + "loss": 68.6495, + "step": 8540 + }, + { + "epoch": 1.3806140368583597, + "grad_norm": 0.7881495952606201, + "learning_rate": 0.0001, + "loss": 70.1335, + "step": 8550 + }, + { + "epoch": 1.382228860943461, + "grad_norm": 0.619380533695221, + "learning_rate": 0.0001, + "loss": 71.7469, + "step": 8560 + }, + { + "epoch": 1.3838436850285623, + "grad_norm": 0.7637182474136353, + "learning_rate": 0.0001, + "loss": 69.4691, + "step": 8570 + }, + { + "epoch": 1.3854585091136635, + "grad_norm": 0.8711991906166077, + "learning_rate": 0.0001, + "loss": 70.8131, + "step": 8580 + }, + { + "epoch": 1.3870733331987646, + "grad_norm": 0.6550050973892212, + "learning_rate": 0.0001, + "loss": 70.0096, + "step": 8590 + }, + { + "epoch": 1.3886881572838659, + "grad_norm": 0.6310496926307678, + "learning_rate": 0.0001, + "loss": 70.4305, + "step": 8600 + }, + { + "epoch": 1.3903029813689671, + "grad_norm": 0.5780142545700073, + "learning_rate": 0.0001, + "loss": 68.9639, + "step": 8610 + }, + { + "epoch": 1.3919178054540684, + "grad_norm": 0.6526662111282349, + "learning_rate": 0.0001, + "loss": 69.1345, + "step": 8620 + }, + { + "epoch": 1.3935326295391697, + "grad_norm": 1.2579848766326904, + "learning_rate": 0.0001, + "loss": 71.6908, + "step": 8630 + }, + { + "epoch": 1.3951474536242707, + "grad_norm": 0.7702625393867493, + "learning_rate": 0.0001, + "loss": 69.1064, + "step": 8640 + }, + { + "epoch": 1.396762277709372, + "grad_norm": 0.7414085865020752, + "learning_rate": 0.0001, + "loss": 69.5106, + "step": 8650 + }, + { + "epoch": 1.3983771017944733, + "grad_norm": 0.666235089302063, + "learning_rate": 0.0001, + "loss": 67.938, + "step": 8660 + }, + { + "epoch": 1.3999919258795746, + "grad_norm": 0.7879519462585449, + "learning_rate": 0.0001, + "loss": 69.5422, + "step": 8670 + }, + { + "epoch": 1.4016067499646758, + "grad_norm": 0.6552250981330872, + "learning_rate": 0.0001, + "loss": 69.4161, + "step": 8680 + }, + { + "epoch": 1.4032215740497769, + "grad_norm": 0.7558363676071167, + "learning_rate": 0.0001, + "loss": 71.3052, + "step": 8690 + }, + { + "epoch": 1.4048363981348782, + "grad_norm": 0.677007794380188, + "learning_rate": 0.0001, + "loss": 70.0824, + "step": 8700 + }, + { + "epoch": 1.4064512222199794, + "grad_norm": 0.6416738629341125, + "learning_rate": 0.0001, + "loss": 68.3357, + "step": 8710 + }, + { + "epoch": 1.4080660463050807, + "grad_norm": 0.8067912459373474, + "learning_rate": 0.0001, + "loss": 69.8551, + "step": 8720 + }, + { + "epoch": 1.409680870390182, + "grad_norm": 0.6017255783081055, + "learning_rate": 0.0001, + "loss": 70.3936, + "step": 8730 + }, + { + "epoch": 1.411295694475283, + "grad_norm": 0.5473611354827881, + "learning_rate": 0.0001, + "loss": 69.1556, + "step": 8740 + }, + { + "epoch": 1.4129105185603843, + "grad_norm": 0.641984760761261, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 8750 + }, + { + "epoch": 1.4145253426454856, + "grad_norm": 0.7509932518005371, + "learning_rate": 0.0001, + "loss": 68.793, + "step": 8760 + }, + { + "epoch": 1.4161401667305868, + "grad_norm": 0.615459144115448, + "learning_rate": 0.0001, + "loss": 69.0787, + "step": 8770 + }, + { + "epoch": 1.4177549908156881, + "grad_norm": 0.625276505947113, + "learning_rate": 0.0001, + "loss": 68.5643, + "step": 8780 + }, + { + "epoch": 1.4193698149007892, + "grad_norm": 0.7294536828994751, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 8790 + }, + { + "epoch": 1.4209846389858904, + "grad_norm": 0.8188890814781189, + "learning_rate": 0.0001, + "loss": 70.7486, + "step": 8800 + }, + { + "epoch": 1.4225994630709917, + "grad_norm": 0.8662137389183044, + "learning_rate": 0.0001, + "loss": 69.0625, + "step": 8810 + }, + { + "epoch": 1.424214287156093, + "grad_norm": 0.7548071146011353, + "learning_rate": 0.0001, + "loss": 68.9828, + "step": 8820 + }, + { + "epoch": 1.4258291112411943, + "grad_norm": 0.7295185327529907, + "learning_rate": 0.0001, + "loss": 69.0606, + "step": 8830 + }, + { + "epoch": 1.4274439353262953, + "grad_norm": 0.7937467098236084, + "learning_rate": 0.0001, + "loss": 69.0135, + "step": 8840 + }, + { + "epoch": 1.4290587594113966, + "grad_norm": 0.5957842469215393, + "learning_rate": 0.0001, + "loss": 68.3992, + "step": 8850 + }, + { + "epoch": 1.4306735834964979, + "grad_norm": 0.786580502986908, + "learning_rate": 0.0001, + "loss": 69.4485, + "step": 8860 + }, + { + "epoch": 1.4322884075815991, + "grad_norm": 0.8539725542068481, + "learning_rate": 0.0001, + "loss": 67.1037, + "step": 8870 + }, + { + "epoch": 1.4339032316667004, + "grad_norm": 0.684476375579834, + "learning_rate": 0.0001, + "loss": 68.9814, + "step": 8880 + }, + { + "epoch": 1.4355180557518015, + "grad_norm": 0.6335768103599548, + "learning_rate": 0.0001, + "loss": 69.9716, + "step": 8890 + }, + { + "epoch": 1.4371328798369027, + "grad_norm": 0.9881808757781982, + "learning_rate": 0.0001, + "loss": 68.0338, + "step": 8900 + }, + { + "epoch": 1.438747703922004, + "grad_norm": 0.783983588218689, + "learning_rate": 0.0001, + "loss": 70.124, + "step": 8910 + }, + { + "epoch": 1.4403625280071053, + "grad_norm": 0.817820131778717, + "learning_rate": 0.0001, + "loss": 67.933, + "step": 8920 + }, + { + "epoch": 1.4419773520922066, + "grad_norm": 0.6900199055671692, + "learning_rate": 0.0001, + "loss": 69.8871, + "step": 8930 + }, + { + "epoch": 1.4435921761773076, + "grad_norm": 0.6702716946601868, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8940 + }, + { + "epoch": 1.445207000262409, + "grad_norm": 0.6699396967887878, + "learning_rate": 0.0001, + "loss": 67.8654, + "step": 8950 + }, + { + "epoch": 1.4468218243475102, + "grad_norm": 0.6388489603996277, + "learning_rate": 0.0001, + "loss": 69.5605, + "step": 8960 + }, + { + "epoch": 1.4484366484326114, + "grad_norm": 0.735910952091217, + "learning_rate": 0.0001, + "loss": 66.7865, + "step": 8970 + }, + { + "epoch": 1.4500514725177127, + "grad_norm": 0.6826426386833191, + "learning_rate": 0.0001, + "loss": 68.2896, + "step": 8980 + }, + { + "epoch": 1.4516662966028138, + "grad_norm": 0.8190213441848755, + "learning_rate": 0.0001, + "loss": 70.3524, + "step": 8990 + }, + { + "epoch": 1.453281120687915, + "grad_norm": 0.6774603724479675, + "learning_rate": 0.0001, + "loss": 68.9084, + "step": 9000 + }, + { + "epoch": 1.4548959447730163, + "grad_norm": 0.8953077793121338, + "learning_rate": 0.0001, + "loss": 69.228, + "step": 9010 + }, + { + "epoch": 1.4565107688581176, + "grad_norm": 1.8509501218795776, + "learning_rate": 0.0001, + "loss": 68.0847, + "step": 9020 + }, + { + "epoch": 1.4581255929432189, + "grad_norm": 0.671280026435852, + "learning_rate": 0.0001, + "loss": 70.0691, + "step": 9030 + }, + { + "epoch": 1.45974041702832, + "grad_norm": 0.7242400646209717, + "learning_rate": 0.0001, + "loss": 68.8001, + "step": 9040 + }, + { + "epoch": 1.4613552411134212, + "grad_norm": 0.6609033942222595, + "learning_rate": 0.0001, + "loss": 69.0398, + "step": 9050 + }, + { + "epoch": 1.4629700651985225, + "grad_norm": 0.6850711703300476, + "learning_rate": 0.0001, + "loss": 70.8849, + "step": 9060 + }, + { + "epoch": 1.4645848892836237, + "grad_norm": 0.7409449219703674, + "learning_rate": 0.0001, + "loss": 68.0411, + "step": 9070 + }, + { + "epoch": 1.466199713368725, + "grad_norm": 0.6832754611968994, + "learning_rate": 0.0001, + "loss": 69.1219, + "step": 9080 + }, + { + "epoch": 1.467814537453826, + "grad_norm": 0.6671826243400574, + "learning_rate": 0.0001, + "loss": 68.1225, + "step": 9090 + }, + { + "epoch": 1.4694293615389273, + "grad_norm": 0.6758706569671631, + "learning_rate": 0.0001, + "loss": 68.6981, + "step": 9100 + }, + { + "epoch": 1.4710441856240286, + "grad_norm": 0.7885150909423828, + "learning_rate": 0.0001, + "loss": 69.29, + "step": 9110 + }, + { + "epoch": 1.4726590097091299, + "grad_norm": 0.7636108994483948, + "learning_rate": 0.0001, + "loss": 67.3151, + "step": 9120 + }, + { + "epoch": 1.4742738337942312, + "grad_norm": 0.6518180966377258, + "learning_rate": 0.0001, + "loss": 68.6208, + "step": 9130 + }, + { + "epoch": 1.4758886578793322, + "grad_norm": 0.6715161204338074, + "learning_rate": 0.0001, + "loss": 70.5106, + "step": 9140 + }, + { + "epoch": 1.4775034819644335, + "grad_norm": 0.6342995762825012, + "learning_rate": 0.0001, + "loss": 70.3067, + "step": 9150 + }, + { + "epoch": 1.4791183060495348, + "grad_norm": 0.6773675084114075, + "learning_rate": 0.0001, + "loss": 69.2888, + "step": 9160 + }, + { + "epoch": 1.480733130134636, + "grad_norm": 0.6887578368186951, + "learning_rate": 0.0001, + "loss": 69.2788, + "step": 9170 + }, + { + "epoch": 1.4823479542197373, + "grad_norm": 0.6793885231018066, + "learning_rate": 0.0001, + "loss": 68.3523, + "step": 9180 + }, + { + "epoch": 1.4839627783048384, + "grad_norm": 0.7483563423156738, + "learning_rate": 0.0001, + "loss": 67.9991, + "step": 9190 + }, + { + "epoch": 1.4855776023899396, + "grad_norm": 0.7007381916046143, + "learning_rate": 0.0001, + "loss": 67.5335, + "step": 9200 + }, + { + "epoch": 1.487192426475041, + "grad_norm": 0.7565139532089233, + "learning_rate": 0.0001, + "loss": 69.1093, + "step": 9210 + }, + { + "epoch": 1.4888072505601422, + "grad_norm": 1.0212360620498657, + "learning_rate": 0.0001, + "loss": 68.5061, + "step": 9220 + }, + { + "epoch": 1.4904220746452435, + "grad_norm": 0.9198092818260193, + "learning_rate": 0.0001, + "loss": 67.9735, + "step": 9230 + }, + { + "epoch": 1.4920368987303445, + "grad_norm": 0.6228001713752747, + "learning_rate": 0.0001, + "loss": 69.2231, + "step": 9240 + }, + { + "epoch": 1.4936517228154458, + "grad_norm": 0.6618618965148926, + "learning_rate": 0.0001, + "loss": 68.0434, + "step": 9250 + }, + { + "epoch": 1.495266546900547, + "grad_norm": 0.6241064071655273, + "learning_rate": 0.0001, + "loss": 68.4004, + "step": 9260 + }, + { + "epoch": 1.4968813709856483, + "grad_norm": 0.7336916327476501, + "learning_rate": 0.0001, + "loss": 69.0389, + "step": 9270 + }, + { + "epoch": 1.4984961950707496, + "grad_norm": 0.718125581741333, + "learning_rate": 0.0001, + "loss": 68.4923, + "step": 9280 + }, + { + "epoch": 1.5001110191558507, + "grad_norm": 0.6694948673248291, + "learning_rate": 0.0001, + "loss": 68.1006, + "step": 9290 + }, + { + "epoch": 1.501725843240952, + "grad_norm": 0.5815121531486511, + "learning_rate": 0.0001, + "loss": 68.5943, + "step": 9300 + }, + { + "epoch": 1.5033406673260532, + "grad_norm": 1.4076049327850342, + "learning_rate": 0.0001, + "loss": 68.1251, + "step": 9310 + }, + { + "epoch": 1.5049554914111543, + "grad_norm": 0.7294307947158813, + "learning_rate": 0.0001, + "loss": 68.0539, + "step": 9320 + }, + { + "epoch": 1.5065703154962558, + "grad_norm": 0.627821683883667, + "learning_rate": 0.0001, + "loss": 68.6859, + "step": 9330 + }, + { + "epoch": 1.5081851395813568, + "grad_norm": 0.6683267951011658, + "learning_rate": 0.0001, + "loss": 68.2727, + "step": 9340 + }, + { + "epoch": 1.509799963666458, + "grad_norm": 0.7070158123970032, + "learning_rate": 0.0001, + "loss": 69.1171, + "step": 9350 + }, + { + "epoch": 1.5114147877515594, + "grad_norm": 0.6324070692062378, + "learning_rate": 0.0001, + "loss": 69.7498, + "step": 9360 + }, + { + "epoch": 1.5130296118366604, + "grad_norm": 0.9814477562904358, + "learning_rate": 0.0001, + "loss": 69.1426, + "step": 9370 + }, + { + "epoch": 1.514644435921762, + "grad_norm": 0.5883511304855347, + "learning_rate": 0.0001, + "loss": 68.5363, + "step": 9380 + }, + { + "epoch": 1.516259260006863, + "grad_norm": 0.8325281739234924, + "learning_rate": 0.0001, + "loss": 67.5951, + "step": 9390 + }, + { + "epoch": 1.5178740840919642, + "grad_norm": 0.7456872463226318, + "learning_rate": 0.0001, + "loss": 68.1876, + "step": 9400 + }, + { + "epoch": 1.5194889081770655, + "grad_norm": 0.7692755460739136, + "learning_rate": 0.0001, + "loss": 69.1677, + "step": 9410 + }, + { + "epoch": 1.5211037322621666, + "grad_norm": 0.7346193194389343, + "learning_rate": 0.0001, + "loss": 69.8608, + "step": 9420 + }, + { + "epoch": 1.522718556347268, + "grad_norm": 0.7658166289329529, + "learning_rate": 0.0001, + "loss": 68.8584, + "step": 9430 + }, + { + "epoch": 1.524333380432369, + "grad_norm": 0.7892371416091919, + "learning_rate": 0.0001, + "loss": 68.3882, + "step": 9440 + }, + { + "epoch": 1.5259482045174704, + "grad_norm": 0.5943968296051025, + "learning_rate": 0.0001, + "loss": 68.7935, + "step": 9450 + }, + { + "epoch": 1.5275630286025716, + "grad_norm": 0.6611922383308411, + "learning_rate": 0.0001, + "loss": 69.0865, + "step": 9460 + }, + { + "epoch": 1.5291778526876727, + "grad_norm": 0.6697778105735779, + "learning_rate": 0.0001, + "loss": 68.6131, + "step": 9470 + }, + { + "epoch": 1.5307926767727742, + "grad_norm": 0.862293541431427, + "learning_rate": 0.0001, + "loss": 71.0167, + "step": 9480 + }, + { + "epoch": 1.5324075008578752, + "grad_norm": 0.7098677754402161, + "learning_rate": 0.0001, + "loss": 67.7216, + "step": 9490 + }, + { + "epoch": 1.5340223249429765, + "grad_norm": 0.6578338742256165, + "learning_rate": 0.0001, + "loss": 69.872, + "step": 9500 + }, + { + "epoch": 1.5356371490280778, + "grad_norm": 0.7272375822067261, + "learning_rate": 0.0001, + "loss": 70.0971, + "step": 9510 + }, + { + "epoch": 1.5372519731131788, + "grad_norm": 0.6497064828872681, + "learning_rate": 0.0001, + "loss": 70.5921, + "step": 9520 + }, + { + "epoch": 1.5388667971982803, + "grad_norm": 0.6691288352012634, + "learning_rate": 0.0001, + "loss": 68.5213, + "step": 9530 + }, + { + "epoch": 1.5404816212833814, + "grad_norm": 0.9180259108543396, + "learning_rate": 0.0001, + "loss": 67.0699, + "step": 9540 + }, + { + "epoch": 1.5420964453684827, + "grad_norm": 0.7062082886695862, + "learning_rate": 0.0001, + "loss": 69.3965, + "step": 9550 + }, + { + "epoch": 1.543711269453584, + "grad_norm": 0.711613655090332, + "learning_rate": 0.0001, + "loss": 67.8276, + "step": 9560 + }, + { + "epoch": 1.545326093538685, + "grad_norm": 0.670032799243927, + "learning_rate": 0.0001, + "loss": 68.1687, + "step": 9570 + }, + { + "epoch": 1.5469409176237865, + "grad_norm": 0.7531955242156982, + "learning_rate": 0.0001, + "loss": 69.2144, + "step": 9580 + }, + { + "epoch": 1.5485557417088875, + "grad_norm": 0.6759967803955078, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 9590 + }, + { + "epoch": 1.5501705657939888, + "grad_norm": 0.6134868860244751, + "learning_rate": 0.0001, + "loss": 67.4348, + "step": 9600 + }, + { + "epoch": 1.55178538987909, + "grad_norm": 0.638785719871521, + "learning_rate": 0.0001, + "loss": 68.4635, + "step": 9610 + }, + { + "epoch": 1.5534002139641911, + "grad_norm": 0.7901169657707214, + "learning_rate": 0.0001, + "loss": 68.1079, + "step": 9620 + }, + { + "epoch": 1.5550150380492926, + "grad_norm": 0.6970444917678833, + "learning_rate": 0.0001, + "loss": 69.3883, + "step": 9630 + }, + { + "epoch": 1.5566298621343937, + "grad_norm": 0.6954355239868164, + "learning_rate": 0.0001, + "loss": 69.3841, + "step": 9640 + }, + { + "epoch": 1.558244686219495, + "grad_norm": 0.7097641825675964, + "learning_rate": 0.0001, + "loss": 68.3417, + "step": 9650 + }, + { + "epoch": 1.5598595103045962, + "grad_norm": 0.7082354426383972, + "learning_rate": 0.0001, + "loss": 69.9314, + "step": 9660 + }, + { + "epoch": 1.5614743343896973, + "grad_norm": 0.6570103764533997, + "learning_rate": 0.0001, + "loss": 69.7219, + "step": 9670 + }, + { + "epoch": 1.5630891584747988, + "grad_norm": 0.5705522894859314, + "learning_rate": 0.0001, + "loss": 67.5177, + "step": 9680 + }, + { + "epoch": 1.5647039825598998, + "grad_norm": 0.6992695331573486, + "learning_rate": 0.0001, + "loss": 67.6538, + "step": 9690 + }, + { + "epoch": 1.5663188066450011, + "grad_norm": 0.7523599863052368, + "learning_rate": 0.0001, + "loss": 66.869, + "step": 9700 + }, + { + "epoch": 1.5679336307301024, + "grad_norm": 0.6267555356025696, + "learning_rate": 0.0001, + "loss": 66.6805, + "step": 9710 + }, + { + "epoch": 1.5695484548152034, + "grad_norm": 0.6138116717338562, + "learning_rate": 0.0001, + "loss": 71.3217, + "step": 9720 + }, + { + "epoch": 1.571163278900305, + "grad_norm": 0.7408527731895447, + "learning_rate": 0.0001, + "loss": 69.2759, + "step": 9730 + }, + { + "epoch": 1.572778102985406, + "grad_norm": 0.813761830329895, + "learning_rate": 0.0001, + "loss": 70.4959, + "step": 9740 + }, + { + "epoch": 1.5743929270705073, + "grad_norm": 1.0363599061965942, + "learning_rate": 0.0001, + "loss": 68.1996, + "step": 9750 + }, + { + "epoch": 1.5760077511556085, + "grad_norm": 0.7773903012275696, + "learning_rate": 0.0001, + "loss": 67.9354, + "step": 9760 + }, + { + "epoch": 1.5776225752407096, + "grad_norm": 0.6465738415718079, + "learning_rate": 0.0001, + "loss": 69.5247, + "step": 9770 + }, + { + "epoch": 1.579237399325811, + "grad_norm": 0.8433385491371155, + "learning_rate": 0.0001, + "loss": 66.9448, + "step": 9780 + }, + { + "epoch": 1.5808522234109121, + "grad_norm": 0.8651545643806458, + "learning_rate": 0.0001, + "loss": 68.6994, + "step": 9790 + }, + { + "epoch": 1.5824670474960134, + "grad_norm": 0.6940448880195618, + "learning_rate": 0.0001, + "loss": 67.622, + "step": 9800 + }, + { + "epoch": 1.5840818715811147, + "grad_norm": 0.622756838798523, + "learning_rate": 0.0001, + "loss": 69.0994, + "step": 9810 + }, + { + "epoch": 1.5856966956662157, + "grad_norm": 0.6119604110717773, + "learning_rate": 0.0001, + "loss": 67.7682, + "step": 9820 + }, + { + "epoch": 1.5873115197513172, + "grad_norm": 0.5729681849479675, + "learning_rate": 0.0001, + "loss": 67.3113, + "step": 9830 + }, + { + "epoch": 1.5889263438364183, + "grad_norm": 0.7143844366073608, + "learning_rate": 0.0001, + "loss": 68.8128, + "step": 9840 + }, + { + "epoch": 1.5905411679215196, + "grad_norm": 0.9814873337745667, + "learning_rate": 0.0001, + "loss": 69.3099, + "step": 9850 + }, + { + "epoch": 1.5921559920066208, + "grad_norm": 0.8044945001602173, + "learning_rate": 0.0001, + "loss": 69.3939, + "step": 9860 + }, + { + "epoch": 1.5937708160917219, + "grad_norm": 0.8494839668273926, + "learning_rate": 0.0001, + "loss": 69.3889, + "step": 9870 + }, + { + "epoch": 1.5953856401768234, + "grad_norm": 0.6468870639801025, + "learning_rate": 0.0001, + "loss": 67.2047, + "step": 9880 + }, + { + "epoch": 1.5970004642619244, + "grad_norm": 0.6270474791526794, + "learning_rate": 0.0001, + "loss": 69.6813, + "step": 9890 + }, + { + "epoch": 1.5986152883470257, + "grad_norm": 0.7102645039558411, + "learning_rate": 0.0001, + "loss": 69.1546, + "step": 9900 + }, + { + "epoch": 1.600230112432127, + "grad_norm": 0.5929123759269714, + "learning_rate": 0.0001, + "loss": 67.2187, + "step": 9910 + }, + { + "epoch": 1.601844936517228, + "grad_norm": 0.6466200947761536, + "learning_rate": 0.0001, + "loss": 68.6876, + "step": 9920 + }, + { + "epoch": 1.6034597606023295, + "grad_norm": 0.6441522836685181, + "learning_rate": 0.0001, + "loss": 68.7955, + "step": 9930 + }, + { + "epoch": 1.6050745846874306, + "grad_norm": 0.7264062166213989, + "learning_rate": 0.0001, + "loss": 67.8876, + "step": 9940 + }, + { + "epoch": 1.6066894087725319, + "grad_norm": 0.6973199248313904, + "learning_rate": 0.0001, + "loss": 69.0904, + "step": 9950 + }, + { + "epoch": 1.6083042328576331, + "grad_norm": 0.582053005695343, + "learning_rate": 0.0001, + "loss": 69.3876, + "step": 9960 + }, + { + "epoch": 1.6099190569427342, + "grad_norm": 0.6431038975715637, + "learning_rate": 0.0001, + "loss": 67.4232, + "step": 9970 + }, + { + "epoch": 1.6115338810278357, + "grad_norm": 0.6313247084617615, + "learning_rate": 0.0001, + "loss": 68.3759, + "step": 9980 + }, + { + "epoch": 1.6131487051129367, + "grad_norm": 1.2003700733184814, + "learning_rate": 0.0001, + "loss": 68.3413, + "step": 9990 + }, + { + "epoch": 1.614763529198038, + "grad_norm": 1.0262068510055542, + "learning_rate": 0.0001, + "loss": 66.7308, + "step": 10000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2244552702599627e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6a151f546555c78ebc502dd7b703b665f0672d6f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35235df8408202e91954a585851d27187bafaf4e1d75c4e63c51fbf721136ff5 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-10000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/latest b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/latest new file mode 100644 index 0000000000000000000000000000000000000000..b8f4d8b87537b41ea73a42841f5fb1704b16b234 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/latest @@ -0,0 +1 @@ +global_step12000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5c158afcc5912e147b9c89eed9a74ba48b227bbd --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:032c32db2c33d9db7744595d872526eee28ded81b9e37a2dfb872e127ca7548d +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..c918e00a63c6b2e54ab045fe75c8862554479196 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed9bee09c1444af2a394097d24b5dd9578686182d2313c9ad17624cc6a763435 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..12b99ef274c8bbf4c897866bd141e0b7193a5dfd --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc27e3e547602d6bf66f9e8f6c0671aa14a1696071fd9a48468194ac59a00db5 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..3278683430910df2fd0472b8dc321eed3515e325 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:396961ffa1ee8723a0cb6a98874e1f4f7709374a7c25e695133b3273b157abc5 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..9433be16f8cb19e29e17764bcc71b37839a8f7b0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f4a9528f6c0bf5c218eafb99f97ad103f8b2c77404695027251d13cedc5b077 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..29f56058268fbcea6c845cbdbf49366c944cdd8e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba1e3730e51a0f5ccb4a0e9c31896c2d93849c6e182931fbec42425c5d7ff857 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..46983197e78651294633914d760ae69faa27793d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89875532e2bb88ff00dd18ad243286d399958c62da342e6e1ddd1c9f9c8465d2 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..535397093c1b34822955b159e0d42ce537d97465 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a18f73a4985b028b374cf2985a9fe1706adbe72e059fd176596eade024ae6f10 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..765645fc4a94fb831b2d53bb7f148f064367d852 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cc8927a984794c0ba328a1160953232e3759429d0dcacbad46abc5c4b0f038a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf54a839aaa564cb41c7b758f7e7846a4a7a40d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b65dd4f7c4351bc70e2ca42eddd7e6f1c0fdc194263d9f4061210a8ec7559c0e +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..191d8a4ba8a4c935ed32f4e3c31ac3719d48c513 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/trainer_state.json @@ -0,0 +1,8434 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.9377283462182837, + "eval_steps": 2000, + "global_step": 12000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0016148240851012292, + "grad_norm": 43.90101623535156, + "learning_rate": 0.0001, + "loss": 307.2084, + "step": 10 + }, + { + "epoch": 0.0032296481702024584, + "grad_norm": 19.106313705444336, + "learning_rate": 0.0001, + "loss": 320.2794, + "step": 20 + }, + { + "epoch": 0.004844472255303688, + "grad_norm": 5.771847248077393, + "learning_rate": 0.0001, + "loss": 247.4977, + "step": 30 + }, + { + "epoch": 0.006459296340404917, + "grad_norm": 2.630659580230713, + "learning_rate": 0.0001, + "loss": 229.9593, + "step": 40 + }, + { + "epoch": 0.008074120425506147, + "grad_norm": 2.487499475479126, + "learning_rate": 0.0001, + "loss": 228.65, + "step": 50 + }, + { + "epoch": 0.009688944510607376, + "grad_norm": 2.078263521194458, + "learning_rate": 0.0001, + "loss": 227.648, + "step": 60 + }, + { + "epoch": 0.011303768595708604, + "grad_norm": 3.105417490005493, + "learning_rate": 0.0001, + "loss": 227.9345, + "step": 70 + }, + { + "epoch": 0.012918592680809833, + "grad_norm": 2.2145848274230957, + "learning_rate": 0.0001, + "loss": 227.3662, + "step": 80 + }, + { + "epoch": 0.014533416765911064, + "grad_norm": 1.9644615650177002, + "learning_rate": 0.0001, + "loss": 226.9145, + "step": 90 + }, + { + "epoch": 0.016148240851012293, + "grad_norm": 1.867382287979126, + "learning_rate": 0.0001, + "loss": 227.5703, + "step": 100 + }, + { + "epoch": 0.017763064936113522, + "grad_norm": 1.4613710641860962, + "learning_rate": 0.0001, + "loss": 227.1107, + "step": 110 + }, + { + "epoch": 0.01937788902121475, + "grad_norm": 2.8953869342803955, + "learning_rate": 0.0001, + "loss": 226.8185, + "step": 120 + }, + { + "epoch": 0.02099271310631598, + "grad_norm": 1.8446482419967651, + "learning_rate": 0.0001, + "loss": 226.9624, + "step": 130 + }, + { + "epoch": 0.02260753719141721, + "grad_norm": 1.3048362731933594, + "learning_rate": 0.0001, + "loss": 226.8404, + "step": 140 + }, + { + "epoch": 0.024222361276518438, + "grad_norm": 1.7100074291229248, + "learning_rate": 0.0001, + "loss": 226.1071, + "step": 150 + }, + { + "epoch": 0.025837185361619667, + "grad_norm": 1.7875330448150635, + "learning_rate": 0.0001, + "loss": 226.0764, + "step": 160 + }, + { + "epoch": 0.0274520094467209, + "grad_norm": 1.7503769397735596, + "learning_rate": 0.0001, + "loss": 226.0772, + "step": 170 + }, + { + "epoch": 0.029066833531822128, + "grad_norm": 2.7083983421325684, + "learning_rate": 0.0001, + "loss": 224.7667, + "step": 180 + }, + { + "epoch": 0.030681657616923357, + "grad_norm": 1.8192371129989624, + "learning_rate": 0.0001, + "loss": 223.6494, + "step": 190 + }, + { + "epoch": 0.032296481702024586, + "grad_norm": 3.210358142852783, + "learning_rate": 0.0001, + "loss": 223.3545, + "step": 200 + }, + { + "epoch": 0.03391130578712581, + "grad_norm": 5.819669246673584, + "learning_rate": 0.0001, + "loss": 222.1732, + "step": 210 + }, + { + "epoch": 0.035526129872227044, + "grad_norm": 4.185793399810791, + "learning_rate": 0.0001, + "loss": 221.6866, + "step": 220 + }, + { + "epoch": 0.03714095395732828, + "grad_norm": 2.7514216899871826, + "learning_rate": 0.0001, + "loss": 221.0769, + "step": 230 + }, + { + "epoch": 0.0387557780424295, + "grad_norm": 1.5478988885879517, + "learning_rate": 0.0001, + "loss": 219.7232, + "step": 240 + }, + { + "epoch": 0.040370602127530734, + "grad_norm": 1.4958223104476929, + "learning_rate": 0.0001, + "loss": 218.5454, + "step": 250 + }, + { + "epoch": 0.04198542621263196, + "grad_norm": 6.36635684967041, + "learning_rate": 0.0001, + "loss": 218.5061, + "step": 260 + }, + { + "epoch": 0.04360025029773319, + "grad_norm": 3.029601812362671, + "learning_rate": 0.0001, + "loss": 216.3466, + "step": 270 + }, + { + "epoch": 0.04521507438283442, + "grad_norm": 2.6436100006103516, + "learning_rate": 0.0001, + "loss": 214.0003, + "step": 280 + }, + { + "epoch": 0.04682989846793565, + "grad_norm": 5.329655647277832, + "learning_rate": 0.0001, + "loss": 212.9169, + "step": 290 + }, + { + "epoch": 0.048444722553036876, + "grad_norm": 4.930875301361084, + "learning_rate": 0.0001, + "loss": 209.7737, + "step": 300 + }, + { + "epoch": 0.05005954663813811, + "grad_norm": 18.902585983276367, + "learning_rate": 0.0001, + "loss": 209.4267, + "step": 310 + }, + { + "epoch": 0.051674370723239334, + "grad_norm": 5.364975929260254, + "learning_rate": 0.0001, + "loss": 206.49, + "step": 320 + }, + { + "epoch": 0.053289194808340566, + "grad_norm": 3.1421146392822266, + "learning_rate": 0.0001, + "loss": 204.8132, + "step": 330 + }, + { + "epoch": 0.0549040188934418, + "grad_norm": 3.399320602416992, + "learning_rate": 0.0001, + "loss": 202.4785, + "step": 340 + }, + { + "epoch": 0.056518842978543024, + "grad_norm": 5.137377738952637, + "learning_rate": 0.0001, + "loss": 199.5143, + "step": 350 + }, + { + "epoch": 0.058133667063644257, + "grad_norm": 5.4967522621154785, + "learning_rate": 0.0001, + "loss": 196.2809, + "step": 360 + }, + { + "epoch": 0.05974849114874548, + "grad_norm": 6.435070037841797, + "learning_rate": 0.0001, + "loss": 196.8831, + "step": 370 + }, + { + "epoch": 0.061363315233846714, + "grad_norm": 5.7334370613098145, + "learning_rate": 0.0001, + "loss": 193.141, + "step": 380 + }, + { + "epoch": 0.06297813931894794, + "grad_norm": 3.6522979736328125, + "learning_rate": 0.0001, + "loss": 191.5819, + "step": 390 + }, + { + "epoch": 0.06459296340404917, + "grad_norm": 2.614051103591919, + "learning_rate": 0.0001, + "loss": 190.046, + "step": 400 + }, + { + "epoch": 0.0662077874891504, + "grad_norm": 4.845924377441406, + "learning_rate": 0.0001, + "loss": 188.7121, + "step": 410 + }, + { + "epoch": 0.06782261157425162, + "grad_norm": 4.319681644439697, + "learning_rate": 0.0001, + "loss": 185.3697, + "step": 420 + }, + { + "epoch": 0.06943743565935286, + "grad_norm": 2.7018792629241943, + "learning_rate": 0.0001, + "loss": 183.5171, + "step": 430 + }, + { + "epoch": 0.07105225974445409, + "grad_norm": 2.6407113075256348, + "learning_rate": 0.0001, + "loss": 181.245, + "step": 440 + }, + { + "epoch": 0.07266708382955532, + "grad_norm": 3.314251184463501, + "learning_rate": 0.0001, + "loss": 179.3874, + "step": 450 + }, + { + "epoch": 0.07428190791465655, + "grad_norm": 2.635136365890503, + "learning_rate": 0.0001, + "loss": 177.8257, + "step": 460 + }, + { + "epoch": 0.07589673199975777, + "grad_norm": 5.227738857269287, + "learning_rate": 0.0001, + "loss": 176.4699, + "step": 470 + }, + { + "epoch": 0.077511556084859, + "grad_norm": 4.998415470123291, + "learning_rate": 0.0001, + "loss": 175.7601, + "step": 480 + }, + { + "epoch": 0.07912638016996024, + "grad_norm": 2.946892261505127, + "learning_rate": 0.0001, + "loss": 173.936, + "step": 490 + }, + { + "epoch": 0.08074120425506147, + "grad_norm": 4.20358943939209, + "learning_rate": 0.0001, + "loss": 172.065, + "step": 500 + }, + { + "epoch": 0.08235602834016269, + "grad_norm": 3.125765562057495, + "learning_rate": 0.0001, + "loss": 171.3727, + "step": 510 + }, + { + "epoch": 0.08397085242526392, + "grad_norm": 2.4870612621307373, + "learning_rate": 0.0001, + "loss": 169.9, + "step": 520 + }, + { + "epoch": 0.08558567651036515, + "grad_norm": 3.917384147644043, + "learning_rate": 0.0001, + "loss": 168.5049, + "step": 530 + }, + { + "epoch": 0.08720050059546638, + "grad_norm": 2.302142858505249, + "learning_rate": 0.0001, + "loss": 167.2027, + "step": 540 + }, + { + "epoch": 0.08881532468056762, + "grad_norm": 4.794841289520264, + "learning_rate": 0.0001, + "loss": 166.0887, + "step": 550 + }, + { + "epoch": 0.09043014876566884, + "grad_norm": 2.200291395187378, + "learning_rate": 0.0001, + "loss": 164.5882, + "step": 560 + }, + { + "epoch": 0.09204497285077007, + "grad_norm": 1.9572436809539795, + "learning_rate": 0.0001, + "loss": 163.3007, + "step": 570 + }, + { + "epoch": 0.0936597969358713, + "grad_norm": 3.443633556365967, + "learning_rate": 0.0001, + "loss": 160.6805, + "step": 580 + }, + { + "epoch": 0.09527462102097253, + "grad_norm": 2.6079020500183105, + "learning_rate": 0.0001, + "loss": 160.3372, + "step": 590 + }, + { + "epoch": 0.09688944510607375, + "grad_norm": 2.4401192665100098, + "learning_rate": 0.0001, + "loss": 160.0956, + "step": 600 + }, + { + "epoch": 0.09850426919117498, + "grad_norm": 2.686101198196411, + "learning_rate": 0.0001, + "loss": 157.6138, + "step": 610 + }, + { + "epoch": 0.10011909327627622, + "grad_norm": 2.594151496887207, + "learning_rate": 0.0001, + "loss": 156.0726, + "step": 620 + }, + { + "epoch": 0.10173391736137745, + "grad_norm": 2.509141683578491, + "learning_rate": 0.0001, + "loss": 155.9923, + "step": 630 + }, + { + "epoch": 0.10334874144647867, + "grad_norm": 2.4077377319335938, + "learning_rate": 0.0001, + "loss": 155.214, + "step": 640 + }, + { + "epoch": 0.1049635655315799, + "grad_norm": 4.083552837371826, + "learning_rate": 0.0001, + "loss": 152.8257, + "step": 650 + }, + { + "epoch": 0.10657838961668113, + "grad_norm": 2.7188713550567627, + "learning_rate": 0.0001, + "loss": 151.875, + "step": 660 + }, + { + "epoch": 0.10819321370178236, + "grad_norm": 2.41287899017334, + "learning_rate": 0.0001, + "loss": 151.935, + "step": 670 + }, + { + "epoch": 0.1098080377868836, + "grad_norm": 3.995708465576172, + "learning_rate": 0.0001, + "loss": 151.0069, + "step": 680 + }, + { + "epoch": 0.11142286187198482, + "grad_norm": 3.217705488204956, + "learning_rate": 0.0001, + "loss": 149.5922, + "step": 690 + }, + { + "epoch": 0.11303768595708605, + "grad_norm": 3.4349842071533203, + "learning_rate": 0.0001, + "loss": 148.0044, + "step": 700 + }, + { + "epoch": 0.11465251004218728, + "grad_norm": 2.106865167617798, + "learning_rate": 0.0001, + "loss": 147.8404, + "step": 710 + }, + { + "epoch": 0.11626733412728851, + "grad_norm": 1.8729510307312012, + "learning_rate": 0.0001, + "loss": 145.1299, + "step": 720 + }, + { + "epoch": 0.11788215821238973, + "grad_norm": 4.14620304107666, + "learning_rate": 0.0001, + "loss": 144.3928, + "step": 730 + }, + { + "epoch": 0.11949698229749096, + "grad_norm": 2.202505350112915, + "learning_rate": 0.0001, + "loss": 144.8357, + "step": 740 + }, + { + "epoch": 0.1211118063825922, + "grad_norm": 2.201470375061035, + "learning_rate": 0.0001, + "loss": 143.2202, + "step": 750 + }, + { + "epoch": 0.12272663046769343, + "grad_norm": 1.5977853536605835, + "learning_rate": 0.0001, + "loss": 143.3537, + "step": 760 + }, + { + "epoch": 0.12434145455279466, + "grad_norm": 2.4403927326202393, + "learning_rate": 0.0001, + "loss": 141.2914, + "step": 770 + }, + { + "epoch": 0.12595627863789588, + "grad_norm": 1.6895846128463745, + "learning_rate": 0.0001, + "loss": 139.9639, + "step": 780 + }, + { + "epoch": 0.12757110272299713, + "grad_norm": 3.4370665550231934, + "learning_rate": 0.0001, + "loss": 140.715, + "step": 790 + }, + { + "epoch": 0.12918592680809834, + "grad_norm": 2.1449077129364014, + "learning_rate": 0.0001, + "loss": 139.2094, + "step": 800 + }, + { + "epoch": 0.13080075089319956, + "grad_norm": 2.4578912258148193, + "learning_rate": 0.0001, + "loss": 137.6724, + "step": 810 + }, + { + "epoch": 0.1324155749783008, + "grad_norm": 1.8994814157485962, + "learning_rate": 0.0001, + "loss": 136.9551, + "step": 820 + }, + { + "epoch": 0.13403039906340203, + "grad_norm": 2.6531031131744385, + "learning_rate": 0.0001, + "loss": 136.5812, + "step": 830 + }, + { + "epoch": 0.13564522314850325, + "grad_norm": 3.8680083751678467, + "learning_rate": 0.0001, + "loss": 134.3655, + "step": 840 + }, + { + "epoch": 0.1372600472336045, + "grad_norm": 1.9594208002090454, + "learning_rate": 0.0001, + "loss": 135.4958, + "step": 850 + }, + { + "epoch": 0.1388748713187057, + "grad_norm": 1.5475000143051147, + "learning_rate": 0.0001, + "loss": 135.2142, + "step": 860 + }, + { + "epoch": 0.14048969540380696, + "grad_norm": 1.6569288969039917, + "learning_rate": 0.0001, + "loss": 133.4162, + "step": 870 + }, + { + "epoch": 0.14210451948890818, + "grad_norm": 1.6724224090576172, + "learning_rate": 0.0001, + "loss": 133.7519, + "step": 880 + }, + { + "epoch": 0.1437193435740094, + "grad_norm": 1.7299500703811646, + "learning_rate": 0.0001, + "loss": 131.471, + "step": 890 + }, + { + "epoch": 0.14533416765911064, + "grad_norm": 1.9444470405578613, + "learning_rate": 0.0001, + "loss": 128.5966, + "step": 900 + }, + { + "epoch": 0.14694899174421186, + "grad_norm": 2.404723644256592, + "learning_rate": 0.0001, + "loss": 129.9824, + "step": 910 + }, + { + "epoch": 0.1485638158293131, + "grad_norm": 2.1493897438049316, + "learning_rate": 0.0001, + "loss": 128.0219, + "step": 920 + }, + { + "epoch": 0.15017863991441432, + "grad_norm": 2.8861117362976074, + "learning_rate": 0.0001, + "loss": 128.5297, + "step": 930 + }, + { + "epoch": 0.15179346399951554, + "grad_norm": 2.546415090560913, + "learning_rate": 0.0001, + "loss": 126.6817, + "step": 940 + }, + { + "epoch": 0.1534082880846168, + "grad_norm": 1.727663278579712, + "learning_rate": 0.0001, + "loss": 125.0413, + "step": 950 + }, + { + "epoch": 0.155023112169718, + "grad_norm": 3.7156920433044434, + "learning_rate": 0.0001, + "loss": 124.8085, + "step": 960 + }, + { + "epoch": 0.15663793625481925, + "grad_norm": 3.0576517581939697, + "learning_rate": 0.0001, + "loss": 125.9107, + "step": 970 + }, + { + "epoch": 0.15825276033992047, + "grad_norm": 1.9951574802398682, + "learning_rate": 0.0001, + "loss": 123.5359, + "step": 980 + }, + { + "epoch": 0.1598675844250217, + "grad_norm": 1.4620885848999023, + "learning_rate": 0.0001, + "loss": 121.9363, + "step": 990 + }, + { + "epoch": 0.16148240851012294, + "grad_norm": 1.7640169858932495, + "learning_rate": 0.0001, + "loss": 121.1764, + "step": 1000 + }, + { + "epoch": 0.16309723259522416, + "grad_norm": 2.921750545501709, + "learning_rate": 0.0001, + "loss": 119.6931, + "step": 1010 + }, + { + "epoch": 0.16471205668032537, + "grad_norm": 1.5623588562011719, + "learning_rate": 0.0001, + "loss": 119.097, + "step": 1020 + }, + { + "epoch": 0.16632688076542662, + "grad_norm": 2.2386627197265625, + "learning_rate": 0.0001, + "loss": 120.9224, + "step": 1030 + }, + { + "epoch": 0.16794170485052784, + "grad_norm": 1.3213791847229004, + "learning_rate": 0.0001, + "loss": 119.5649, + "step": 1040 + }, + { + "epoch": 0.16955652893562909, + "grad_norm": 1.5255440473556519, + "learning_rate": 0.0001, + "loss": 118.1524, + "step": 1050 + }, + { + "epoch": 0.1711713530207303, + "grad_norm": 1.6038514375686646, + "learning_rate": 0.0001, + "loss": 118.6307, + "step": 1060 + }, + { + "epoch": 0.17278617710583152, + "grad_norm": 2.356739044189453, + "learning_rate": 0.0001, + "loss": 116.8472, + "step": 1070 + }, + { + "epoch": 0.17440100119093277, + "grad_norm": 2.5326743125915527, + "learning_rate": 0.0001, + "loss": 115.6625, + "step": 1080 + }, + { + "epoch": 0.176015825276034, + "grad_norm": 1.2823396921157837, + "learning_rate": 0.0001, + "loss": 115.3312, + "step": 1090 + }, + { + "epoch": 0.17763064936113523, + "grad_norm": 1.981788158416748, + "learning_rate": 0.0001, + "loss": 116.2257, + "step": 1100 + }, + { + "epoch": 0.17924547344623645, + "grad_norm": 1.540440559387207, + "learning_rate": 0.0001, + "loss": 114.5031, + "step": 1110 + }, + { + "epoch": 0.18086029753133767, + "grad_norm": 1.7667800188064575, + "learning_rate": 0.0001, + "loss": 113.7663, + "step": 1120 + }, + { + "epoch": 0.18247512161643892, + "grad_norm": 2.2793102264404297, + "learning_rate": 0.0001, + "loss": 115.219, + "step": 1130 + }, + { + "epoch": 0.18408994570154014, + "grad_norm": 2.132046699523926, + "learning_rate": 0.0001, + "loss": 112.8482, + "step": 1140 + }, + { + "epoch": 0.18570476978664135, + "grad_norm": 1.3153988122940063, + "learning_rate": 0.0001, + "loss": 111.4546, + "step": 1150 + }, + { + "epoch": 0.1873195938717426, + "grad_norm": 1.5246292352676392, + "learning_rate": 0.0001, + "loss": 112.4348, + "step": 1160 + }, + { + "epoch": 0.18893441795684382, + "grad_norm": 1.365096092224121, + "learning_rate": 0.0001, + "loss": 109.755, + "step": 1170 + }, + { + "epoch": 0.19054924204194507, + "grad_norm": 1.2777953147888184, + "learning_rate": 0.0001, + "loss": 111.0318, + "step": 1180 + }, + { + "epoch": 0.19216406612704628, + "grad_norm": 1.8672829866409302, + "learning_rate": 0.0001, + "loss": 109.1648, + "step": 1190 + }, + { + "epoch": 0.1937788902121475, + "grad_norm": 1.709987759590149, + "learning_rate": 0.0001, + "loss": 108.1616, + "step": 1200 + }, + { + "epoch": 0.19539371429724875, + "grad_norm": 1.176277995109558, + "learning_rate": 0.0001, + "loss": 109.3591, + "step": 1210 + }, + { + "epoch": 0.19700853838234997, + "grad_norm": 1.4181180000305176, + "learning_rate": 0.0001, + "loss": 110.0216, + "step": 1220 + }, + { + "epoch": 0.19862336246745121, + "grad_norm": 1.3246116638183594, + "learning_rate": 0.0001, + "loss": 107.9769, + "step": 1230 + }, + { + "epoch": 0.20023818655255243, + "grad_norm": 1.2501498460769653, + "learning_rate": 0.0001, + "loss": 108.2601, + "step": 1240 + }, + { + "epoch": 0.20185301063765365, + "grad_norm": 1.2277404069900513, + "learning_rate": 0.0001, + "loss": 107.1258, + "step": 1250 + }, + { + "epoch": 0.2034678347227549, + "grad_norm": 1.8064894676208496, + "learning_rate": 0.0001, + "loss": 105.6278, + "step": 1260 + }, + { + "epoch": 0.20508265880785612, + "grad_norm": 1.708897352218628, + "learning_rate": 0.0001, + "loss": 106.9269, + "step": 1270 + }, + { + "epoch": 0.20669748289295733, + "grad_norm": 1.4507478475570679, + "learning_rate": 0.0001, + "loss": 106.9236, + "step": 1280 + }, + { + "epoch": 0.20831230697805858, + "grad_norm": 1.4904310703277588, + "learning_rate": 0.0001, + "loss": 106.0653, + "step": 1290 + }, + { + "epoch": 0.2099271310631598, + "grad_norm": 1.2637165784835815, + "learning_rate": 0.0001, + "loss": 104.8665, + "step": 1300 + }, + { + "epoch": 0.21154195514826105, + "grad_norm": 1.4085133075714111, + "learning_rate": 0.0001, + "loss": 104.8891, + "step": 1310 + }, + { + "epoch": 0.21315677923336226, + "grad_norm": 1.5710629224777222, + "learning_rate": 0.0001, + "loss": 104.3009, + "step": 1320 + }, + { + "epoch": 0.21477160331846348, + "grad_norm": 1.3634663820266724, + "learning_rate": 0.0001, + "loss": 104.4545, + "step": 1330 + }, + { + "epoch": 0.21638642740356473, + "grad_norm": 1.5565859079360962, + "learning_rate": 0.0001, + "loss": 103.6205, + "step": 1340 + }, + { + "epoch": 0.21800125148866595, + "grad_norm": 1.4491888284683228, + "learning_rate": 0.0001, + "loss": 104.0892, + "step": 1350 + }, + { + "epoch": 0.2196160755737672, + "grad_norm": 1.7609063386917114, + "learning_rate": 0.0001, + "loss": 104.5177, + "step": 1360 + }, + { + "epoch": 0.2212308996588684, + "grad_norm": 1.6723753213882446, + "learning_rate": 0.0001, + "loss": 104.5552, + "step": 1370 + }, + { + "epoch": 0.22284572374396963, + "grad_norm": 1.3781665563583374, + "learning_rate": 0.0001, + "loss": 102.1022, + "step": 1380 + }, + { + "epoch": 0.22446054782907088, + "grad_norm": 1.5397366285324097, + "learning_rate": 0.0001, + "loss": 102.5882, + "step": 1390 + }, + { + "epoch": 0.2260753719141721, + "grad_norm": 1.4567879438400269, + "learning_rate": 0.0001, + "loss": 101.3781, + "step": 1400 + }, + { + "epoch": 0.22769019599927334, + "grad_norm": 1.2882301807403564, + "learning_rate": 0.0001, + "loss": 100.9378, + "step": 1410 + }, + { + "epoch": 0.22930502008437456, + "grad_norm": 1.4510424137115479, + "learning_rate": 0.0001, + "loss": 101.5383, + "step": 1420 + }, + { + "epoch": 0.23091984416947578, + "grad_norm": 2.8331449031829834, + "learning_rate": 0.0001, + "loss": 100.2773, + "step": 1430 + }, + { + "epoch": 0.23253466825457703, + "grad_norm": 1.363616943359375, + "learning_rate": 0.0001, + "loss": 101.646, + "step": 1440 + }, + { + "epoch": 0.23414949233967824, + "grad_norm": 1.2184548377990723, + "learning_rate": 0.0001, + "loss": 101.1133, + "step": 1450 + }, + { + "epoch": 0.23576431642477946, + "grad_norm": 2.235656976699829, + "learning_rate": 0.0001, + "loss": 100.2044, + "step": 1460 + }, + { + "epoch": 0.2373791405098807, + "grad_norm": 1.2334418296813965, + "learning_rate": 0.0001, + "loss": 99.4031, + "step": 1470 + }, + { + "epoch": 0.23899396459498193, + "grad_norm": 1.2394684553146362, + "learning_rate": 0.0001, + "loss": 101.1247, + "step": 1480 + }, + { + "epoch": 0.24060878868008317, + "grad_norm": 1.5218186378479004, + "learning_rate": 0.0001, + "loss": 100.4413, + "step": 1490 + }, + { + "epoch": 0.2422236127651844, + "grad_norm": 1.2112364768981934, + "learning_rate": 0.0001, + "loss": 99.8722, + "step": 1500 + }, + { + "epoch": 0.2438384368502856, + "grad_norm": 1.2277610301971436, + "learning_rate": 0.0001, + "loss": 99.0276, + "step": 1510 + }, + { + "epoch": 0.24545326093538686, + "grad_norm": 1.1005053520202637, + "learning_rate": 0.0001, + "loss": 98.6764, + "step": 1520 + }, + { + "epoch": 0.24706808502048808, + "grad_norm": 1.0754456520080566, + "learning_rate": 0.0001, + "loss": 97.8968, + "step": 1530 + }, + { + "epoch": 0.24868290910558932, + "grad_norm": 1.0695950984954834, + "learning_rate": 0.0001, + "loss": 98.4032, + "step": 1540 + }, + { + "epoch": 0.25029773319069054, + "grad_norm": 1.5098627805709839, + "learning_rate": 0.0001, + "loss": 97.9391, + "step": 1550 + }, + { + "epoch": 0.25191255727579176, + "grad_norm": 1.006681203842163, + "learning_rate": 0.0001, + "loss": 96.5525, + "step": 1560 + }, + { + "epoch": 0.253527381360893, + "grad_norm": 1.3987332582473755, + "learning_rate": 0.0001, + "loss": 98.383, + "step": 1570 + }, + { + "epoch": 0.25514220544599425, + "grad_norm": 1.290887713432312, + "learning_rate": 0.0001, + "loss": 99.0901, + "step": 1580 + }, + { + "epoch": 0.25675702953109547, + "grad_norm": 0.991177499294281, + "learning_rate": 0.0001, + "loss": 97.7093, + "step": 1590 + }, + { + "epoch": 0.2583718536161967, + "grad_norm": 1.147647738456726, + "learning_rate": 0.0001, + "loss": 96.9181, + "step": 1600 + }, + { + "epoch": 0.2599866777012979, + "grad_norm": 1.1756514310836792, + "learning_rate": 0.0001, + "loss": 98.3935, + "step": 1610 + }, + { + "epoch": 0.2616015017863991, + "grad_norm": 1.1708602905273438, + "learning_rate": 0.0001, + "loss": 96.715, + "step": 1620 + }, + { + "epoch": 0.2632163258715004, + "grad_norm": 1.4577715396881104, + "learning_rate": 0.0001, + "loss": 97.2971, + "step": 1630 + }, + { + "epoch": 0.2648311499566016, + "grad_norm": 1.1172800064086914, + "learning_rate": 0.0001, + "loss": 97.6494, + "step": 1640 + }, + { + "epoch": 0.26644597404170284, + "grad_norm": 1.1491258144378662, + "learning_rate": 0.0001, + "loss": 97.5503, + "step": 1650 + }, + { + "epoch": 0.26806079812680406, + "grad_norm": 1.2188555002212524, + "learning_rate": 0.0001, + "loss": 95.3423, + "step": 1660 + }, + { + "epoch": 0.2696756222119053, + "grad_norm": 1.8595614433288574, + "learning_rate": 0.0001, + "loss": 96.0112, + "step": 1670 + }, + { + "epoch": 0.2712904462970065, + "grad_norm": 1.1887950897216797, + "learning_rate": 0.0001, + "loss": 95.4939, + "step": 1680 + }, + { + "epoch": 0.27290527038210777, + "grad_norm": 1.1777839660644531, + "learning_rate": 0.0001, + "loss": 95.9619, + "step": 1690 + }, + { + "epoch": 0.274520094467209, + "grad_norm": 1.1241233348846436, + "learning_rate": 0.0001, + "loss": 96.2834, + "step": 1700 + }, + { + "epoch": 0.2761349185523102, + "grad_norm": 1.0900121927261353, + "learning_rate": 0.0001, + "loss": 94.7518, + "step": 1710 + }, + { + "epoch": 0.2777497426374114, + "grad_norm": 1.0615205764770508, + "learning_rate": 0.0001, + "loss": 95.8032, + "step": 1720 + }, + { + "epoch": 0.27936456672251264, + "grad_norm": 1.2626168727874756, + "learning_rate": 0.0001, + "loss": 93.9454, + "step": 1730 + }, + { + "epoch": 0.2809793908076139, + "grad_norm": 0.8869941830635071, + "learning_rate": 0.0001, + "loss": 95.4085, + "step": 1740 + }, + { + "epoch": 0.28259421489271513, + "grad_norm": 1.0350773334503174, + "learning_rate": 0.0001, + "loss": 95.7954, + "step": 1750 + }, + { + "epoch": 0.28420903897781635, + "grad_norm": 1.364768147468567, + "learning_rate": 0.0001, + "loss": 93.6447, + "step": 1760 + }, + { + "epoch": 0.28582386306291757, + "grad_norm": 1.0115883350372314, + "learning_rate": 0.0001, + "loss": 93.7905, + "step": 1770 + }, + { + "epoch": 0.2874386871480188, + "grad_norm": 0.9393435120582581, + "learning_rate": 0.0001, + "loss": 94.3108, + "step": 1780 + }, + { + "epoch": 0.28905351123312006, + "grad_norm": 1.0769627094268799, + "learning_rate": 0.0001, + "loss": 92.566, + "step": 1790 + }, + { + "epoch": 0.2906683353182213, + "grad_norm": 2.526201009750366, + "learning_rate": 0.0001, + "loss": 92.8161, + "step": 1800 + }, + { + "epoch": 0.2922831594033225, + "grad_norm": 1.3066468238830566, + "learning_rate": 0.0001, + "loss": 92.0459, + "step": 1810 + }, + { + "epoch": 0.2938979834884237, + "grad_norm": 1.121713638305664, + "learning_rate": 0.0001, + "loss": 93.5032, + "step": 1820 + }, + { + "epoch": 0.29551280757352494, + "grad_norm": 1.3744316101074219, + "learning_rate": 0.0001, + "loss": 94.2623, + "step": 1830 + }, + { + "epoch": 0.2971276316586262, + "grad_norm": 1.1561540365219116, + "learning_rate": 0.0001, + "loss": 94.1611, + "step": 1840 + }, + { + "epoch": 0.29874245574372743, + "grad_norm": 1.3947464227676392, + "learning_rate": 0.0001, + "loss": 93.5962, + "step": 1850 + }, + { + "epoch": 0.30035727982882865, + "grad_norm": 1.0011566877365112, + "learning_rate": 0.0001, + "loss": 92.0199, + "step": 1860 + }, + { + "epoch": 0.30197210391392987, + "grad_norm": 0.9978747963905334, + "learning_rate": 0.0001, + "loss": 91.0945, + "step": 1870 + }, + { + "epoch": 0.3035869279990311, + "grad_norm": 0.92522132396698, + "learning_rate": 0.0001, + "loss": 93.274, + "step": 1880 + }, + { + "epoch": 0.30520175208413236, + "grad_norm": 1.0089120864868164, + "learning_rate": 0.0001, + "loss": 91.9308, + "step": 1890 + }, + { + "epoch": 0.3068165761692336, + "grad_norm": 1.0535134077072144, + "learning_rate": 0.0001, + "loss": 92.563, + "step": 1900 + }, + { + "epoch": 0.3084314002543348, + "grad_norm": 0.9782275557518005, + "learning_rate": 0.0001, + "loss": 90.3486, + "step": 1910 + }, + { + "epoch": 0.310046224339436, + "grad_norm": 1.1433162689208984, + "learning_rate": 0.0001, + "loss": 90.4977, + "step": 1920 + }, + { + "epoch": 0.31166104842453723, + "grad_norm": 0.98392653465271, + "learning_rate": 0.0001, + "loss": 92.6529, + "step": 1930 + }, + { + "epoch": 0.3132758725096385, + "grad_norm": 0.8538148403167725, + "learning_rate": 0.0001, + "loss": 90.6859, + "step": 1940 + }, + { + "epoch": 0.3148906965947397, + "grad_norm": 1.019405484199524, + "learning_rate": 0.0001, + "loss": 91.0028, + "step": 1950 + }, + { + "epoch": 0.31650552067984095, + "grad_norm": 0.9828067421913147, + "learning_rate": 0.0001, + "loss": 89.9826, + "step": 1960 + }, + { + "epoch": 0.31812034476494216, + "grad_norm": 0.9725450277328491, + "learning_rate": 0.0001, + "loss": 91.7342, + "step": 1970 + }, + { + "epoch": 0.3197351688500434, + "grad_norm": 1.0139408111572266, + "learning_rate": 0.0001, + "loss": 90.5362, + "step": 1980 + }, + { + "epoch": 0.3213499929351446, + "grad_norm": 1.2493853569030762, + "learning_rate": 0.0001, + "loss": 92.7222, + "step": 1990 + }, + { + "epoch": 0.3229648170202459, + "grad_norm": 1.0152915716171265, + "learning_rate": 0.0001, + "loss": 92.2271, + "step": 2000 + }, + { + "epoch": 0.3245796411053471, + "grad_norm": 0.9745819568634033, + "learning_rate": 0.0001, + "loss": 88.7034, + "step": 2010 + }, + { + "epoch": 0.3261944651904483, + "grad_norm": 1.009624719619751, + "learning_rate": 0.0001, + "loss": 90.7428, + "step": 2020 + }, + { + "epoch": 0.32780928927554953, + "grad_norm": 0.910858690738678, + "learning_rate": 0.0001, + "loss": 89.8199, + "step": 2030 + }, + { + "epoch": 0.32942411336065075, + "grad_norm": 1.1745878458023071, + "learning_rate": 0.0001, + "loss": 90.8267, + "step": 2040 + }, + { + "epoch": 0.331038937445752, + "grad_norm": 0.9409273266792297, + "learning_rate": 0.0001, + "loss": 90.2937, + "step": 2050 + }, + { + "epoch": 0.33265376153085324, + "grad_norm": 1.4152942895889282, + "learning_rate": 0.0001, + "loss": 90.3965, + "step": 2060 + }, + { + "epoch": 0.33426858561595446, + "grad_norm": 0.9529693722724915, + "learning_rate": 0.0001, + "loss": 90.4739, + "step": 2070 + }, + { + "epoch": 0.3358834097010557, + "grad_norm": 0.8531096577644348, + "learning_rate": 0.0001, + "loss": 90.9301, + "step": 2080 + }, + { + "epoch": 0.3374982337861569, + "grad_norm": 0.9314167499542236, + "learning_rate": 0.0001, + "loss": 90.4235, + "step": 2090 + }, + { + "epoch": 0.33911305787125817, + "grad_norm": 0.9253250360488892, + "learning_rate": 0.0001, + "loss": 88.1893, + "step": 2100 + }, + { + "epoch": 0.3407278819563594, + "grad_norm": 1.773053526878357, + "learning_rate": 0.0001, + "loss": 88.3417, + "step": 2110 + }, + { + "epoch": 0.3423427060414606, + "grad_norm": 0.9432951211929321, + "learning_rate": 0.0001, + "loss": 89.7141, + "step": 2120 + }, + { + "epoch": 0.34395753012656183, + "grad_norm": 3.8847527503967285, + "learning_rate": 0.0001, + "loss": 89.1009, + "step": 2130 + }, + { + "epoch": 0.34557235421166305, + "grad_norm": 1.0309821367263794, + "learning_rate": 0.0001, + "loss": 89.2965, + "step": 2140 + }, + { + "epoch": 0.3471871782967643, + "grad_norm": 0.9590495824813843, + "learning_rate": 0.0001, + "loss": 89.1734, + "step": 2150 + }, + { + "epoch": 0.34880200238186554, + "grad_norm": 0.9943560361862183, + "learning_rate": 0.0001, + "loss": 85.425, + "step": 2160 + }, + { + "epoch": 0.35041682646696676, + "grad_norm": 1.3322339057922363, + "learning_rate": 0.0001, + "loss": 89.7791, + "step": 2170 + }, + { + "epoch": 0.352031650552068, + "grad_norm": 1.0212253332138062, + "learning_rate": 0.0001, + "loss": 89.3771, + "step": 2180 + }, + { + "epoch": 0.3536464746371692, + "grad_norm": 1.0446089506149292, + "learning_rate": 0.0001, + "loss": 88.6894, + "step": 2190 + }, + { + "epoch": 0.35526129872227047, + "grad_norm": 1.05767822265625, + "learning_rate": 0.0001, + "loss": 89.5341, + "step": 2200 + }, + { + "epoch": 0.3568761228073717, + "grad_norm": 1.0373681783676147, + "learning_rate": 0.0001, + "loss": 89.7173, + "step": 2210 + }, + { + "epoch": 0.3584909468924729, + "grad_norm": 0.9098039269447327, + "learning_rate": 0.0001, + "loss": 89.2116, + "step": 2220 + }, + { + "epoch": 0.3601057709775741, + "grad_norm": 1.028130292892456, + "learning_rate": 0.0001, + "loss": 88.6627, + "step": 2230 + }, + { + "epoch": 0.36172059506267534, + "grad_norm": 1.0295171737670898, + "learning_rate": 0.0001, + "loss": 89.608, + "step": 2240 + }, + { + "epoch": 0.36333541914777656, + "grad_norm": 0.9720722436904907, + "learning_rate": 0.0001, + "loss": 88.3624, + "step": 2250 + }, + { + "epoch": 0.36495024323287784, + "grad_norm": 0.839654803276062, + "learning_rate": 0.0001, + "loss": 87.2983, + "step": 2260 + }, + { + "epoch": 0.36656506731797905, + "grad_norm": 1.1600219011306763, + "learning_rate": 0.0001, + "loss": 86.6004, + "step": 2270 + }, + { + "epoch": 0.3681798914030803, + "grad_norm": 1.0513924360275269, + "learning_rate": 0.0001, + "loss": 86.6062, + "step": 2280 + }, + { + "epoch": 0.3697947154881815, + "grad_norm": 0.9728474617004395, + "learning_rate": 0.0001, + "loss": 86.1856, + "step": 2290 + }, + { + "epoch": 0.3714095395732827, + "grad_norm": 1.0196386575698853, + "learning_rate": 0.0001, + "loss": 87.6088, + "step": 2300 + }, + { + "epoch": 0.373024363658384, + "grad_norm": 1.0439444780349731, + "learning_rate": 0.0001, + "loss": 85.2374, + "step": 2310 + }, + { + "epoch": 0.3746391877434852, + "grad_norm": 1.0020275115966797, + "learning_rate": 0.0001, + "loss": 86.0596, + "step": 2320 + }, + { + "epoch": 0.3762540118285864, + "grad_norm": 1.0107691287994385, + "learning_rate": 0.0001, + "loss": 86.9513, + "step": 2330 + }, + { + "epoch": 0.37786883591368764, + "grad_norm": 1.680018663406372, + "learning_rate": 0.0001, + "loss": 87.8885, + "step": 2340 + }, + { + "epoch": 0.37948365999878886, + "grad_norm": 0.9181709885597229, + "learning_rate": 0.0001, + "loss": 86.7662, + "step": 2350 + }, + { + "epoch": 0.38109848408389013, + "grad_norm": 0.9598278403282166, + "learning_rate": 0.0001, + "loss": 88.3775, + "step": 2360 + }, + { + "epoch": 0.38271330816899135, + "grad_norm": 1.0707905292510986, + "learning_rate": 0.0001, + "loss": 86.0997, + "step": 2370 + }, + { + "epoch": 0.38432813225409257, + "grad_norm": 1.125797152519226, + "learning_rate": 0.0001, + "loss": 85.8117, + "step": 2380 + }, + { + "epoch": 0.3859429563391938, + "grad_norm": 1.0251439809799194, + "learning_rate": 0.0001, + "loss": 86.0806, + "step": 2390 + }, + { + "epoch": 0.387557780424295, + "grad_norm": 1.0193125009536743, + "learning_rate": 0.0001, + "loss": 87.264, + "step": 2400 + }, + { + "epoch": 0.3891726045093963, + "grad_norm": 0.9309751391410828, + "learning_rate": 0.0001, + "loss": 85.9187, + "step": 2410 + }, + { + "epoch": 0.3907874285944975, + "grad_norm": 1.1025906801223755, + "learning_rate": 0.0001, + "loss": 86.4365, + "step": 2420 + }, + { + "epoch": 0.3924022526795987, + "grad_norm": 0.9880627393722534, + "learning_rate": 0.0001, + "loss": 86.2303, + "step": 2430 + }, + { + "epoch": 0.39401707676469994, + "grad_norm": 0.9520611763000488, + "learning_rate": 0.0001, + "loss": 84.9035, + "step": 2440 + }, + { + "epoch": 0.39563190084980115, + "grad_norm": 0.8641896843910217, + "learning_rate": 0.0001, + "loss": 85.4754, + "step": 2450 + }, + { + "epoch": 0.39724672493490243, + "grad_norm": 1.1162904500961304, + "learning_rate": 0.0001, + "loss": 85.6767, + "step": 2460 + }, + { + "epoch": 0.39886154902000365, + "grad_norm": 0.9073636531829834, + "learning_rate": 0.0001, + "loss": 85.4198, + "step": 2470 + }, + { + "epoch": 0.40047637310510487, + "grad_norm": 1.1135166883468628, + "learning_rate": 0.0001, + "loss": 86.0884, + "step": 2480 + }, + { + "epoch": 0.4020911971902061, + "grad_norm": 0.9273980259895325, + "learning_rate": 0.0001, + "loss": 85.5473, + "step": 2490 + }, + { + "epoch": 0.4037060212753073, + "grad_norm": 0.8653831481933594, + "learning_rate": 0.0001, + "loss": 87.279, + "step": 2500 + }, + { + "epoch": 0.4053208453604086, + "grad_norm": 0.8535159826278687, + "learning_rate": 0.0001, + "loss": 86.7072, + "step": 2510 + }, + { + "epoch": 0.4069356694455098, + "grad_norm": 0.8864684700965881, + "learning_rate": 0.0001, + "loss": 86.441, + "step": 2520 + }, + { + "epoch": 0.408550493530611, + "grad_norm": 0.8042080998420715, + "learning_rate": 0.0001, + "loss": 83.1706, + "step": 2530 + }, + { + "epoch": 0.41016531761571223, + "grad_norm": 1.0486373901367188, + "learning_rate": 0.0001, + "loss": 83.54, + "step": 2540 + }, + { + "epoch": 0.41178014170081345, + "grad_norm": 0.8087959289550781, + "learning_rate": 0.0001, + "loss": 85.7615, + "step": 2550 + }, + { + "epoch": 0.41339496578591467, + "grad_norm": 1.5032018423080444, + "learning_rate": 0.0001, + "loss": 85.7368, + "step": 2560 + }, + { + "epoch": 0.41500978987101594, + "grad_norm": 1.0786182880401611, + "learning_rate": 0.0001, + "loss": 86.3484, + "step": 2570 + }, + { + "epoch": 0.41662461395611716, + "grad_norm": 0.8744294047355652, + "learning_rate": 0.0001, + "loss": 83.5945, + "step": 2580 + }, + { + "epoch": 0.4182394380412184, + "grad_norm": 0.7840721607208252, + "learning_rate": 0.0001, + "loss": 86.3831, + "step": 2590 + }, + { + "epoch": 0.4198542621263196, + "grad_norm": 0.7977836728096008, + "learning_rate": 0.0001, + "loss": 83.6314, + "step": 2600 + }, + { + "epoch": 0.4214690862114208, + "grad_norm": 0.892484188079834, + "learning_rate": 0.0001, + "loss": 86.4223, + "step": 2610 + }, + { + "epoch": 0.4230839102965221, + "grad_norm": 0.912995457649231, + "learning_rate": 0.0001, + "loss": 83.8971, + "step": 2620 + }, + { + "epoch": 0.4246987343816233, + "grad_norm": 0.9566861987113953, + "learning_rate": 0.0001, + "loss": 85.291, + "step": 2630 + }, + { + "epoch": 0.42631355846672453, + "grad_norm": 0.8324154615402222, + "learning_rate": 0.0001, + "loss": 84.6233, + "step": 2640 + }, + { + "epoch": 0.42792838255182575, + "grad_norm": 0.9490360617637634, + "learning_rate": 0.0001, + "loss": 81.799, + "step": 2650 + }, + { + "epoch": 0.42954320663692697, + "grad_norm": 0.9744566679000854, + "learning_rate": 0.0001, + "loss": 85.4837, + "step": 2660 + }, + { + "epoch": 0.43115803072202824, + "grad_norm": 0.941614031791687, + "learning_rate": 0.0001, + "loss": 82.838, + "step": 2670 + }, + { + "epoch": 0.43277285480712946, + "grad_norm": 1.338364601135254, + "learning_rate": 0.0001, + "loss": 84.4224, + "step": 2680 + }, + { + "epoch": 0.4343876788922307, + "grad_norm": 0.8643816113471985, + "learning_rate": 0.0001, + "loss": 84.3959, + "step": 2690 + }, + { + "epoch": 0.4360025029773319, + "grad_norm": 1.0231473445892334, + "learning_rate": 0.0001, + "loss": 82.3144, + "step": 2700 + }, + { + "epoch": 0.4376173270624331, + "grad_norm": 0.9089650511741638, + "learning_rate": 0.0001, + "loss": 83.0326, + "step": 2710 + }, + { + "epoch": 0.4392321511475344, + "grad_norm": 0.893628716468811, + "learning_rate": 0.0001, + "loss": 83.0981, + "step": 2720 + }, + { + "epoch": 0.4408469752326356, + "grad_norm": 0.869076132774353, + "learning_rate": 0.0001, + "loss": 84.2571, + "step": 2730 + }, + { + "epoch": 0.4424617993177368, + "grad_norm": 0.7964235544204712, + "learning_rate": 0.0001, + "loss": 83.3329, + "step": 2740 + }, + { + "epoch": 0.44407662340283804, + "grad_norm": 3.1645138263702393, + "learning_rate": 0.0001, + "loss": 83.7232, + "step": 2750 + }, + { + "epoch": 0.44569144748793926, + "grad_norm": 0.934644877910614, + "learning_rate": 0.0001, + "loss": 85.0688, + "step": 2760 + }, + { + "epoch": 0.44730627157304054, + "grad_norm": 0.8120711445808411, + "learning_rate": 0.0001, + "loss": 82.7706, + "step": 2770 + }, + { + "epoch": 0.44892109565814176, + "grad_norm": 0.8161435127258301, + "learning_rate": 0.0001, + "loss": 84.4826, + "step": 2780 + }, + { + "epoch": 0.450535919743243, + "grad_norm": 0.8659493327140808, + "learning_rate": 0.0001, + "loss": 82.6222, + "step": 2790 + }, + { + "epoch": 0.4521507438283442, + "grad_norm": 0.9306702613830566, + "learning_rate": 0.0001, + "loss": 83.5897, + "step": 2800 + }, + { + "epoch": 0.4537655679134454, + "grad_norm": 2.2544431686401367, + "learning_rate": 0.0001, + "loss": 82.9603, + "step": 2810 + }, + { + "epoch": 0.4553803919985467, + "grad_norm": 0.8868395686149597, + "learning_rate": 0.0001, + "loss": 82.6674, + "step": 2820 + }, + { + "epoch": 0.4569952160836479, + "grad_norm": 0.9336455464363098, + "learning_rate": 0.0001, + "loss": 84.6974, + "step": 2830 + }, + { + "epoch": 0.4586100401687491, + "grad_norm": 0.8814165592193604, + "learning_rate": 0.0001, + "loss": 83.9329, + "step": 2840 + }, + { + "epoch": 0.46022486425385034, + "grad_norm": 0.8125660419464111, + "learning_rate": 0.0001, + "loss": 84.6168, + "step": 2850 + }, + { + "epoch": 0.46183968833895156, + "grad_norm": 0.9447190165519714, + "learning_rate": 0.0001, + "loss": 82.4098, + "step": 2860 + }, + { + "epoch": 0.4634545124240528, + "grad_norm": 0.8080969452857971, + "learning_rate": 0.0001, + "loss": 83.4076, + "step": 2870 + }, + { + "epoch": 0.46506933650915405, + "grad_norm": 1.0586694478988647, + "learning_rate": 0.0001, + "loss": 83.3295, + "step": 2880 + }, + { + "epoch": 0.46668416059425527, + "grad_norm": 0.8207664489746094, + "learning_rate": 0.0001, + "loss": 84.5056, + "step": 2890 + }, + { + "epoch": 0.4682989846793565, + "grad_norm": 1.0647571086883545, + "learning_rate": 0.0001, + "loss": 83.0766, + "step": 2900 + }, + { + "epoch": 0.4699138087644577, + "grad_norm": 0.9416915774345398, + "learning_rate": 0.0001, + "loss": 83.8166, + "step": 2910 + }, + { + "epoch": 0.4715286328495589, + "grad_norm": 0.7479362487792969, + "learning_rate": 0.0001, + "loss": 82.7507, + "step": 2920 + }, + { + "epoch": 0.4731434569346602, + "grad_norm": 1.0775514841079712, + "learning_rate": 0.0001, + "loss": 84.4641, + "step": 2930 + }, + { + "epoch": 0.4747582810197614, + "grad_norm": 0.8736841082572937, + "learning_rate": 0.0001, + "loss": 82.0451, + "step": 2940 + }, + { + "epoch": 0.47637310510486264, + "grad_norm": 0.9500495791435242, + "learning_rate": 0.0001, + "loss": 82.6425, + "step": 2950 + }, + { + "epoch": 0.47798792918996386, + "grad_norm": 0.8551210165023804, + "learning_rate": 0.0001, + "loss": 81.1971, + "step": 2960 + }, + { + "epoch": 0.4796027532750651, + "grad_norm": 0.8149814605712891, + "learning_rate": 0.0001, + "loss": 82.7919, + "step": 2970 + }, + { + "epoch": 0.48121757736016635, + "grad_norm": 0.9012249112129211, + "learning_rate": 0.0001, + "loss": 81.975, + "step": 2980 + }, + { + "epoch": 0.48283240144526757, + "grad_norm": 1.3784620761871338, + "learning_rate": 0.0001, + "loss": 81.541, + "step": 2990 + }, + { + "epoch": 0.4844472255303688, + "grad_norm": 0.7921801209449768, + "learning_rate": 0.0001, + "loss": 83.6215, + "step": 3000 + }, + { + "epoch": 0.48606204961547, + "grad_norm": 0.9051580429077148, + "learning_rate": 0.0001, + "loss": 82.4048, + "step": 3010 + }, + { + "epoch": 0.4876768737005712, + "grad_norm": 1.0717192888259888, + "learning_rate": 0.0001, + "loss": 82.4079, + "step": 3020 + }, + { + "epoch": 0.4892916977856725, + "grad_norm": 0.8971781730651855, + "learning_rate": 0.0001, + "loss": 83.3668, + "step": 3030 + }, + { + "epoch": 0.4909065218707737, + "grad_norm": 0.9635320901870728, + "learning_rate": 0.0001, + "loss": 82.8244, + "step": 3040 + }, + { + "epoch": 0.49252134595587493, + "grad_norm": 1.1009949445724487, + "learning_rate": 0.0001, + "loss": 82.2174, + "step": 3050 + }, + { + "epoch": 0.49413617004097615, + "grad_norm": 0.9805473685264587, + "learning_rate": 0.0001, + "loss": 83.7897, + "step": 3060 + }, + { + "epoch": 0.49575099412607737, + "grad_norm": 1.2107361555099487, + "learning_rate": 0.0001, + "loss": 82.5108, + "step": 3070 + }, + { + "epoch": 0.49736581821117865, + "grad_norm": 0.9171829223632812, + "learning_rate": 0.0001, + "loss": 80.9126, + "step": 3080 + }, + { + "epoch": 0.49898064229627986, + "grad_norm": 0.8093295097351074, + "learning_rate": 0.0001, + "loss": 83.8147, + "step": 3090 + }, + { + "epoch": 0.5005954663813811, + "grad_norm": 1.2166653871536255, + "learning_rate": 0.0001, + "loss": 82.0288, + "step": 3100 + }, + { + "epoch": 0.5022102904664824, + "grad_norm": 1.0471502542495728, + "learning_rate": 0.0001, + "loss": 82.0706, + "step": 3110 + }, + { + "epoch": 0.5038251145515835, + "grad_norm": 1.007645845413208, + "learning_rate": 0.0001, + "loss": 81.7387, + "step": 3120 + }, + { + "epoch": 0.5054399386366848, + "grad_norm": 0.7252638339996338, + "learning_rate": 0.0001, + "loss": 82.9107, + "step": 3130 + }, + { + "epoch": 0.507054762721786, + "grad_norm": 0.8565683364868164, + "learning_rate": 0.0001, + "loss": 80.2935, + "step": 3140 + }, + { + "epoch": 0.5086695868068872, + "grad_norm": 0.9905404448509216, + "learning_rate": 0.0001, + "loss": 82.5201, + "step": 3150 + }, + { + "epoch": 0.5102844108919885, + "grad_norm": 0.887428343296051, + "learning_rate": 0.0001, + "loss": 80.8528, + "step": 3160 + }, + { + "epoch": 0.5118992349770897, + "grad_norm": 1.062942624092102, + "learning_rate": 0.0001, + "loss": 80.7035, + "step": 3170 + }, + { + "epoch": 0.5135140590621909, + "grad_norm": 1.203599214553833, + "learning_rate": 0.0001, + "loss": 81.2521, + "step": 3180 + }, + { + "epoch": 0.5151288831472921, + "grad_norm": 0.8452383875846863, + "learning_rate": 0.0001, + "loss": 81.6079, + "step": 3190 + }, + { + "epoch": 0.5167437072323934, + "grad_norm": 1.2530587911605835, + "learning_rate": 0.0001, + "loss": 82.2148, + "step": 3200 + }, + { + "epoch": 0.5183585313174947, + "grad_norm": 0.9351627826690674, + "learning_rate": 0.0001, + "loss": 82.7857, + "step": 3210 + }, + { + "epoch": 0.5199733554025958, + "grad_norm": 1.0128077268600464, + "learning_rate": 0.0001, + "loss": 81.0532, + "step": 3220 + }, + { + "epoch": 0.5215881794876971, + "grad_norm": 0.9316917061805725, + "learning_rate": 0.0001, + "loss": 80.7443, + "step": 3230 + }, + { + "epoch": 0.5232030035727983, + "grad_norm": 1.502705693244934, + "learning_rate": 0.0001, + "loss": 80.0608, + "step": 3240 + }, + { + "epoch": 0.5248178276578995, + "grad_norm": 0.8166837692260742, + "learning_rate": 0.0001, + "loss": 81.2013, + "step": 3250 + }, + { + "epoch": 0.5264326517430008, + "grad_norm": 0.8210042119026184, + "learning_rate": 0.0001, + "loss": 80.2491, + "step": 3260 + }, + { + "epoch": 0.528047475828102, + "grad_norm": 0.7854771018028259, + "learning_rate": 0.0001, + "loss": 81.1017, + "step": 3270 + }, + { + "epoch": 0.5296622999132032, + "grad_norm": 0.8530565500259399, + "learning_rate": 0.0001, + "loss": 80.8123, + "step": 3280 + }, + { + "epoch": 0.5312771239983044, + "grad_norm": 0.9446498155593872, + "learning_rate": 0.0001, + "loss": 80.0029, + "step": 3290 + }, + { + "epoch": 0.5328919480834057, + "grad_norm": 0.9646537899971008, + "learning_rate": 0.0001, + "loss": 81.5413, + "step": 3300 + }, + { + "epoch": 0.5345067721685068, + "grad_norm": 0.9262019395828247, + "learning_rate": 0.0001, + "loss": 80.9399, + "step": 3310 + }, + { + "epoch": 0.5361215962536081, + "grad_norm": 0.8515486121177673, + "learning_rate": 0.0001, + "loss": 80.9421, + "step": 3320 + }, + { + "epoch": 0.5377364203387094, + "grad_norm": 1.679321050643921, + "learning_rate": 0.0001, + "loss": 82.4851, + "step": 3330 + }, + { + "epoch": 0.5393512444238105, + "grad_norm": 0.8289812207221985, + "learning_rate": 0.0001, + "loss": 80.5664, + "step": 3340 + }, + { + "epoch": 0.5409660685089118, + "grad_norm": 0.7463789582252502, + "learning_rate": 0.0001, + "loss": 81.5193, + "step": 3350 + }, + { + "epoch": 0.542580892594013, + "grad_norm": 0.8332446217536926, + "learning_rate": 0.0001, + "loss": 80.8091, + "step": 3360 + }, + { + "epoch": 0.5441957166791143, + "grad_norm": 0.7612847089767456, + "learning_rate": 0.0001, + "loss": 79.8893, + "step": 3370 + }, + { + "epoch": 0.5458105407642155, + "grad_norm": 1.0557948350906372, + "learning_rate": 0.0001, + "loss": 81.4305, + "step": 3380 + }, + { + "epoch": 0.5474253648493167, + "grad_norm": 0.7975788116455078, + "learning_rate": 0.0001, + "loss": 80.0105, + "step": 3390 + }, + { + "epoch": 0.549040188934418, + "grad_norm": 0.9049227237701416, + "learning_rate": 0.0001, + "loss": 81.2689, + "step": 3400 + }, + { + "epoch": 0.5506550130195191, + "grad_norm": 1.1195895671844482, + "learning_rate": 0.0001, + "loss": 80.7163, + "step": 3410 + }, + { + "epoch": 0.5522698371046204, + "grad_norm": 0.7506540417671204, + "learning_rate": 0.0001, + "loss": 79.0855, + "step": 3420 + }, + { + "epoch": 0.5538846611897217, + "grad_norm": 0.73989337682724, + "learning_rate": 0.0001, + "loss": 81.1865, + "step": 3430 + }, + { + "epoch": 0.5554994852748228, + "grad_norm": 0.8418401479721069, + "learning_rate": 0.0001, + "loss": 79.5714, + "step": 3440 + }, + { + "epoch": 0.5571143093599241, + "grad_norm": 0.9807226061820984, + "learning_rate": 0.0001, + "loss": 78.5699, + "step": 3450 + }, + { + "epoch": 0.5587291334450253, + "grad_norm": 1.2890545129776, + "learning_rate": 0.0001, + "loss": 81.0128, + "step": 3460 + }, + { + "epoch": 0.5603439575301266, + "grad_norm": 0.8593361973762512, + "learning_rate": 0.0001, + "loss": 80.1638, + "step": 3470 + }, + { + "epoch": 0.5619587816152278, + "grad_norm": 0.7766354084014893, + "learning_rate": 0.0001, + "loss": 81.5597, + "step": 3480 + }, + { + "epoch": 0.563573605700329, + "grad_norm": 0.7785626649856567, + "learning_rate": 0.0001, + "loss": 79.3862, + "step": 3490 + }, + { + "epoch": 0.5651884297854303, + "grad_norm": 0.8429279327392578, + "learning_rate": 0.0001, + "loss": 76.6294, + "step": 3500 + }, + { + "epoch": 0.5668032538705314, + "grad_norm": 0.8141765594482422, + "learning_rate": 0.0001, + "loss": 80.2584, + "step": 3510 + }, + { + "epoch": 0.5684180779556327, + "grad_norm": 0.7065490484237671, + "learning_rate": 0.0001, + "loss": 79.3626, + "step": 3520 + }, + { + "epoch": 0.570032902040734, + "grad_norm": 0.7002031803131104, + "learning_rate": 0.0001, + "loss": 79.283, + "step": 3530 + }, + { + "epoch": 0.5716477261258351, + "grad_norm": 0.8715317845344543, + "learning_rate": 0.0001, + "loss": 80.3145, + "step": 3540 + }, + { + "epoch": 0.5732625502109364, + "grad_norm": 0.9584882855415344, + "learning_rate": 0.0001, + "loss": 78.7443, + "step": 3550 + }, + { + "epoch": 0.5748773742960376, + "grad_norm": 0.7908946871757507, + "learning_rate": 0.0001, + "loss": 79.1906, + "step": 3560 + }, + { + "epoch": 0.5764921983811389, + "grad_norm": 0.8416746258735657, + "learning_rate": 0.0001, + "loss": 77.8172, + "step": 3570 + }, + { + "epoch": 0.5781070224662401, + "grad_norm": 0.823667049407959, + "learning_rate": 0.0001, + "loss": 81.864, + "step": 3580 + }, + { + "epoch": 0.5797218465513413, + "grad_norm": 1.4282617568969727, + "learning_rate": 0.0001, + "loss": 79.9052, + "step": 3590 + }, + { + "epoch": 0.5813366706364426, + "grad_norm": 0.8747813701629639, + "learning_rate": 0.0001, + "loss": 78.8234, + "step": 3600 + }, + { + "epoch": 0.5829514947215437, + "grad_norm": 0.8415511846542358, + "learning_rate": 0.0001, + "loss": 78.8171, + "step": 3610 + }, + { + "epoch": 0.584566318806645, + "grad_norm": 0.9014492034912109, + "learning_rate": 0.0001, + "loss": 80.5011, + "step": 3620 + }, + { + "epoch": 0.5861811428917463, + "grad_norm": 0.7535204887390137, + "learning_rate": 0.0001, + "loss": 79.4895, + "step": 3630 + }, + { + "epoch": 0.5877959669768474, + "grad_norm": 0.8196508884429932, + "learning_rate": 0.0001, + "loss": 79.7101, + "step": 3640 + }, + { + "epoch": 0.5894107910619487, + "grad_norm": 1.5975492000579834, + "learning_rate": 0.0001, + "loss": 79.0687, + "step": 3650 + }, + { + "epoch": 0.5910256151470499, + "grad_norm": 0.9251692295074463, + "learning_rate": 0.0001, + "loss": 80.2377, + "step": 3660 + }, + { + "epoch": 0.5926404392321512, + "grad_norm": 0.7143557071685791, + "learning_rate": 0.0001, + "loss": 78.0671, + "step": 3670 + }, + { + "epoch": 0.5942552633172524, + "grad_norm": 0.7876290678977966, + "learning_rate": 0.0001, + "loss": 79.1455, + "step": 3680 + }, + { + "epoch": 0.5958700874023536, + "grad_norm": 1.5361895561218262, + "learning_rate": 0.0001, + "loss": 78.1253, + "step": 3690 + }, + { + "epoch": 0.5974849114874549, + "grad_norm": 0.7637040019035339, + "learning_rate": 0.0001, + "loss": 77.4658, + "step": 3700 + }, + { + "epoch": 0.599099735572556, + "grad_norm": 0.8166250586509705, + "learning_rate": 0.0001, + "loss": 79.5806, + "step": 3710 + }, + { + "epoch": 0.6007145596576573, + "grad_norm": 1.2767380475997925, + "learning_rate": 0.0001, + "loss": 79.1894, + "step": 3720 + }, + { + "epoch": 0.6023293837427586, + "grad_norm": 1.3048640489578247, + "learning_rate": 0.0001, + "loss": 78.0593, + "step": 3730 + }, + { + "epoch": 0.6039442078278597, + "grad_norm": 0.7945445775985718, + "learning_rate": 0.0001, + "loss": 79.7623, + "step": 3740 + }, + { + "epoch": 0.605559031912961, + "grad_norm": 0.7366836667060852, + "learning_rate": 0.0001, + "loss": 77.9571, + "step": 3750 + }, + { + "epoch": 0.6071738559980622, + "grad_norm": 0.9126115441322327, + "learning_rate": 0.0001, + "loss": 79.9502, + "step": 3760 + }, + { + "epoch": 0.6087886800831634, + "grad_norm": 0.800355851650238, + "learning_rate": 0.0001, + "loss": 78.7244, + "step": 3770 + }, + { + "epoch": 0.6104035041682647, + "grad_norm": 0.8290980458259583, + "learning_rate": 0.0001, + "loss": 78.7724, + "step": 3780 + }, + { + "epoch": 0.6120183282533659, + "grad_norm": 0.8512316942214966, + "learning_rate": 0.0001, + "loss": 78.6277, + "step": 3790 + }, + { + "epoch": 0.6136331523384672, + "grad_norm": 0.8909353613853455, + "learning_rate": 0.0001, + "loss": 78.5956, + "step": 3800 + }, + { + "epoch": 0.6152479764235683, + "grad_norm": 0.8328800797462463, + "learning_rate": 0.0001, + "loss": 78.1323, + "step": 3810 + }, + { + "epoch": 0.6168628005086696, + "grad_norm": 0.7950423359870911, + "learning_rate": 0.0001, + "loss": 78.7529, + "step": 3820 + }, + { + "epoch": 0.6184776245937709, + "grad_norm": 0.9988545179367065, + "learning_rate": 0.0001, + "loss": 80.7727, + "step": 3830 + }, + { + "epoch": 0.620092448678872, + "grad_norm": 0.7243085503578186, + "learning_rate": 0.0001, + "loss": 76.7616, + "step": 3840 + }, + { + "epoch": 0.6217072727639733, + "grad_norm": 0.7674855589866638, + "learning_rate": 0.0001, + "loss": 79.1938, + "step": 3850 + }, + { + "epoch": 0.6233220968490745, + "grad_norm": 0.8979483842849731, + "learning_rate": 0.0001, + "loss": 79.1522, + "step": 3860 + }, + { + "epoch": 0.6249369209341757, + "grad_norm": 0.7759745717048645, + "learning_rate": 0.0001, + "loss": 77.7001, + "step": 3870 + }, + { + "epoch": 0.626551745019277, + "grad_norm": 0.9940371513366699, + "learning_rate": 0.0001, + "loss": 79.8712, + "step": 3880 + }, + { + "epoch": 0.6281665691043782, + "grad_norm": 0.8103354573249817, + "learning_rate": 0.0001, + "loss": 77.5229, + "step": 3890 + }, + { + "epoch": 0.6297813931894795, + "grad_norm": 0.9188411831855774, + "learning_rate": 0.0001, + "loss": 78.2933, + "step": 3900 + }, + { + "epoch": 0.6313962172745806, + "grad_norm": 0.823017954826355, + "learning_rate": 0.0001, + "loss": 77.8692, + "step": 3910 + }, + { + "epoch": 0.6330110413596819, + "grad_norm": 0.8833358287811279, + "learning_rate": 0.0001, + "loss": 78.4465, + "step": 3920 + }, + { + "epoch": 0.634625865444783, + "grad_norm": 0.6880685687065125, + "learning_rate": 0.0001, + "loss": 78.8881, + "step": 3930 + }, + { + "epoch": 0.6362406895298843, + "grad_norm": 0.778455913066864, + "learning_rate": 0.0001, + "loss": 76.0277, + "step": 3940 + }, + { + "epoch": 0.6378555136149856, + "grad_norm": 0.8132772445678711, + "learning_rate": 0.0001, + "loss": 79.1284, + "step": 3950 + }, + { + "epoch": 0.6394703377000868, + "grad_norm": 0.7703788876533508, + "learning_rate": 0.0001, + "loss": 76.9013, + "step": 3960 + }, + { + "epoch": 0.641085161785188, + "grad_norm": 0.7876151204109192, + "learning_rate": 0.0001, + "loss": 77.1354, + "step": 3970 + }, + { + "epoch": 0.6426999858702892, + "grad_norm": 0.8027064800262451, + "learning_rate": 0.0001, + "loss": 77.4936, + "step": 3980 + }, + { + "epoch": 0.6443148099553905, + "grad_norm": 0.8261818885803223, + "learning_rate": 0.0001, + "loss": 76.7634, + "step": 3990 + }, + { + "epoch": 0.6459296340404918, + "grad_norm": 0.7024852633476257, + "learning_rate": 0.0001, + "loss": 79.951, + "step": 4000 + }, + { + "epoch": 0.6475444581255929, + "grad_norm": 1.015307068824768, + "learning_rate": 0.0001, + "loss": 76.0354, + "step": 4010 + }, + { + "epoch": 0.6491592822106942, + "grad_norm": 0.715346097946167, + "learning_rate": 0.0001, + "loss": 78.9021, + "step": 4020 + }, + { + "epoch": 0.6507741062957954, + "grad_norm": 0.8994929790496826, + "learning_rate": 0.0001, + "loss": 77.0587, + "step": 4030 + }, + { + "epoch": 0.6523889303808966, + "grad_norm": 0.7683436274528503, + "learning_rate": 0.0001, + "loss": 78.6943, + "step": 4040 + }, + { + "epoch": 0.6540037544659979, + "grad_norm": 0.7273952960968018, + "learning_rate": 0.0001, + "loss": 77.8396, + "step": 4050 + }, + { + "epoch": 0.6556185785510991, + "grad_norm": 1.0676878690719604, + "learning_rate": 0.0001, + "loss": 79.3449, + "step": 4060 + }, + { + "epoch": 0.6572334026362003, + "grad_norm": 0.6916168928146362, + "learning_rate": 0.0001, + "loss": 76.5462, + "step": 4070 + }, + { + "epoch": 0.6588482267213015, + "grad_norm": 0.6466719508171082, + "learning_rate": 0.0001, + "loss": 77.9354, + "step": 4080 + }, + { + "epoch": 0.6604630508064028, + "grad_norm": 0.7406665086746216, + "learning_rate": 0.0001, + "loss": 76.0135, + "step": 4090 + }, + { + "epoch": 0.662077874891504, + "grad_norm": 0.7104353904724121, + "learning_rate": 0.0001, + "loss": 78.3727, + "step": 4100 + }, + { + "epoch": 0.6636926989766052, + "grad_norm": 0.8328452110290527, + "learning_rate": 0.0001, + "loss": 77.5354, + "step": 4110 + }, + { + "epoch": 0.6653075230617065, + "grad_norm": 0.7012260556221008, + "learning_rate": 0.0001, + "loss": 78.7406, + "step": 4120 + }, + { + "epoch": 0.6669223471468076, + "grad_norm": 1.0436632633209229, + "learning_rate": 0.0001, + "loss": 78.0063, + "step": 4130 + }, + { + "epoch": 0.6685371712319089, + "grad_norm": 0.8471246361732483, + "learning_rate": 0.0001, + "loss": 77.8871, + "step": 4140 + }, + { + "epoch": 0.6701519953170102, + "grad_norm": 0.7378994822502136, + "learning_rate": 0.0001, + "loss": 76.9109, + "step": 4150 + }, + { + "epoch": 0.6717668194021114, + "grad_norm": 0.7778745889663696, + "learning_rate": 0.0001, + "loss": 78.3374, + "step": 4160 + }, + { + "epoch": 0.6733816434872126, + "grad_norm": 0.7841216325759888, + "learning_rate": 0.0001, + "loss": 78.1657, + "step": 4170 + }, + { + "epoch": 0.6749964675723138, + "grad_norm": 1.0027995109558105, + "learning_rate": 0.0001, + "loss": 76.6232, + "step": 4180 + }, + { + "epoch": 0.6766112916574151, + "grad_norm": 1.0034840106964111, + "learning_rate": 0.0001, + "loss": 79.5785, + "step": 4190 + }, + { + "epoch": 0.6782261157425163, + "grad_norm": 0.8399350643157959, + "learning_rate": 0.0001, + "loss": 76.8101, + "step": 4200 + }, + { + "epoch": 0.6798409398276175, + "grad_norm": 0.8864424824714661, + "learning_rate": 0.0001, + "loss": 77.8671, + "step": 4210 + }, + { + "epoch": 0.6814557639127188, + "grad_norm": 0.8322188258171082, + "learning_rate": 0.0001, + "loss": 77.6597, + "step": 4220 + }, + { + "epoch": 0.68307058799782, + "grad_norm": 0.8962367177009583, + "learning_rate": 0.0001, + "loss": 77.7324, + "step": 4230 + }, + { + "epoch": 0.6846854120829212, + "grad_norm": 0.9727205038070679, + "learning_rate": 0.0001, + "loss": 77.8531, + "step": 4240 + }, + { + "epoch": 0.6863002361680225, + "grad_norm": 0.956584095954895, + "learning_rate": 0.0001, + "loss": 78.709, + "step": 4250 + }, + { + "epoch": 0.6879150602531237, + "grad_norm": 0.8531600832939148, + "learning_rate": 0.0001, + "loss": 78.7755, + "step": 4260 + }, + { + "epoch": 0.6895298843382249, + "grad_norm": 0.8059844374656677, + "learning_rate": 0.0001, + "loss": 77.1636, + "step": 4270 + }, + { + "epoch": 0.6911447084233261, + "grad_norm": 0.8373456597328186, + "learning_rate": 0.0001, + "loss": 78.6561, + "step": 4280 + }, + { + "epoch": 0.6927595325084274, + "grad_norm": 0.7819933891296387, + "learning_rate": 0.0001, + "loss": 77.6061, + "step": 4290 + }, + { + "epoch": 0.6943743565935286, + "grad_norm": 0.8829817771911621, + "learning_rate": 0.0001, + "loss": 77.3387, + "step": 4300 + }, + { + "epoch": 0.6959891806786298, + "grad_norm": 0.791772723197937, + "learning_rate": 0.0001, + "loss": 77.4368, + "step": 4310 + }, + { + "epoch": 0.6976040047637311, + "grad_norm": 0.9949231147766113, + "learning_rate": 0.0001, + "loss": 73.4359, + "step": 4320 + }, + { + "epoch": 0.6992188288488322, + "grad_norm": 0.8676188588142395, + "learning_rate": 0.0001, + "loss": 78.4836, + "step": 4330 + }, + { + "epoch": 0.7008336529339335, + "grad_norm": 1.1704007387161255, + "learning_rate": 0.0001, + "loss": 76.5091, + "step": 4340 + }, + { + "epoch": 0.7024484770190348, + "grad_norm": 0.8877903819084167, + "learning_rate": 0.0001, + "loss": 78.021, + "step": 4350 + }, + { + "epoch": 0.704063301104136, + "grad_norm": 1.381914496421814, + "learning_rate": 0.0001, + "loss": 77.8544, + "step": 4360 + }, + { + "epoch": 0.7056781251892372, + "grad_norm": 0.835023045539856, + "learning_rate": 0.0001, + "loss": 76.1552, + "step": 4370 + }, + { + "epoch": 0.7072929492743384, + "grad_norm": 0.9814149141311646, + "learning_rate": 0.0001, + "loss": 76.7646, + "step": 4380 + }, + { + "epoch": 0.7089077733594397, + "grad_norm": 0.8012816309928894, + "learning_rate": 0.0001, + "loss": 75.794, + "step": 4390 + }, + { + "epoch": 0.7105225974445409, + "grad_norm": 0.8340234160423279, + "learning_rate": 0.0001, + "loss": 77.4417, + "step": 4400 + }, + { + "epoch": 0.7121374215296421, + "grad_norm": 0.7257950305938721, + "learning_rate": 0.0001, + "loss": 77.1807, + "step": 4410 + }, + { + "epoch": 0.7137522456147434, + "grad_norm": 0.7342708706855774, + "learning_rate": 0.0001, + "loss": 75.5091, + "step": 4420 + }, + { + "epoch": 0.7153670696998445, + "grad_norm": 0.8298904895782471, + "learning_rate": 0.0001, + "loss": 76.6651, + "step": 4430 + }, + { + "epoch": 0.7169818937849458, + "grad_norm": 0.8564396500587463, + "learning_rate": 0.0001, + "loss": 76.0055, + "step": 4440 + }, + { + "epoch": 0.7185967178700471, + "grad_norm": 0.9145296812057495, + "learning_rate": 0.0001, + "loss": 76.0369, + "step": 4450 + }, + { + "epoch": 0.7202115419551482, + "grad_norm": 0.8074191212654114, + "learning_rate": 0.0001, + "loss": 77.4436, + "step": 4460 + }, + { + "epoch": 0.7218263660402495, + "grad_norm": 0.8212682008743286, + "learning_rate": 0.0001, + "loss": 76.3129, + "step": 4470 + }, + { + "epoch": 0.7234411901253507, + "grad_norm": 0.751323938369751, + "learning_rate": 0.0001, + "loss": 77.603, + "step": 4480 + }, + { + "epoch": 0.725056014210452, + "grad_norm": 0.9664170145988464, + "learning_rate": 0.0001, + "loss": 77.481, + "step": 4490 + }, + { + "epoch": 0.7266708382955531, + "grad_norm": 0.7369455695152283, + "learning_rate": 0.0001, + "loss": 76.3238, + "step": 4500 + }, + { + "epoch": 0.7282856623806544, + "grad_norm": 0.8448513746261597, + "learning_rate": 0.0001, + "loss": 76.5679, + "step": 4510 + }, + { + "epoch": 0.7299004864657557, + "grad_norm": 0.8605676293373108, + "learning_rate": 0.0001, + "loss": 75.7667, + "step": 4520 + }, + { + "epoch": 0.7315153105508568, + "grad_norm": 0.8034664392471313, + "learning_rate": 0.0001, + "loss": 76.2163, + "step": 4530 + }, + { + "epoch": 0.7331301346359581, + "grad_norm": 0.7846826910972595, + "learning_rate": 0.0001, + "loss": 73.5979, + "step": 4540 + }, + { + "epoch": 0.7347449587210593, + "grad_norm": 0.7422606348991394, + "learning_rate": 0.0001, + "loss": 76.2334, + "step": 4550 + }, + { + "epoch": 0.7363597828061605, + "grad_norm": 1.4478483200073242, + "learning_rate": 0.0001, + "loss": 77.6168, + "step": 4560 + }, + { + "epoch": 0.7379746068912618, + "grad_norm": 2.0957767963409424, + "learning_rate": 0.0001, + "loss": 76.3199, + "step": 4570 + }, + { + "epoch": 0.739589430976363, + "grad_norm": 0.6757177114486694, + "learning_rate": 0.0001, + "loss": 77.2923, + "step": 4580 + }, + { + "epoch": 0.7412042550614643, + "grad_norm": 0.9281920790672302, + "learning_rate": 0.0001, + "loss": 76.9963, + "step": 4590 + }, + { + "epoch": 0.7428190791465654, + "grad_norm": 0.8702669143676758, + "learning_rate": 0.0001, + "loss": 76.2553, + "step": 4600 + }, + { + "epoch": 0.7444339032316667, + "grad_norm": 0.7976512312889099, + "learning_rate": 0.0001, + "loss": 74.7622, + "step": 4610 + }, + { + "epoch": 0.746048727316768, + "grad_norm": 0.9116189479827881, + "learning_rate": 0.0001, + "loss": 76.3198, + "step": 4620 + }, + { + "epoch": 0.7476635514018691, + "grad_norm": 0.6239303946495056, + "learning_rate": 0.0001, + "loss": 75.7678, + "step": 4630 + }, + { + "epoch": 0.7492783754869704, + "grad_norm": 0.8380976915359497, + "learning_rate": 0.0001, + "loss": 76.8305, + "step": 4640 + }, + { + "epoch": 0.7508931995720716, + "grad_norm": 0.7394141554832458, + "learning_rate": 0.0001, + "loss": 76.5033, + "step": 4650 + }, + { + "epoch": 0.7525080236571728, + "grad_norm": 1.0694036483764648, + "learning_rate": 0.0001, + "loss": 75.0084, + "step": 4660 + }, + { + "epoch": 0.7541228477422741, + "grad_norm": 0.7530115246772766, + "learning_rate": 0.0001, + "loss": 74.4977, + "step": 4670 + }, + { + "epoch": 0.7557376718273753, + "grad_norm": 1.9561909437179565, + "learning_rate": 0.0001, + "loss": 74.7077, + "step": 4680 + }, + { + "epoch": 0.7573524959124766, + "grad_norm": 0.625341534614563, + "learning_rate": 0.0001, + "loss": 76.1238, + "step": 4690 + }, + { + "epoch": 0.7589673199975777, + "grad_norm": 0.9790500998497009, + "learning_rate": 0.0001, + "loss": 76.3997, + "step": 4700 + }, + { + "epoch": 0.760582144082679, + "grad_norm": 0.7087915539741516, + "learning_rate": 0.0001, + "loss": 75.6893, + "step": 4710 + }, + { + "epoch": 0.7621969681677803, + "grad_norm": 1.0995875597000122, + "learning_rate": 0.0001, + "loss": 74.8588, + "step": 4720 + }, + { + "epoch": 0.7638117922528814, + "grad_norm": 1.0309056043624878, + "learning_rate": 0.0001, + "loss": 73.5934, + "step": 4730 + }, + { + "epoch": 0.7654266163379827, + "grad_norm": 1.0132921934127808, + "learning_rate": 0.0001, + "loss": 73.9868, + "step": 4740 + }, + { + "epoch": 0.7670414404230839, + "grad_norm": 0.7138039469718933, + "learning_rate": 0.0001, + "loss": 76.4459, + "step": 4750 + }, + { + "epoch": 0.7686562645081851, + "grad_norm": 0.7055657505989075, + "learning_rate": 0.0001, + "loss": 75.1068, + "step": 4760 + }, + { + "epoch": 0.7702710885932864, + "grad_norm": 0.7398490905761719, + "learning_rate": 0.0001, + "loss": 75.3697, + "step": 4770 + }, + { + "epoch": 0.7718859126783876, + "grad_norm": 0.6836295127868652, + "learning_rate": 0.0001, + "loss": 77.4988, + "step": 4780 + }, + { + "epoch": 0.7735007367634888, + "grad_norm": 0.7552574276924133, + "learning_rate": 0.0001, + "loss": 75.149, + "step": 4790 + }, + { + "epoch": 0.77511556084859, + "grad_norm": 0.7930851578712463, + "learning_rate": 0.0001, + "loss": 73.8661, + "step": 4800 + }, + { + "epoch": 0.7767303849336913, + "grad_norm": 0.7549886107444763, + "learning_rate": 0.0001, + "loss": 75.042, + "step": 4810 + }, + { + "epoch": 0.7783452090187926, + "grad_norm": 0.9932214617729187, + "learning_rate": 0.0001, + "loss": 75.6619, + "step": 4820 + }, + { + "epoch": 0.7799600331038937, + "grad_norm": 0.7685471773147583, + "learning_rate": 0.0001, + "loss": 75.8404, + "step": 4830 + }, + { + "epoch": 0.781574857188995, + "grad_norm": 0.828737199306488, + "learning_rate": 0.0001, + "loss": 75.5839, + "step": 4840 + }, + { + "epoch": 0.7831896812740962, + "grad_norm": 0.8575420379638672, + "learning_rate": 0.0001, + "loss": 75.7889, + "step": 4850 + }, + { + "epoch": 0.7848045053591974, + "grad_norm": 0.7053834199905396, + "learning_rate": 0.0001, + "loss": 75.9372, + "step": 4860 + }, + { + "epoch": 0.7864193294442987, + "grad_norm": 0.7861505150794983, + "learning_rate": 0.0001, + "loss": 76.1532, + "step": 4870 + }, + { + "epoch": 0.7880341535293999, + "grad_norm": 0.8357386589050293, + "learning_rate": 0.0001, + "loss": 75.8536, + "step": 4880 + }, + { + "epoch": 0.7896489776145011, + "grad_norm": 3.448834180831909, + "learning_rate": 0.0001, + "loss": 74.1794, + "step": 4890 + }, + { + "epoch": 0.7912638016996023, + "grad_norm": 0.813522458076477, + "learning_rate": 0.0001, + "loss": 74.5202, + "step": 4900 + }, + { + "epoch": 0.7928786257847036, + "grad_norm": 0.840230405330658, + "learning_rate": 0.0001, + "loss": 74.6634, + "step": 4910 + }, + { + "epoch": 0.7944934498698049, + "grad_norm": 0.7855745553970337, + "learning_rate": 0.0001, + "loss": 73.5369, + "step": 4920 + }, + { + "epoch": 0.796108273954906, + "grad_norm": 0.7708212733268738, + "learning_rate": 0.0001, + "loss": 74.6225, + "step": 4930 + }, + { + "epoch": 0.7977230980400073, + "grad_norm": 0.7140830159187317, + "learning_rate": 0.0001, + "loss": 75.0585, + "step": 4940 + }, + { + "epoch": 0.7993379221251085, + "grad_norm": 0.7960936427116394, + "learning_rate": 0.0001, + "loss": 74.0574, + "step": 4950 + }, + { + "epoch": 0.8009527462102097, + "grad_norm": 0.7269165515899658, + "learning_rate": 0.0001, + "loss": 74.0379, + "step": 4960 + }, + { + "epoch": 0.802567570295311, + "grad_norm": 0.8505013585090637, + "learning_rate": 0.0001, + "loss": 74.7448, + "step": 4970 + }, + { + "epoch": 0.8041823943804122, + "grad_norm": 0.8935654759407043, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 4980 + }, + { + "epoch": 0.8057972184655134, + "grad_norm": 0.9447595477104187, + "learning_rate": 0.0001, + "loss": 75.9674, + "step": 4990 + }, + { + "epoch": 0.8074120425506146, + "grad_norm": 1.6739715337753296, + "learning_rate": 0.0001, + "loss": 73.9035, + "step": 5000 + }, + { + "epoch": 0.8090268666357159, + "grad_norm": 0.861902117729187, + "learning_rate": 0.0001, + "loss": 74.906, + "step": 5010 + }, + { + "epoch": 0.8106416907208172, + "grad_norm": 0.8328655362129211, + "learning_rate": 0.0001, + "loss": 74.1493, + "step": 5020 + }, + { + "epoch": 0.8122565148059183, + "grad_norm": 0.8193135261535645, + "learning_rate": 0.0001, + "loss": 74.6614, + "step": 5030 + }, + { + "epoch": 0.8138713388910196, + "grad_norm": 0.6954565048217773, + "learning_rate": 0.0001, + "loss": 73.7366, + "step": 5040 + }, + { + "epoch": 0.8154861629761208, + "grad_norm": 0.7381289005279541, + "learning_rate": 0.0001, + "loss": 73.5086, + "step": 5050 + }, + { + "epoch": 0.817100987061222, + "grad_norm": 1.090443730354309, + "learning_rate": 0.0001, + "loss": 75.0695, + "step": 5060 + }, + { + "epoch": 0.8187158111463233, + "grad_norm": 1.2544814348220825, + "learning_rate": 0.0001, + "loss": 72.89, + "step": 5070 + }, + { + "epoch": 0.8203306352314245, + "grad_norm": 0.632146418094635, + "learning_rate": 0.0001, + "loss": 74.8604, + "step": 5080 + }, + { + "epoch": 0.8219454593165257, + "grad_norm": 1.1038259267807007, + "learning_rate": 0.0001, + "loss": 74.7354, + "step": 5090 + }, + { + "epoch": 0.8235602834016269, + "grad_norm": 0.6886508464813232, + "learning_rate": 0.0001, + "loss": 75.4243, + "step": 5100 + }, + { + "epoch": 0.8251751074867282, + "grad_norm": 0.8367568850517273, + "learning_rate": 0.0001, + "loss": 75.2732, + "step": 5110 + }, + { + "epoch": 0.8267899315718293, + "grad_norm": 0.9302715063095093, + "learning_rate": 0.0001, + "loss": 75.7924, + "step": 5120 + }, + { + "epoch": 0.8284047556569306, + "grad_norm": 0.6871219873428345, + "learning_rate": 0.0001, + "loss": 74.2905, + "step": 5130 + }, + { + "epoch": 0.8300195797420319, + "grad_norm": 0.7319445013999939, + "learning_rate": 0.0001, + "loss": 73.7774, + "step": 5140 + }, + { + "epoch": 0.831634403827133, + "grad_norm": 0.7184522151947021, + "learning_rate": 0.0001, + "loss": 73.0554, + "step": 5150 + }, + { + "epoch": 0.8332492279122343, + "grad_norm": 0.8462343811988831, + "learning_rate": 0.0001, + "loss": 73.7615, + "step": 5160 + }, + { + "epoch": 0.8348640519973355, + "grad_norm": 0.9436241984367371, + "learning_rate": 0.0001, + "loss": 73.9464, + "step": 5170 + }, + { + "epoch": 0.8364788760824368, + "grad_norm": 0.9079630374908447, + "learning_rate": 0.0001, + "loss": 73.2727, + "step": 5180 + }, + { + "epoch": 0.838093700167538, + "grad_norm": 0.8280741572380066, + "learning_rate": 0.0001, + "loss": 73.9751, + "step": 5190 + }, + { + "epoch": 0.8397085242526392, + "grad_norm": 0.6475157141685486, + "learning_rate": 0.0001, + "loss": 74.7105, + "step": 5200 + }, + { + "epoch": 0.8413233483377405, + "grad_norm": 0.7090312838554382, + "learning_rate": 0.0001, + "loss": 73.6489, + "step": 5210 + }, + { + "epoch": 0.8429381724228416, + "grad_norm": 1.20967698097229, + "learning_rate": 0.0001, + "loss": 75.1335, + "step": 5220 + }, + { + "epoch": 0.8445529965079429, + "grad_norm": 2.053532600402832, + "learning_rate": 0.0001, + "loss": 75.3111, + "step": 5230 + }, + { + "epoch": 0.8461678205930442, + "grad_norm": 0.7563302516937256, + "learning_rate": 0.0001, + "loss": 72.9935, + "step": 5240 + }, + { + "epoch": 0.8477826446781453, + "grad_norm": 0.7801489233970642, + "learning_rate": 0.0001, + "loss": 74.7856, + "step": 5250 + }, + { + "epoch": 0.8493974687632466, + "grad_norm": 0.737870454788208, + "learning_rate": 0.0001, + "loss": 75.6281, + "step": 5260 + }, + { + "epoch": 0.8510122928483478, + "grad_norm": 0.704352617263794, + "learning_rate": 0.0001, + "loss": 75.4598, + "step": 5270 + }, + { + "epoch": 0.8526271169334491, + "grad_norm": 0.6452057361602783, + "learning_rate": 0.0001, + "loss": 74.0341, + "step": 5280 + }, + { + "epoch": 0.8542419410185503, + "grad_norm": 0.6863471865653992, + "learning_rate": 0.0001, + "loss": 74.3727, + "step": 5290 + }, + { + "epoch": 0.8558567651036515, + "grad_norm": 9.756170272827148, + "learning_rate": 0.0001, + "loss": 75.5231, + "step": 5300 + }, + { + "epoch": 0.8574715891887528, + "grad_norm": 0.8624544739723206, + "learning_rate": 0.0001, + "loss": 74.7821, + "step": 5310 + }, + { + "epoch": 0.8590864132738539, + "grad_norm": 0.6637155413627625, + "learning_rate": 0.0001, + "loss": 74.1153, + "step": 5320 + }, + { + "epoch": 0.8607012373589552, + "grad_norm": 1.0764445066452026, + "learning_rate": 0.0001, + "loss": 73.7208, + "step": 5330 + }, + { + "epoch": 0.8623160614440565, + "grad_norm": 0.8567569255828857, + "learning_rate": 0.0001, + "loss": 73.1399, + "step": 5340 + }, + { + "epoch": 0.8639308855291576, + "grad_norm": 0.7681576013565063, + "learning_rate": 0.0001, + "loss": 73.7577, + "step": 5350 + }, + { + "epoch": 0.8655457096142589, + "grad_norm": 0.9336384534835815, + "learning_rate": 0.0001, + "loss": 74.1022, + "step": 5360 + }, + { + "epoch": 0.8671605336993601, + "grad_norm": 0.8603724837303162, + "learning_rate": 0.0001, + "loss": 75.9579, + "step": 5370 + }, + { + "epoch": 0.8687753577844614, + "grad_norm": 0.77194744348526, + "learning_rate": 0.0001, + "loss": 72.4799, + "step": 5380 + }, + { + "epoch": 0.8703901818695626, + "grad_norm": 0.7480376958847046, + "learning_rate": 0.0001, + "loss": 76.1257, + "step": 5390 + }, + { + "epoch": 0.8720050059546638, + "grad_norm": 0.7841023206710815, + "learning_rate": 0.0001, + "loss": 73.1512, + "step": 5400 + }, + { + "epoch": 0.8736198300397651, + "grad_norm": 0.8741798996925354, + "learning_rate": 0.0001, + "loss": 74.2151, + "step": 5410 + }, + { + "epoch": 0.8752346541248662, + "grad_norm": 0.9762082099914551, + "learning_rate": 0.0001, + "loss": 76.7138, + "step": 5420 + }, + { + "epoch": 0.8768494782099675, + "grad_norm": 0.7670348286628723, + "learning_rate": 0.0001, + "loss": 72.9567, + "step": 5430 + }, + { + "epoch": 0.8784643022950688, + "grad_norm": 0.7449469566345215, + "learning_rate": 0.0001, + "loss": 74.8841, + "step": 5440 + }, + { + "epoch": 0.8800791263801699, + "grad_norm": 0.6984641551971436, + "learning_rate": 0.0001, + "loss": 73.8906, + "step": 5450 + }, + { + "epoch": 0.8816939504652712, + "grad_norm": 0.7854361534118652, + "learning_rate": 0.0001, + "loss": 74.626, + "step": 5460 + }, + { + "epoch": 0.8833087745503724, + "grad_norm": 0.9584240317344666, + "learning_rate": 0.0001, + "loss": 74.7048, + "step": 5470 + }, + { + "epoch": 0.8849235986354737, + "grad_norm": 0.6040284037590027, + "learning_rate": 0.0001, + "loss": 74.7194, + "step": 5480 + }, + { + "epoch": 0.8865384227205749, + "grad_norm": 0.7756215929985046, + "learning_rate": 0.0001, + "loss": 73.8178, + "step": 5490 + }, + { + "epoch": 0.8881532468056761, + "grad_norm": 0.9070586562156677, + "learning_rate": 0.0001, + "loss": 72.8309, + "step": 5500 + }, + { + "epoch": 0.8897680708907774, + "grad_norm": 0.8464986681938171, + "learning_rate": 0.0001, + "loss": 73.8847, + "step": 5510 + }, + { + "epoch": 0.8913828949758785, + "grad_norm": 0.7968605160713196, + "learning_rate": 0.0001, + "loss": 73.1691, + "step": 5520 + }, + { + "epoch": 0.8929977190609798, + "grad_norm": 0.9934583306312561, + "learning_rate": 0.0001, + "loss": 76.2662, + "step": 5530 + }, + { + "epoch": 0.8946125431460811, + "grad_norm": 0.7272152900695801, + "learning_rate": 0.0001, + "loss": 75.5295, + "step": 5540 + }, + { + "epoch": 0.8962273672311822, + "grad_norm": 0.857345461845398, + "learning_rate": 0.0001, + "loss": 75.3703, + "step": 5550 + }, + { + "epoch": 0.8978421913162835, + "grad_norm": 0.713918924331665, + "learning_rate": 0.0001, + "loss": 73.4563, + "step": 5560 + }, + { + "epoch": 0.8994570154013847, + "grad_norm": 0.759233832359314, + "learning_rate": 0.0001, + "loss": 73.2744, + "step": 5570 + }, + { + "epoch": 0.901071839486486, + "grad_norm": 0.7085754871368408, + "learning_rate": 0.0001, + "loss": 74.7792, + "step": 5580 + }, + { + "epoch": 0.9026866635715872, + "grad_norm": 0.7370456457138062, + "learning_rate": 0.0001, + "loss": 74.4216, + "step": 5590 + }, + { + "epoch": 0.9043014876566884, + "grad_norm": 0.779746949672699, + "learning_rate": 0.0001, + "loss": 75.9822, + "step": 5600 + }, + { + "epoch": 0.9059163117417897, + "grad_norm": 0.7915763854980469, + "learning_rate": 0.0001, + "loss": 74.3706, + "step": 5610 + }, + { + "epoch": 0.9075311358268908, + "grad_norm": 0.7886125445365906, + "learning_rate": 0.0001, + "loss": 72.9453, + "step": 5620 + }, + { + "epoch": 0.9091459599119921, + "grad_norm": 0.7804751992225647, + "learning_rate": 0.0001, + "loss": 73.514, + "step": 5630 + }, + { + "epoch": 0.9107607839970934, + "grad_norm": 0.7168806195259094, + "learning_rate": 0.0001, + "loss": 71.4159, + "step": 5640 + }, + { + "epoch": 0.9123756080821945, + "grad_norm": 0.7553626894950867, + "learning_rate": 0.0001, + "loss": 74.4353, + "step": 5650 + }, + { + "epoch": 0.9139904321672958, + "grad_norm": 0.6688916683197021, + "learning_rate": 0.0001, + "loss": 74.3786, + "step": 5660 + }, + { + "epoch": 0.915605256252397, + "grad_norm": 0.7425532341003418, + "learning_rate": 0.0001, + "loss": 73.3355, + "step": 5670 + }, + { + "epoch": 0.9172200803374982, + "grad_norm": 0.6473882794380188, + "learning_rate": 0.0001, + "loss": 74.2315, + "step": 5680 + }, + { + "epoch": 0.9188349044225994, + "grad_norm": 0.7222439050674438, + "learning_rate": 0.0001, + "loss": 74.2885, + "step": 5690 + }, + { + "epoch": 0.9204497285077007, + "grad_norm": 1.5716948509216309, + "learning_rate": 0.0001, + "loss": 74.6739, + "step": 5700 + }, + { + "epoch": 0.922064552592802, + "grad_norm": 0.8077759742736816, + "learning_rate": 0.0001, + "loss": 72.6617, + "step": 5710 + }, + { + "epoch": 0.9236793766779031, + "grad_norm": 0.6948631405830383, + "learning_rate": 0.0001, + "loss": 73.6247, + "step": 5720 + }, + { + "epoch": 0.9252942007630044, + "grad_norm": 0.8823757767677307, + "learning_rate": 0.0001, + "loss": 73.2536, + "step": 5730 + }, + { + "epoch": 0.9269090248481056, + "grad_norm": 0.7066764235496521, + "learning_rate": 0.0001, + "loss": 74.1748, + "step": 5740 + }, + { + "epoch": 0.9285238489332068, + "grad_norm": 0.6979793906211853, + "learning_rate": 0.0001, + "loss": 72.3189, + "step": 5750 + }, + { + "epoch": 0.9301386730183081, + "grad_norm": 0.6961462497711182, + "learning_rate": 0.0001, + "loss": 74.6196, + "step": 5760 + }, + { + "epoch": 0.9317534971034093, + "grad_norm": 0.7797656655311584, + "learning_rate": 0.0001, + "loss": 74.5472, + "step": 5770 + }, + { + "epoch": 0.9333683211885105, + "grad_norm": 1.0671838521957397, + "learning_rate": 0.0001, + "loss": 73.241, + "step": 5780 + }, + { + "epoch": 0.9349831452736117, + "grad_norm": 0.7566089034080505, + "learning_rate": 0.0001, + "loss": 73.6784, + "step": 5790 + }, + { + "epoch": 0.936597969358713, + "grad_norm": 0.8563273549079895, + "learning_rate": 0.0001, + "loss": 73.6871, + "step": 5800 + }, + { + "epoch": 0.9382127934438143, + "grad_norm": 0.7885987162590027, + "learning_rate": 0.0001, + "loss": 73.4447, + "step": 5810 + }, + { + "epoch": 0.9398276175289154, + "grad_norm": 0.8751112818717957, + "learning_rate": 0.0001, + "loss": 74.0531, + "step": 5820 + }, + { + "epoch": 0.9414424416140167, + "grad_norm": 0.6663336157798767, + "learning_rate": 0.0001, + "loss": 73.0557, + "step": 5830 + }, + { + "epoch": 0.9430572656991179, + "grad_norm": 0.7993099689483643, + "learning_rate": 0.0001, + "loss": 74.4467, + "step": 5840 + }, + { + "epoch": 0.9446720897842191, + "grad_norm": 0.6987188458442688, + "learning_rate": 0.0001, + "loss": 74.0327, + "step": 5850 + }, + { + "epoch": 0.9462869138693204, + "grad_norm": 0.8458583354949951, + "learning_rate": 0.0001, + "loss": 73.1757, + "step": 5860 + }, + { + "epoch": 0.9479017379544216, + "grad_norm": 0.6372436881065369, + "learning_rate": 0.0001, + "loss": 73.1067, + "step": 5870 + }, + { + "epoch": 0.9495165620395228, + "grad_norm": 0.7447753548622131, + "learning_rate": 0.0001, + "loss": 75.5461, + "step": 5880 + }, + { + "epoch": 0.951131386124624, + "grad_norm": 0.6633971929550171, + "learning_rate": 0.0001, + "loss": 72.9966, + "step": 5890 + }, + { + "epoch": 0.9527462102097253, + "grad_norm": 0.8576784729957581, + "learning_rate": 0.0001, + "loss": 71.5712, + "step": 5900 + }, + { + "epoch": 0.9543610342948265, + "grad_norm": 1.4696853160858154, + "learning_rate": 0.0001, + "loss": 74.7893, + "step": 5910 + }, + { + "epoch": 0.9559758583799277, + "grad_norm": 0.611210286617279, + "learning_rate": 0.0001, + "loss": 74.3426, + "step": 5920 + }, + { + "epoch": 0.957590682465029, + "grad_norm": 0.5947410464286804, + "learning_rate": 0.0001, + "loss": 71.2563, + "step": 5930 + }, + { + "epoch": 0.9592055065501301, + "grad_norm": 0.7169992923736572, + "learning_rate": 0.0001, + "loss": 71.6049, + "step": 5940 + }, + { + "epoch": 0.9608203306352314, + "grad_norm": 0.6773818731307983, + "learning_rate": 0.0001, + "loss": 75.0328, + "step": 5950 + }, + { + "epoch": 0.9624351547203327, + "grad_norm": 0.6714807748794556, + "learning_rate": 0.0001, + "loss": 71.5105, + "step": 5960 + }, + { + "epoch": 0.9640499788054339, + "grad_norm": 1.0064337253570557, + "learning_rate": 0.0001, + "loss": 75.8887, + "step": 5970 + }, + { + "epoch": 0.9656648028905351, + "grad_norm": 0.9802227020263672, + "learning_rate": 0.0001, + "loss": 73.5655, + "step": 5980 + }, + { + "epoch": 0.9672796269756363, + "grad_norm": 0.7097306251525879, + "learning_rate": 0.0001, + "loss": 73.298, + "step": 5990 + }, + { + "epoch": 0.9688944510607376, + "grad_norm": 0.6680361032485962, + "learning_rate": 0.0001, + "loss": 73.8873, + "step": 6000 + }, + { + "epoch": 0.9705092751458388, + "grad_norm": 1.1302506923675537, + "learning_rate": 0.0001, + "loss": 73.3328, + "step": 6010 + }, + { + "epoch": 0.97212409923094, + "grad_norm": 0.7163954377174377, + "learning_rate": 0.0001, + "loss": 70.5955, + "step": 6020 + }, + { + "epoch": 0.9737389233160413, + "grad_norm": 0.8182240128517151, + "learning_rate": 0.0001, + "loss": 72.6397, + "step": 6030 + }, + { + "epoch": 0.9753537474011424, + "grad_norm": 0.6752052903175354, + "learning_rate": 0.0001, + "loss": 73.1188, + "step": 6040 + }, + { + "epoch": 0.9769685714862437, + "grad_norm": 0.8829680681228638, + "learning_rate": 0.0001, + "loss": 75.7346, + "step": 6050 + }, + { + "epoch": 0.978583395571345, + "grad_norm": 0.9254767894744873, + "learning_rate": 0.0001, + "loss": 73.6221, + "step": 6060 + }, + { + "epoch": 0.9801982196564462, + "grad_norm": 0.7527495622634888, + "learning_rate": 0.0001, + "loss": 73.948, + "step": 6070 + }, + { + "epoch": 0.9818130437415474, + "grad_norm": 1.0135066509246826, + "learning_rate": 0.0001, + "loss": 72.8929, + "step": 6080 + }, + { + "epoch": 0.9834278678266486, + "grad_norm": 0.7093394994735718, + "learning_rate": 0.0001, + "loss": 73.5826, + "step": 6090 + }, + { + "epoch": 0.9850426919117499, + "grad_norm": 0.6651231646537781, + "learning_rate": 0.0001, + "loss": 73.167, + "step": 6100 + }, + { + "epoch": 0.9866575159968511, + "grad_norm": 0.720552921295166, + "learning_rate": 0.0001, + "loss": 74.8985, + "step": 6110 + }, + { + "epoch": 0.9882723400819523, + "grad_norm": 1.7821632623672485, + "learning_rate": 0.0001, + "loss": 74.1414, + "step": 6120 + }, + { + "epoch": 0.9898871641670536, + "grad_norm": 0.6405150890350342, + "learning_rate": 0.0001, + "loss": 73.5989, + "step": 6130 + }, + { + "epoch": 0.9915019882521547, + "grad_norm": 0.8207482695579529, + "learning_rate": 0.0001, + "loss": 73.0977, + "step": 6140 + }, + { + "epoch": 0.993116812337256, + "grad_norm": 0.7320757508277893, + "learning_rate": 0.0001, + "loss": 72.0241, + "step": 6150 + }, + { + "epoch": 0.9947316364223573, + "grad_norm": 0.9522950649261475, + "learning_rate": 0.0001, + "loss": 73.9831, + "step": 6160 + }, + { + "epoch": 0.9963464605074585, + "grad_norm": 0.7437159419059753, + "learning_rate": 0.0001, + "loss": 72.1054, + "step": 6170 + }, + { + "epoch": 0.9979612845925597, + "grad_norm": 0.7035623788833618, + "learning_rate": 0.0001, + "loss": 73.959, + "step": 6180 + }, + { + "epoch": 0.9995761086776609, + "grad_norm": 0.7559277415275574, + "learning_rate": 0.0001, + "loss": 71.4651, + "step": 6190 + }, + { + "epoch": 1.001130376859571, + "grad_norm": 0.852342426776886, + "learning_rate": 0.0001, + "loss": 71.8844, + "step": 6200 + }, + { + "epoch": 1.002745200944672, + "grad_norm": 0.8018911480903625, + "learning_rate": 0.0001, + "loss": 74.5093, + "step": 6210 + }, + { + "epoch": 1.0043600250297733, + "grad_norm": 0.6613730192184448, + "learning_rate": 0.0001, + "loss": 71.9953, + "step": 6220 + }, + { + "epoch": 1.0059748491148746, + "grad_norm": 0.9815977215766907, + "learning_rate": 0.0001, + "loss": 70.8632, + "step": 6230 + }, + { + "epoch": 1.0075896731999758, + "grad_norm": 0.781581699848175, + "learning_rate": 0.0001, + "loss": 72.3603, + "step": 6240 + }, + { + "epoch": 1.009204497285077, + "grad_norm": 0.8079054355621338, + "learning_rate": 0.0001, + "loss": 73.6153, + "step": 6250 + }, + { + "epoch": 1.0108193213701782, + "grad_norm": 0.8356053233146667, + "learning_rate": 0.0001, + "loss": 73.1562, + "step": 6260 + }, + { + "epoch": 1.0124341454552794, + "grad_norm": 0.7107778787612915, + "learning_rate": 0.0001, + "loss": 74.5276, + "step": 6270 + }, + { + "epoch": 1.0140489695403807, + "grad_norm": 0.8601720929145813, + "learning_rate": 0.0001, + "loss": 70.103, + "step": 6280 + }, + { + "epoch": 1.015663793625482, + "grad_norm": 0.675837516784668, + "learning_rate": 0.0001, + "loss": 72.6102, + "step": 6290 + }, + { + "epoch": 1.0172786177105833, + "grad_norm": 0.7160016298294067, + "learning_rate": 0.0001, + "loss": 72.2404, + "step": 6300 + }, + { + "epoch": 1.0188934417956843, + "grad_norm": 0.7286407351493835, + "learning_rate": 0.0001, + "loss": 72.7149, + "step": 6310 + }, + { + "epoch": 1.0205082658807856, + "grad_norm": 0.8183980584144592, + "learning_rate": 0.0001, + "loss": 73.5827, + "step": 6320 + }, + { + "epoch": 1.0221230899658869, + "grad_norm": 0.7300626039505005, + "learning_rate": 0.0001, + "loss": 72.3459, + "step": 6330 + }, + { + "epoch": 1.0237379140509881, + "grad_norm": 0.7222613096237183, + "learning_rate": 0.0001, + "loss": 71.881, + "step": 6340 + }, + { + "epoch": 1.0253527381360894, + "grad_norm": 0.7324674725532532, + "learning_rate": 0.0001, + "loss": 74.4781, + "step": 6350 + }, + { + "epoch": 1.0269675622211905, + "grad_norm": 0.9173555374145508, + "learning_rate": 0.0001, + "loss": 74.3222, + "step": 6360 + }, + { + "epoch": 1.0285823863062917, + "grad_norm": 0.8404964208602905, + "learning_rate": 0.0001, + "loss": 71.8959, + "step": 6370 + }, + { + "epoch": 1.030197210391393, + "grad_norm": 0.8735223412513733, + "learning_rate": 0.0001, + "loss": 72.7251, + "step": 6380 + }, + { + "epoch": 1.0318120344764943, + "grad_norm": 0.8467233777046204, + "learning_rate": 0.0001, + "loss": 72.5218, + "step": 6390 + }, + { + "epoch": 1.0334268585615956, + "grad_norm": 0.673565149307251, + "learning_rate": 0.0001, + "loss": 70.6392, + "step": 6400 + }, + { + "epoch": 1.0350416826466966, + "grad_norm": 0.6558571457862854, + "learning_rate": 0.0001, + "loss": 73.5305, + "step": 6410 + }, + { + "epoch": 1.0366565067317979, + "grad_norm": 0.7251600623130798, + "learning_rate": 0.0001, + "loss": 72.76, + "step": 6420 + }, + { + "epoch": 1.0382713308168992, + "grad_norm": 0.7458305358886719, + "learning_rate": 0.0001, + "loss": 73.6812, + "step": 6430 + }, + { + "epoch": 1.0398861549020004, + "grad_norm": 1.493802547454834, + "learning_rate": 0.0001, + "loss": 74.2455, + "step": 6440 + }, + { + "epoch": 1.0415009789871017, + "grad_norm": 0.894632875919342, + "learning_rate": 0.0001, + "loss": 74.2072, + "step": 6450 + }, + { + "epoch": 1.0431158030722028, + "grad_norm": 0.7142218947410583, + "learning_rate": 0.0001, + "loss": 71.4293, + "step": 6460 + }, + { + "epoch": 1.044730627157304, + "grad_norm": 0.702639639377594, + "learning_rate": 0.0001, + "loss": 71.5383, + "step": 6470 + }, + { + "epoch": 1.0463454512424053, + "grad_norm": 0.9378212094306946, + "learning_rate": 0.0001, + "loss": 72.8916, + "step": 6480 + }, + { + "epoch": 1.0479602753275066, + "grad_norm": 0.8298363089561462, + "learning_rate": 0.0001, + "loss": 72.3344, + "step": 6490 + }, + { + "epoch": 1.0495750994126078, + "grad_norm": 0.6617016792297363, + "learning_rate": 0.0001, + "loss": 71.0487, + "step": 6500 + }, + { + "epoch": 1.051189923497709, + "grad_norm": 0.638520359992981, + "learning_rate": 0.0001, + "loss": 72.2583, + "step": 6510 + }, + { + "epoch": 1.0528047475828102, + "grad_norm": 0.7101534008979797, + "learning_rate": 0.0001, + "loss": 70.8697, + "step": 6520 + }, + { + "epoch": 1.0544195716679114, + "grad_norm": 0.9425339698791504, + "learning_rate": 0.0001, + "loss": 71.0641, + "step": 6530 + }, + { + "epoch": 1.0560343957530127, + "grad_norm": 0.6572297811508179, + "learning_rate": 0.0001, + "loss": 72.6265, + "step": 6540 + }, + { + "epoch": 1.057649219838114, + "grad_norm": 0.745622456073761, + "learning_rate": 0.0001, + "loss": 73.0587, + "step": 6550 + }, + { + "epoch": 1.059264043923215, + "grad_norm": 0.7667096257209778, + "learning_rate": 0.0001, + "loss": 72.1779, + "step": 6560 + }, + { + "epoch": 1.0608788680083163, + "grad_norm": 0.7322410941123962, + "learning_rate": 0.0001, + "loss": 71.5503, + "step": 6570 + }, + { + "epoch": 1.0624936920934176, + "grad_norm": 0.5739616751670837, + "learning_rate": 0.0001, + "loss": 72.3846, + "step": 6580 + }, + { + "epoch": 1.0641085161785189, + "grad_norm": 0.7149553894996643, + "learning_rate": 0.0001, + "loss": 71.5459, + "step": 6590 + }, + { + "epoch": 1.0657233402636201, + "grad_norm": 0.6634597778320312, + "learning_rate": 0.0001, + "loss": 72.2528, + "step": 6600 + }, + { + "epoch": 1.0673381643487212, + "grad_norm": 0.7208803296089172, + "learning_rate": 0.0001, + "loss": 71.1115, + "step": 6610 + }, + { + "epoch": 1.0689529884338225, + "grad_norm": 0.9845868945121765, + "learning_rate": 0.0001, + "loss": 72.6418, + "step": 6620 + }, + { + "epoch": 1.0705678125189237, + "grad_norm": 0.9322167634963989, + "learning_rate": 0.0001, + "loss": 71.937, + "step": 6630 + }, + { + "epoch": 1.072182636604025, + "grad_norm": 0.6323093771934509, + "learning_rate": 0.0001, + "loss": 71.1537, + "step": 6640 + }, + { + "epoch": 1.0737974606891263, + "grad_norm": 0.6476802229881287, + "learning_rate": 0.0001, + "loss": 73.7811, + "step": 6650 + }, + { + "epoch": 1.0754122847742273, + "grad_norm": 0.7882822751998901, + "learning_rate": 0.0001, + "loss": 71.9548, + "step": 6660 + }, + { + "epoch": 1.0770271088593286, + "grad_norm": 0.7651168704032898, + "learning_rate": 0.0001, + "loss": 73.4066, + "step": 6670 + }, + { + "epoch": 1.07864193294443, + "grad_norm": 0.7536904215812683, + "learning_rate": 0.0001, + "loss": 72.1063, + "step": 6680 + }, + { + "epoch": 1.0802567570295312, + "grad_norm": 0.7908427119255066, + "learning_rate": 0.0001, + "loss": 72.6374, + "step": 6690 + }, + { + "epoch": 1.0818715811146324, + "grad_norm": 0.7796691060066223, + "learning_rate": 0.0001, + "loss": 72.2938, + "step": 6700 + }, + { + "epoch": 1.0834864051997335, + "grad_norm": 0.8512274622917175, + "learning_rate": 0.0001, + "loss": 71.556, + "step": 6710 + }, + { + "epoch": 1.0851012292848348, + "grad_norm": 0.7285851836204529, + "learning_rate": 0.0001, + "loss": 72.406, + "step": 6720 + }, + { + "epoch": 1.086716053369936, + "grad_norm": 0.6613008975982666, + "learning_rate": 0.0001, + "loss": 70.6509, + "step": 6730 + }, + { + "epoch": 1.0883308774550373, + "grad_norm": 0.7158271074295044, + "learning_rate": 0.0001, + "loss": 71.8434, + "step": 6740 + }, + { + "epoch": 1.0899457015401384, + "grad_norm": 0.8635967373847961, + "learning_rate": 0.0001, + "loss": 71.8279, + "step": 6750 + }, + { + "epoch": 1.0915605256252396, + "grad_norm": 0.7020159959793091, + "learning_rate": 0.0001, + "loss": 72.2417, + "step": 6760 + }, + { + "epoch": 1.093175349710341, + "grad_norm": 0.6389853954315186, + "learning_rate": 0.0001, + "loss": 72.1879, + "step": 6770 + }, + { + "epoch": 1.0947901737954422, + "grad_norm": 0.6460697650909424, + "learning_rate": 0.0001, + "loss": 72.3356, + "step": 6780 + }, + { + "epoch": 1.0964049978805435, + "grad_norm": 0.7152006030082703, + "learning_rate": 0.0001, + "loss": 71.3725, + "step": 6790 + }, + { + "epoch": 1.0980198219656445, + "grad_norm": 1.0050997734069824, + "learning_rate": 0.0001, + "loss": 72.5452, + "step": 6800 + }, + { + "epoch": 1.0996346460507458, + "grad_norm": 0.6967180371284485, + "learning_rate": 0.0001, + "loss": 72.0832, + "step": 6810 + }, + { + "epoch": 1.101249470135847, + "grad_norm": 0.7472837567329407, + "learning_rate": 0.0001, + "loss": 72.8122, + "step": 6820 + }, + { + "epoch": 1.1028642942209483, + "grad_norm": 0.8198382258415222, + "learning_rate": 0.0001, + "loss": 72.5116, + "step": 6830 + }, + { + "epoch": 1.1044791183060496, + "grad_norm": 0.8112207651138306, + "learning_rate": 0.0001, + "loss": 72.4218, + "step": 6840 + }, + { + "epoch": 1.1060939423911507, + "grad_norm": 0.8412114977836609, + "learning_rate": 0.0001, + "loss": 71.0198, + "step": 6850 + }, + { + "epoch": 1.107708766476252, + "grad_norm": 0.6730719804763794, + "learning_rate": 0.0001, + "loss": 71.2045, + "step": 6860 + }, + { + "epoch": 1.1093235905613532, + "grad_norm": 0.6895949244499207, + "learning_rate": 0.0001, + "loss": 72.3056, + "step": 6870 + }, + { + "epoch": 1.1109384146464545, + "grad_norm": 2.476609945297241, + "learning_rate": 0.0001, + "loss": 70.3865, + "step": 6880 + }, + { + "epoch": 1.1125532387315558, + "grad_norm": 0.7285001873970032, + "learning_rate": 0.0001, + "loss": 71.5761, + "step": 6890 + }, + { + "epoch": 1.1141680628166568, + "grad_norm": 0.7546679973602295, + "learning_rate": 0.0001, + "loss": 72.5979, + "step": 6900 + }, + { + "epoch": 1.115782886901758, + "grad_norm": 0.8974841237068176, + "learning_rate": 0.0001, + "loss": 74.5888, + "step": 6910 + }, + { + "epoch": 1.1173977109868594, + "grad_norm": 0.8785758018493652, + "learning_rate": 0.0001, + "loss": 71.662, + "step": 6920 + }, + { + "epoch": 1.1190125350719606, + "grad_norm": 0.6805328130722046, + "learning_rate": 0.0001, + "loss": 72.201, + "step": 6930 + }, + { + "epoch": 1.120627359157062, + "grad_norm": 0.8132315278053284, + "learning_rate": 0.0001, + "loss": 72.7473, + "step": 6940 + }, + { + "epoch": 1.122242183242163, + "grad_norm": 1.1625338792800903, + "learning_rate": 0.0001, + "loss": 70.5987, + "step": 6950 + }, + { + "epoch": 1.1238570073272642, + "grad_norm": 1.0984737873077393, + "learning_rate": 0.0001, + "loss": 72.7335, + "step": 6960 + }, + { + "epoch": 1.1254718314123655, + "grad_norm": 0.6798779368400574, + "learning_rate": 0.0001, + "loss": 72.1942, + "step": 6970 + }, + { + "epoch": 1.1270866554974668, + "grad_norm": 0.7546010613441467, + "learning_rate": 0.0001, + "loss": 70.8014, + "step": 6980 + }, + { + "epoch": 1.128701479582568, + "grad_norm": 1.0238053798675537, + "learning_rate": 0.0001, + "loss": 72.5617, + "step": 6990 + }, + { + "epoch": 1.130316303667669, + "grad_norm": 0.6535419821739197, + "learning_rate": 0.0001, + "loss": 70.2193, + "step": 7000 + }, + { + "epoch": 1.1319311277527704, + "grad_norm": 0.7354230284690857, + "learning_rate": 0.0001, + "loss": 72.0685, + "step": 7010 + }, + { + "epoch": 1.1335459518378717, + "grad_norm": 0.7848799824714661, + "learning_rate": 0.0001, + "loss": 72.2869, + "step": 7020 + }, + { + "epoch": 1.135160775922973, + "grad_norm": 0.6519262790679932, + "learning_rate": 0.0001, + "loss": 72.2023, + "step": 7030 + }, + { + "epoch": 1.1367756000080742, + "grad_norm": 0.8428511023521423, + "learning_rate": 0.0001, + "loss": 71.3396, + "step": 7040 + }, + { + "epoch": 1.1383904240931753, + "grad_norm": 0.8873165249824524, + "learning_rate": 0.0001, + "loss": 69.8284, + "step": 7050 + }, + { + "epoch": 1.1400052481782765, + "grad_norm": 0.9141374230384827, + "learning_rate": 0.0001, + "loss": 70.3666, + "step": 7060 + }, + { + "epoch": 1.1416200722633778, + "grad_norm": 0.6556552648544312, + "learning_rate": 0.0001, + "loss": 71.1316, + "step": 7070 + }, + { + "epoch": 1.143234896348479, + "grad_norm": 0.6736201047897339, + "learning_rate": 0.0001, + "loss": 70.2601, + "step": 7080 + }, + { + "epoch": 1.1448497204335804, + "grad_norm": 0.6965599656105042, + "learning_rate": 0.0001, + "loss": 71.9409, + "step": 7090 + }, + { + "epoch": 1.1464645445186814, + "grad_norm": 1.0600581169128418, + "learning_rate": 0.0001, + "loss": 72.7344, + "step": 7100 + }, + { + "epoch": 1.1480793686037827, + "grad_norm": 0.7501764297485352, + "learning_rate": 0.0001, + "loss": 72.3129, + "step": 7110 + }, + { + "epoch": 1.149694192688884, + "grad_norm": 0.7105675935745239, + "learning_rate": 0.0001, + "loss": 71.1356, + "step": 7120 + }, + { + "epoch": 1.1513090167739852, + "grad_norm": 0.7446026802062988, + "learning_rate": 0.0001, + "loss": 69.8208, + "step": 7130 + }, + { + "epoch": 1.1529238408590865, + "grad_norm": 0.6664865016937256, + "learning_rate": 0.0001, + "loss": 72.2466, + "step": 7140 + }, + { + "epoch": 1.1545386649441876, + "grad_norm": 1.7590374946594238, + "learning_rate": 0.0001, + "loss": 71.4987, + "step": 7150 + }, + { + "epoch": 1.1561534890292888, + "grad_norm": 0.7941271066665649, + "learning_rate": 0.0001, + "loss": 70.787, + "step": 7160 + }, + { + "epoch": 1.15776831311439, + "grad_norm": 0.7473573088645935, + "learning_rate": 0.0001, + "loss": 71.2516, + "step": 7170 + }, + { + "epoch": 1.1593831371994914, + "grad_norm": 0.7589637041091919, + "learning_rate": 0.0001, + "loss": 71.4781, + "step": 7180 + }, + { + "epoch": 1.1609979612845926, + "grad_norm": 0.8148510456085205, + "learning_rate": 0.0001, + "loss": 71.1491, + "step": 7190 + }, + { + "epoch": 1.1626127853696937, + "grad_norm": 0.6803365349769592, + "learning_rate": 0.0001, + "loss": 71.7937, + "step": 7200 + }, + { + "epoch": 1.164227609454795, + "grad_norm": 0.7314755320549011, + "learning_rate": 0.0001, + "loss": 70.6566, + "step": 7210 + }, + { + "epoch": 1.1658424335398963, + "grad_norm": 0.6494631171226501, + "learning_rate": 0.0001, + "loss": 70.5057, + "step": 7220 + }, + { + "epoch": 1.1674572576249975, + "grad_norm": 0.6076239943504333, + "learning_rate": 0.0001, + "loss": 72.2985, + "step": 7230 + }, + { + "epoch": 1.1690720817100988, + "grad_norm": 0.6650206446647644, + "learning_rate": 0.0001, + "loss": 71.0719, + "step": 7240 + }, + { + "epoch": 1.1706869057951999, + "grad_norm": 0.6926266551017761, + "learning_rate": 0.0001, + "loss": 70.3714, + "step": 7250 + }, + { + "epoch": 1.1723017298803011, + "grad_norm": 0.6534716486930847, + "learning_rate": 0.0001, + "loss": 71.3194, + "step": 7260 + }, + { + "epoch": 1.1739165539654024, + "grad_norm": 0.7194528579711914, + "learning_rate": 0.0001, + "loss": 72.3277, + "step": 7270 + }, + { + "epoch": 1.1755313780505037, + "grad_norm": 0.7496437430381775, + "learning_rate": 0.0001, + "loss": 71.1136, + "step": 7280 + }, + { + "epoch": 1.177146202135605, + "grad_norm": 0.690522313117981, + "learning_rate": 0.0001, + "loss": 70.065, + "step": 7290 + }, + { + "epoch": 1.178761026220706, + "grad_norm": 0.6966914534568787, + "learning_rate": 0.0001, + "loss": 73.1137, + "step": 7300 + }, + { + "epoch": 1.1803758503058073, + "grad_norm": 0.7304641604423523, + "learning_rate": 0.0001, + "loss": 71.2466, + "step": 7310 + }, + { + "epoch": 1.1819906743909085, + "grad_norm": 0.7564008235931396, + "learning_rate": 0.0001, + "loss": 71.4147, + "step": 7320 + }, + { + "epoch": 1.1836054984760098, + "grad_norm": 0.61886066198349, + "learning_rate": 0.0001, + "loss": 70.3297, + "step": 7330 + }, + { + "epoch": 1.185220322561111, + "grad_norm": 0.6281574964523315, + "learning_rate": 0.0001, + "loss": 70.9607, + "step": 7340 + }, + { + "epoch": 1.1868351466462121, + "grad_norm": 0.6583316326141357, + "learning_rate": 0.0001, + "loss": 70.3443, + "step": 7350 + }, + { + "epoch": 1.1884499707313134, + "grad_norm": 1.8159942626953125, + "learning_rate": 0.0001, + "loss": 71.4792, + "step": 7360 + }, + { + "epoch": 1.1900647948164147, + "grad_norm": 0.7253140807151794, + "learning_rate": 0.0001, + "loss": 72.2182, + "step": 7370 + }, + { + "epoch": 1.191679618901516, + "grad_norm": 0.9141016602516174, + "learning_rate": 0.0001, + "loss": 68.7443, + "step": 7380 + }, + { + "epoch": 1.193294442986617, + "grad_norm": 0.7392430901527405, + "learning_rate": 0.0001, + "loss": 69.1888, + "step": 7390 + }, + { + "epoch": 1.1949092670717183, + "grad_norm": 0.7713924646377563, + "learning_rate": 0.0001, + "loss": 69.1301, + "step": 7400 + }, + { + "epoch": 1.1965240911568196, + "grad_norm": 0.6000798344612122, + "learning_rate": 0.0001, + "loss": 71.2576, + "step": 7410 + }, + { + "epoch": 1.1981389152419208, + "grad_norm": 0.687111496925354, + "learning_rate": 0.0001, + "loss": 70.2919, + "step": 7420 + }, + { + "epoch": 1.1997537393270221, + "grad_norm": 1.1718226671218872, + "learning_rate": 0.0001, + "loss": 71.9498, + "step": 7430 + }, + { + "epoch": 1.2013685634121232, + "grad_norm": 0.7176038026809692, + "learning_rate": 0.0001, + "loss": 71.91, + "step": 7440 + }, + { + "epoch": 1.2029833874972244, + "grad_norm": 0.7705256938934326, + "learning_rate": 0.0001, + "loss": 70.1241, + "step": 7450 + }, + { + "epoch": 1.2045982115823257, + "grad_norm": 0.8004609942436218, + "learning_rate": 0.0001, + "loss": 69.5121, + "step": 7460 + }, + { + "epoch": 1.206213035667427, + "grad_norm": 0.6708329319953918, + "learning_rate": 0.0001, + "loss": 73.0173, + "step": 7470 + }, + { + "epoch": 1.2078278597525283, + "grad_norm": 0.8777983784675598, + "learning_rate": 0.0001, + "loss": 71.6843, + "step": 7480 + }, + { + "epoch": 1.2094426838376293, + "grad_norm": 0.7683864831924438, + "learning_rate": 0.0001, + "loss": 71.6914, + "step": 7490 + }, + { + "epoch": 1.2110575079227306, + "grad_norm": 0.8022210597991943, + "learning_rate": 0.0001, + "loss": 69.1724, + "step": 7500 + }, + { + "epoch": 1.2126723320078319, + "grad_norm": 0.7250640392303467, + "learning_rate": 0.0001, + "loss": 70.6568, + "step": 7510 + }, + { + "epoch": 1.2142871560929331, + "grad_norm": 0.5625214576721191, + "learning_rate": 0.0001, + "loss": 69.4849, + "step": 7520 + }, + { + "epoch": 1.2159019801780344, + "grad_norm": 0.7980468273162842, + "learning_rate": 0.0001, + "loss": 70.8164, + "step": 7530 + }, + { + "epoch": 1.2175168042631355, + "grad_norm": 0.6900951862335205, + "learning_rate": 0.0001, + "loss": 71.1719, + "step": 7540 + }, + { + "epoch": 1.2191316283482367, + "grad_norm": 0.6284971833229065, + "learning_rate": 0.0001, + "loss": 70.1751, + "step": 7550 + }, + { + "epoch": 1.220746452433338, + "grad_norm": 0.6641798615455627, + "learning_rate": 0.0001, + "loss": 71.9043, + "step": 7560 + }, + { + "epoch": 1.2223612765184393, + "grad_norm": 0.7439492344856262, + "learning_rate": 0.0001, + "loss": 70.7286, + "step": 7570 + }, + { + "epoch": 1.2239761006035406, + "grad_norm": 0.8018355369567871, + "learning_rate": 0.0001, + "loss": 71.638, + "step": 7580 + }, + { + "epoch": 1.2255909246886416, + "grad_norm": 0.7318522930145264, + "learning_rate": 0.0001, + "loss": 70.0867, + "step": 7590 + }, + { + "epoch": 1.2272057487737429, + "grad_norm": 0.6799285411834717, + "learning_rate": 0.0001, + "loss": 71.3427, + "step": 7600 + }, + { + "epoch": 1.2288205728588442, + "grad_norm": 0.6796864867210388, + "learning_rate": 0.0001, + "loss": 71.1698, + "step": 7610 + }, + { + "epoch": 1.2304353969439454, + "grad_norm": 0.7256141304969788, + "learning_rate": 0.0001, + "loss": 71.1051, + "step": 7620 + }, + { + "epoch": 1.2320502210290467, + "grad_norm": 0.8318199515342712, + "learning_rate": 0.0001, + "loss": 71.3179, + "step": 7630 + }, + { + "epoch": 1.2336650451141478, + "grad_norm": 0.6594699025154114, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 7640 + }, + { + "epoch": 1.235279869199249, + "grad_norm": 0.7900195717811584, + "learning_rate": 0.0001, + "loss": 72.0139, + "step": 7650 + }, + { + "epoch": 1.2368946932843503, + "grad_norm": 0.7845672965049744, + "learning_rate": 0.0001, + "loss": 70.808, + "step": 7660 + }, + { + "epoch": 1.2385095173694516, + "grad_norm": 0.6598045229911804, + "learning_rate": 0.0001, + "loss": 69.8953, + "step": 7670 + }, + { + "epoch": 1.2401243414545529, + "grad_norm": 1.3222978115081787, + "learning_rate": 0.0001, + "loss": 68.787, + "step": 7680 + }, + { + "epoch": 1.241739165539654, + "grad_norm": 0.6660508513450623, + "learning_rate": 0.0001, + "loss": 71.5831, + "step": 7690 + }, + { + "epoch": 1.2433539896247552, + "grad_norm": 0.6305698156356812, + "learning_rate": 0.0001, + "loss": 70.3232, + "step": 7700 + }, + { + "epoch": 1.2449688137098565, + "grad_norm": 1.2648323774337769, + "learning_rate": 0.0001, + "loss": 67.8901, + "step": 7710 + }, + { + "epoch": 1.2465836377949577, + "grad_norm": 0.7177619934082031, + "learning_rate": 0.0001, + "loss": 69.6582, + "step": 7720 + }, + { + "epoch": 1.248198461880059, + "grad_norm": 0.6716091632843018, + "learning_rate": 0.0001, + "loss": 70.2034, + "step": 7730 + }, + { + "epoch": 1.24981328596516, + "grad_norm": 0.6050158143043518, + "learning_rate": 0.0001, + "loss": 69.3942, + "step": 7740 + }, + { + "epoch": 1.2514281100502613, + "grad_norm": 0.7001014947891235, + "learning_rate": 0.0001, + "loss": 69.5971, + "step": 7750 + }, + { + "epoch": 1.2530429341353626, + "grad_norm": 0.7221280932426453, + "learning_rate": 0.0001, + "loss": 71.3651, + "step": 7760 + }, + { + "epoch": 1.2546577582204639, + "grad_norm": 0.9130340218544006, + "learning_rate": 0.0001, + "loss": 68.3038, + "step": 7770 + }, + { + "epoch": 1.2562725823055652, + "grad_norm": 0.7703199982643127, + "learning_rate": 0.0001, + "loss": 71.6127, + "step": 7780 + }, + { + "epoch": 1.2578874063906662, + "grad_norm": 0.6334695816040039, + "learning_rate": 0.0001, + "loss": 68.9074, + "step": 7790 + }, + { + "epoch": 1.2595022304757675, + "grad_norm": 0.7074242234230042, + "learning_rate": 0.0001, + "loss": 70.0569, + "step": 7800 + }, + { + "epoch": 1.2611170545608688, + "grad_norm": 0.6956025958061218, + "learning_rate": 0.0001, + "loss": 69.892, + "step": 7810 + }, + { + "epoch": 1.26273187864597, + "grad_norm": 0.7235708236694336, + "learning_rate": 0.0001, + "loss": 71.1777, + "step": 7820 + }, + { + "epoch": 1.2643467027310713, + "grad_norm": 0.6157007813453674, + "learning_rate": 0.0001, + "loss": 71.5091, + "step": 7830 + }, + { + "epoch": 1.2659615268161724, + "grad_norm": 0.5352533459663391, + "learning_rate": 0.0001, + "loss": 70.083, + "step": 7840 + }, + { + "epoch": 1.2675763509012736, + "grad_norm": 0.7171182632446289, + "learning_rate": 0.0001, + "loss": 68.4585, + "step": 7850 + }, + { + "epoch": 1.269191174986375, + "grad_norm": 0.7503113746643066, + "learning_rate": 0.0001, + "loss": 70.4206, + "step": 7860 + }, + { + "epoch": 1.2708059990714762, + "grad_norm": 0.6552693843841553, + "learning_rate": 0.0001, + "loss": 70.8455, + "step": 7870 + }, + { + "epoch": 1.2724208231565775, + "grad_norm": 0.7173179984092712, + "learning_rate": 0.0001, + "loss": 68.82, + "step": 7880 + }, + { + "epoch": 1.2740356472416785, + "grad_norm": 0.7352942824363708, + "learning_rate": 0.0001, + "loss": 70.4974, + "step": 7890 + }, + { + "epoch": 1.2756504713267798, + "grad_norm": 0.6120197176933289, + "learning_rate": 0.0001, + "loss": 72.1638, + "step": 7900 + }, + { + "epoch": 1.277265295411881, + "grad_norm": 0.6271925568580627, + "learning_rate": 0.0001, + "loss": 69.3267, + "step": 7910 + }, + { + "epoch": 1.2788801194969823, + "grad_norm": 0.752636194229126, + "learning_rate": 0.0001, + "loss": 70.6761, + "step": 7920 + }, + { + "epoch": 1.2804949435820836, + "grad_norm": 0.7307374477386475, + "learning_rate": 0.0001, + "loss": 70.2909, + "step": 7930 + }, + { + "epoch": 1.2821097676671847, + "grad_norm": 0.5977495908737183, + "learning_rate": 0.0001, + "loss": 72.2969, + "step": 7940 + }, + { + "epoch": 1.283724591752286, + "grad_norm": 0.7927457094192505, + "learning_rate": 0.0001, + "loss": 70.3784, + "step": 7950 + }, + { + "epoch": 1.2853394158373872, + "grad_norm": 0.7520087957382202, + "learning_rate": 0.0001, + "loss": 70.7122, + "step": 7960 + }, + { + "epoch": 1.2869542399224885, + "grad_norm": 0.6036237478256226, + "learning_rate": 0.0001, + "loss": 69.8718, + "step": 7970 + }, + { + "epoch": 1.2885690640075897, + "grad_norm": 0.6805875897407532, + "learning_rate": 0.0001, + "loss": 70.2391, + "step": 7980 + }, + { + "epoch": 1.2901838880926908, + "grad_norm": 0.774559497833252, + "learning_rate": 0.0001, + "loss": 71.1605, + "step": 7990 + }, + { + "epoch": 1.291798712177792, + "grad_norm": 0.6991063356399536, + "learning_rate": 0.0001, + "loss": 69.9779, + "step": 8000 + }, + { + "epoch": 1.2934135362628933, + "grad_norm": 0.5877949595451355, + "learning_rate": 0.0001, + "loss": 71.16, + "step": 8010 + }, + { + "epoch": 1.2950283603479946, + "grad_norm": 0.8840436339378357, + "learning_rate": 0.0001, + "loss": 70.9454, + "step": 8020 + }, + { + "epoch": 1.296643184433096, + "grad_norm": 0.7532126307487488, + "learning_rate": 0.0001, + "loss": 70.141, + "step": 8030 + }, + { + "epoch": 1.298258008518197, + "grad_norm": 0.6880044341087341, + "learning_rate": 0.0001, + "loss": 70.0746, + "step": 8040 + }, + { + "epoch": 1.2998728326032982, + "grad_norm": 0.6319366097450256, + "learning_rate": 0.0001, + "loss": 69.9396, + "step": 8050 + }, + { + "epoch": 1.3014876566883995, + "grad_norm": 0.6329445838928223, + "learning_rate": 0.0001, + "loss": 69.9043, + "step": 8060 + }, + { + "epoch": 1.3031024807735008, + "grad_norm": 0.8892332315444946, + "learning_rate": 0.0001, + "loss": 69.4187, + "step": 8070 + }, + { + "epoch": 1.304717304858602, + "grad_norm": 0.696500301361084, + "learning_rate": 0.0001, + "loss": 68.1885, + "step": 8080 + }, + { + "epoch": 1.306332128943703, + "grad_norm": 0.5933213233947754, + "learning_rate": 0.0001, + "loss": 68.8017, + "step": 8090 + }, + { + "epoch": 1.3079469530288044, + "grad_norm": 0.9274749159812927, + "learning_rate": 0.0001, + "loss": 71.5875, + "step": 8100 + }, + { + "epoch": 1.3095617771139056, + "grad_norm": 0.7540446519851685, + "learning_rate": 0.0001, + "loss": 71.1233, + "step": 8110 + }, + { + "epoch": 1.311176601199007, + "grad_norm": 0.6701658964157104, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8120 + }, + { + "epoch": 1.3127914252841082, + "grad_norm": 0.6096424460411072, + "learning_rate": 0.0001, + "loss": 69.9583, + "step": 8130 + }, + { + "epoch": 1.3144062493692092, + "grad_norm": 0.809524416923523, + "learning_rate": 0.0001, + "loss": 71.8149, + "step": 8140 + }, + { + "epoch": 1.3160210734543105, + "grad_norm": 0.7928242683410645, + "learning_rate": 0.0001, + "loss": 70.2084, + "step": 8150 + }, + { + "epoch": 1.3176358975394118, + "grad_norm": 0.7462850213050842, + "learning_rate": 0.0001, + "loss": 71.4789, + "step": 8160 + }, + { + "epoch": 1.319250721624513, + "grad_norm": 0.8166030049324036, + "learning_rate": 0.0001, + "loss": 69.3154, + "step": 8170 + }, + { + "epoch": 1.3208655457096143, + "grad_norm": 0.7002414464950562, + "learning_rate": 0.0001, + "loss": 70.8048, + "step": 8180 + }, + { + "epoch": 1.3224803697947154, + "grad_norm": 0.7653809189796448, + "learning_rate": 0.0001, + "loss": 70.0221, + "step": 8190 + }, + { + "epoch": 1.3240951938798167, + "grad_norm": 0.6777089238166809, + "learning_rate": 0.0001, + "loss": 70.614, + "step": 8200 + }, + { + "epoch": 1.325710017964918, + "grad_norm": 0.692739725112915, + "learning_rate": 0.0001, + "loss": 71.2311, + "step": 8210 + }, + { + "epoch": 1.3273248420500192, + "grad_norm": 0.7985519766807556, + "learning_rate": 0.0001, + "loss": 68.1982, + "step": 8220 + }, + { + "epoch": 1.3289396661351205, + "grad_norm": 0.7002689242362976, + "learning_rate": 0.0001, + "loss": 70.9181, + "step": 8230 + }, + { + "epoch": 1.3305544902202215, + "grad_norm": 0.7029191255569458, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 8240 + }, + { + "epoch": 1.3321693143053228, + "grad_norm": 0.7292912602424622, + "learning_rate": 0.0001, + "loss": 71.5386, + "step": 8250 + }, + { + "epoch": 1.333784138390424, + "grad_norm": 0.6141605973243713, + "learning_rate": 0.0001, + "loss": 72.3396, + "step": 8260 + }, + { + "epoch": 1.3353989624755254, + "grad_norm": 0.6486957669258118, + "learning_rate": 0.0001, + "loss": 69.7607, + "step": 8270 + }, + { + "epoch": 1.3370137865606266, + "grad_norm": 0.7813800573348999, + "learning_rate": 0.0001, + "loss": 68.8979, + "step": 8280 + }, + { + "epoch": 1.3386286106457277, + "grad_norm": 0.6805126070976257, + "learning_rate": 0.0001, + "loss": 70.2179, + "step": 8290 + }, + { + "epoch": 1.340243434730829, + "grad_norm": 0.7202772498130798, + "learning_rate": 0.0001, + "loss": 72.5008, + "step": 8300 + }, + { + "epoch": 1.3418582588159302, + "grad_norm": 0.6881810426712036, + "learning_rate": 0.0001, + "loss": 70.1987, + "step": 8310 + }, + { + "epoch": 1.3434730829010315, + "grad_norm": 0.6337536573410034, + "learning_rate": 0.0001, + "loss": 70.4073, + "step": 8320 + }, + { + "epoch": 1.3450879069861328, + "grad_norm": 0.7313241958618164, + "learning_rate": 0.0001, + "loss": 69.3598, + "step": 8330 + }, + { + "epoch": 1.3467027310712338, + "grad_norm": 0.635947585105896, + "learning_rate": 0.0001, + "loss": 69.5143, + "step": 8340 + }, + { + "epoch": 1.3483175551563351, + "grad_norm": 0.6219074726104736, + "learning_rate": 0.0001, + "loss": 69.9903, + "step": 8350 + }, + { + "epoch": 1.3499323792414364, + "grad_norm": 0.5928048491477966, + "learning_rate": 0.0001, + "loss": 70.4222, + "step": 8360 + }, + { + "epoch": 1.3515472033265377, + "grad_norm": 0.5927143096923828, + "learning_rate": 0.0001, + "loss": 69.8642, + "step": 8370 + }, + { + "epoch": 1.353162027411639, + "grad_norm": 0.7614993453025818, + "learning_rate": 0.0001, + "loss": 69.5787, + "step": 8380 + }, + { + "epoch": 1.35477685149674, + "grad_norm": 0.5488380193710327, + "learning_rate": 0.0001, + "loss": 69.0028, + "step": 8390 + }, + { + "epoch": 1.3563916755818413, + "grad_norm": 0.6219164133071899, + "learning_rate": 0.0001, + "loss": 69.4591, + "step": 8400 + }, + { + "epoch": 1.3580064996669425, + "grad_norm": 0.6940358877182007, + "learning_rate": 0.0001, + "loss": 71.3795, + "step": 8410 + }, + { + "epoch": 1.3596213237520438, + "grad_norm": 0.5919192433357239, + "learning_rate": 0.0001, + "loss": 71.2761, + "step": 8420 + }, + { + "epoch": 1.361236147837145, + "grad_norm": 1.0803378820419312, + "learning_rate": 0.0001, + "loss": 69.0046, + "step": 8430 + }, + { + "epoch": 1.3628509719222461, + "grad_norm": 0.7028629183769226, + "learning_rate": 0.0001, + "loss": 71.5996, + "step": 8440 + }, + { + "epoch": 1.3644657960073474, + "grad_norm": 0.6829854249954224, + "learning_rate": 0.0001, + "loss": 67.8831, + "step": 8450 + }, + { + "epoch": 1.3660806200924487, + "grad_norm": 0.6937337517738342, + "learning_rate": 0.0001, + "loss": 69.8321, + "step": 8460 + }, + { + "epoch": 1.36769544417755, + "grad_norm": 0.7725892066955566, + "learning_rate": 0.0001, + "loss": 71.0784, + "step": 8470 + }, + { + "epoch": 1.3693102682626512, + "grad_norm": 0.7171871066093445, + "learning_rate": 0.0001, + "loss": 69.0828, + "step": 8480 + }, + { + "epoch": 1.3709250923477523, + "grad_norm": 0.6605113744735718, + "learning_rate": 0.0001, + "loss": 70.9461, + "step": 8490 + }, + { + "epoch": 1.3725399164328536, + "grad_norm": 0.9005604982376099, + "learning_rate": 0.0001, + "loss": 71.474, + "step": 8500 + }, + { + "epoch": 1.3741547405179548, + "grad_norm": 0.7454419732093811, + "learning_rate": 0.0001, + "loss": 70.3113, + "step": 8510 + }, + { + "epoch": 1.375769564603056, + "grad_norm": 0.7574180960655212, + "learning_rate": 0.0001, + "loss": 70.9015, + "step": 8520 + }, + { + "epoch": 1.3773843886881574, + "grad_norm": 0.6525025367736816, + "learning_rate": 0.0001, + "loss": 72.3132, + "step": 8530 + }, + { + "epoch": 1.3789992127732584, + "grad_norm": 0.6548861861228943, + "learning_rate": 0.0001, + "loss": 68.6495, + "step": 8540 + }, + { + "epoch": 1.3806140368583597, + "grad_norm": 0.7881495952606201, + "learning_rate": 0.0001, + "loss": 70.1335, + "step": 8550 + }, + { + "epoch": 1.382228860943461, + "grad_norm": 0.619380533695221, + "learning_rate": 0.0001, + "loss": 71.7469, + "step": 8560 + }, + { + "epoch": 1.3838436850285623, + "grad_norm": 0.7637182474136353, + "learning_rate": 0.0001, + "loss": 69.4691, + "step": 8570 + }, + { + "epoch": 1.3854585091136635, + "grad_norm": 0.8711991906166077, + "learning_rate": 0.0001, + "loss": 70.8131, + "step": 8580 + }, + { + "epoch": 1.3870733331987646, + "grad_norm": 0.6550050973892212, + "learning_rate": 0.0001, + "loss": 70.0096, + "step": 8590 + }, + { + "epoch": 1.3886881572838659, + "grad_norm": 0.6310496926307678, + "learning_rate": 0.0001, + "loss": 70.4305, + "step": 8600 + }, + { + "epoch": 1.3903029813689671, + "grad_norm": 0.5780142545700073, + "learning_rate": 0.0001, + "loss": 68.9639, + "step": 8610 + }, + { + "epoch": 1.3919178054540684, + "grad_norm": 0.6526662111282349, + "learning_rate": 0.0001, + "loss": 69.1345, + "step": 8620 + }, + { + "epoch": 1.3935326295391697, + "grad_norm": 1.2579848766326904, + "learning_rate": 0.0001, + "loss": 71.6908, + "step": 8630 + }, + { + "epoch": 1.3951474536242707, + "grad_norm": 0.7702625393867493, + "learning_rate": 0.0001, + "loss": 69.1064, + "step": 8640 + }, + { + "epoch": 1.396762277709372, + "grad_norm": 0.7414085865020752, + "learning_rate": 0.0001, + "loss": 69.5106, + "step": 8650 + }, + { + "epoch": 1.3983771017944733, + "grad_norm": 0.666235089302063, + "learning_rate": 0.0001, + "loss": 67.938, + "step": 8660 + }, + { + "epoch": 1.3999919258795746, + "grad_norm": 0.7879519462585449, + "learning_rate": 0.0001, + "loss": 69.5422, + "step": 8670 + }, + { + "epoch": 1.4016067499646758, + "grad_norm": 0.6552250981330872, + "learning_rate": 0.0001, + "loss": 69.4161, + "step": 8680 + }, + { + "epoch": 1.4032215740497769, + "grad_norm": 0.7558363676071167, + "learning_rate": 0.0001, + "loss": 71.3052, + "step": 8690 + }, + { + "epoch": 1.4048363981348782, + "grad_norm": 0.677007794380188, + "learning_rate": 0.0001, + "loss": 70.0824, + "step": 8700 + }, + { + "epoch": 1.4064512222199794, + "grad_norm": 0.6416738629341125, + "learning_rate": 0.0001, + "loss": 68.3357, + "step": 8710 + }, + { + "epoch": 1.4080660463050807, + "grad_norm": 0.8067912459373474, + "learning_rate": 0.0001, + "loss": 69.8551, + "step": 8720 + }, + { + "epoch": 1.409680870390182, + "grad_norm": 0.6017255783081055, + "learning_rate": 0.0001, + "loss": 70.3936, + "step": 8730 + }, + { + "epoch": 1.411295694475283, + "grad_norm": 0.5473611354827881, + "learning_rate": 0.0001, + "loss": 69.1556, + "step": 8740 + }, + { + "epoch": 1.4129105185603843, + "grad_norm": 0.641984760761261, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 8750 + }, + { + "epoch": 1.4145253426454856, + "grad_norm": 0.7509932518005371, + "learning_rate": 0.0001, + "loss": 68.793, + "step": 8760 + }, + { + "epoch": 1.4161401667305868, + "grad_norm": 0.615459144115448, + "learning_rate": 0.0001, + "loss": 69.0787, + "step": 8770 + }, + { + "epoch": 1.4177549908156881, + "grad_norm": 0.625276505947113, + "learning_rate": 0.0001, + "loss": 68.5643, + "step": 8780 + }, + { + "epoch": 1.4193698149007892, + "grad_norm": 0.7294536828994751, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 8790 + }, + { + "epoch": 1.4209846389858904, + "grad_norm": 0.8188890814781189, + "learning_rate": 0.0001, + "loss": 70.7486, + "step": 8800 + }, + { + "epoch": 1.4225994630709917, + "grad_norm": 0.8662137389183044, + "learning_rate": 0.0001, + "loss": 69.0625, + "step": 8810 + }, + { + "epoch": 1.424214287156093, + "grad_norm": 0.7548071146011353, + "learning_rate": 0.0001, + "loss": 68.9828, + "step": 8820 + }, + { + "epoch": 1.4258291112411943, + "grad_norm": 0.7295185327529907, + "learning_rate": 0.0001, + "loss": 69.0606, + "step": 8830 + }, + { + "epoch": 1.4274439353262953, + "grad_norm": 0.7937467098236084, + "learning_rate": 0.0001, + "loss": 69.0135, + "step": 8840 + }, + { + "epoch": 1.4290587594113966, + "grad_norm": 0.5957842469215393, + "learning_rate": 0.0001, + "loss": 68.3992, + "step": 8850 + }, + { + "epoch": 1.4306735834964979, + "grad_norm": 0.786580502986908, + "learning_rate": 0.0001, + "loss": 69.4485, + "step": 8860 + }, + { + "epoch": 1.4322884075815991, + "grad_norm": 0.8539725542068481, + "learning_rate": 0.0001, + "loss": 67.1037, + "step": 8870 + }, + { + "epoch": 1.4339032316667004, + "grad_norm": 0.684476375579834, + "learning_rate": 0.0001, + "loss": 68.9814, + "step": 8880 + }, + { + "epoch": 1.4355180557518015, + "grad_norm": 0.6335768103599548, + "learning_rate": 0.0001, + "loss": 69.9716, + "step": 8890 + }, + { + "epoch": 1.4371328798369027, + "grad_norm": 0.9881808757781982, + "learning_rate": 0.0001, + "loss": 68.0338, + "step": 8900 + }, + { + "epoch": 1.438747703922004, + "grad_norm": 0.783983588218689, + "learning_rate": 0.0001, + "loss": 70.124, + "step": 8910 + }, + { + "epoch": 1.4403625280071053, + "grad_norm": 0.817820131778717, + "learning_rate": 0.0001, + "loss": 67.933, + "step": 8920 + }, + { + "epoch": 1.4419773520922066, + "grad_norm": 0.6900199055671692, + "learning_rate": 0.0001, + "loss": 69.8871, + "step": 8930 + }, + { + "epoch": 1.4435921761773076, + "grad_norm": 0.6702716946601868, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8940 + }, + { + "epoch": 1.445207000262409, + "grad_norm": 0.6699396967887878, + "learning_rate": 0.0001, + "loss": 67.8654, + "step": 8950 + }, + { + "epoch": 1.4468218243475102, + "grad_norm": 0.6388489603996277, + "learning_rate": 0.0001, + "loss": 69.5605, + "step": 8960 + }, + { + "epoch": 1.4484366484326114, + "grad_norm": 0.735910952091217, + "learning_rate": 0.0001, + "loss": 66.7865, + "step": 8970 + }, + { + "epoch": 1.4500514725177127, + "grad_norm": 0.6826426386833191, + "learning_rate": 0.0001, + "loss": 68.2896, + "step": 8980 + }, + { + "epoch": 1.4516662966028138, + "grad_norm": 0.8190213441848755, + "learning_rate": 0.0001, + "loss": 70.3524, + "step": 8990 + }, + { + "epoch": 1.453281120687915, + "grad_norm": 0.6774603724479675, + "learning_rate": 0.0001, + "loss": 68.9084, + "step": 9000 + }, + { + "epoch": 1.4548959447730163, + "grad_norm": 0.8953077793121338, + "learning_rate": 0.0001, + "loss": 69.228, + "step": 9010 + }, + { + "epoch": 1.4565107688581176, + "grad_norm": 1.8509501218795776, + "learning_rate": 0.0001, + "loss": 68.0847, + "step": 9020 + }, + { + "epoch": 1.4581255929432189, + "grad_norm": 0.671280026435852, + "learning_rate": 0.0001, + "loss": 70.0691, + "step": 9030 + }, + { + "epoch": 1.45974041702832, + "grad_norm": 0.7242400646209717, + "learning_rate": 0.0001, + "loss": 68.8001, + "step": 9040 + }, + { + "epoch": 1.4613552411134212, + "grad_norm": 0.6609033942222595, + "learning_rate": 0.0001, + "loss": 69.0398, + "step": 9050 + }, + { + "epoch": 1.4629700651985225, + "grad_norm": 0.6850711703300476, + "learning_rate": 0.0001, + "loss": 70.8849, + "step": 9060 + }, + { + "epoch": 1.4645848892836237, + "grad_norm": 0.7409449219703674, + "learning_rate": 0.0001, + "loss": 68.0411, + "step": 9070 + }, + { + "epoch": 1.466199713368725, + "grad_norm": 0.6832754611968994, + "learning_rate": 0.0001, + "loss": 69.1219, + "step": 9080 + }, + { + "epoch": 1.467814537453826, + "grad_norm": 0.6671826243400574, + "learning_rate": 0.0001, + "loss": 68.1225, + "step": 9090 + }, + { + "epoch": 1.4694293615389273, + "grad_norm": 0.6758706569671631, + "learning_rate": 0.0001, + "loss": 68.6981, + "step": 9100 + }, + { + "epoch": 1.4710441856240286, + "grad_norm": 0.7885150909423828, + "learning_rate": 0.0001, + "loss": 69.29, + "step": 9110 + }, + { + "epoch": 1.4726590097091299, + "grad_norm": 0.7636108994483948, + "learning_rate": 0.0001, + "loss": 67.3151, + "step": 9120 + }, + { + "epoch": 1.4742738337942312, + "grad_norm": 0.6518180966377258, + "learning_rate": 0.0001, + "loss": 68.6208, + "step": 9130 + }, + { + "epoch": 1.4758886578793322, + "grad_norm": 0.6715161204338074, + "learning_rate": 0.0001, + "loss": 70.5106, + "step": 9140 + }, + { + "epoch": 1.4775034819644335, + "grad_norm": 0.6342995762825012, + "learning_rate": 0.0001, + "loss": 70.3067, + "step": 9150 + }, + { + "epoch": 1.4791183060495348, + "grad_norm": 0.6773675084114075, + "learning_rate": 0.0001, + "loss": 69.2888, + "step": 9160 + }, + { + "epoch": 1.480733130134636, + "grad_norm": 0.6887578368186951, + "learning_rate": 0.0001, + "loss": 69.2788, + "step": 9170 + }, + { + "epoch": 1.4823479542197373, + "grad_norm": 0.6793885231018066, + "learning_rate": 0.0001, + "loss": 68.3523, + "step": 9180 + }, + { + "epoch": 1.4839627783048384, + "grad_norm": 0.7483563423156738, + "learning_rate": 0.0001, + "loss": 67.9991, + "step": 9190 + }, + { + "epoch": 1.4855776023899396, + "grad_norm": 0.7007381916046143, + "learning_rate": 0.0001, + "loss": 67.5335, + "step": 9200 + }, + { + "epoch": 1.487192426475041, + "grad_norm": 0.7565139532089233, + "learning_rate": 0.0001, + "loss": 69.1093, + "step": 9210 + }, + { + "epoch": 1.4888072505601422, + "grad_norm": 1.0212360620498657, + "learning_rate": 0.0001, + "loss": 68.5061, + "step": 9220 + }, + { + "epoch": 1.4904220746452435, + "grad_norm": 0.9198092818260193, + "learning_rate": 0.0001, + "loss": 67.9735, + "step": 9230 + }, + { + "epoch": 1.4920368987303445, + "grad_norm": 0.6228001713752747, + "learning_rate": 0.0001, + "loss": 69.2231, + "step": 9240 + }, + { + "epoch": 1.4936517228154458, + "grad_norm": 0.6618618965148926, + "learning_rate": 0.0001, + "loss": 68.0434, + "step": 9250 + }, + { + "epoch": 1.495266546900547, + "grad_norm": 0.6241064071655273, + "learning_rate": 0.0001, + "loss": 68.4004, + "step": 9260 + }, + { + "epoch": 1.4968813709856483, + "grad_norm": 0.7336916327476501, + "learning_rate": 0.0001, + "loss": 69.0389, + "step": 9270 + }, + { + "epoch": 1.4984961950707496, + "grad_norm": 0.718125581741333, + "learning_rate": 0.0001, + "loss": 68.4923, + "step": 9280 + }, + { + "epoch": 1.5001110191558507, + "grad_norm": 0.6694948673248291, + "learning_rate": 0.0001, + "loss": 68.1006, + "step": 9290 + }, + { + "epoch": 1.501725843240952, + "grad_norm": 0.5815121531486511, + "learning_rate": 0.0001, + "loss": 68.5943, + "step": 9300 + }, + { + "epoch": 1.5033406673260532, + "grad_norm": 1.4076049327850342, + "learning_rate": 0.0001, + "loss": 68.1251, + "step": 9310 + }, + { + "epoch": 1.5049554914111543, + "grad_norm": 0.7294307947158813, + "learning_rate": 0.0001, + "loss": 68.0539, + "step": 9320 + }, + { + "epoch": 1.5065703154962558, + "grad_norm": 0.627821683883667, + "learning_rate": 0.0001, + "loss": 68.6859, + "step": 9330 + }, + { + "epoch": 1.5081851395813568, + "grad_norm": 0.6683267951011658, + "learning_rate": 0.0001, + "loss": 68.2727, + "step": 9340 + }, + { + "epoch": 1.509799963666458, + "grad_norm": 0.7070158123970032, + "learning_rate": 0.0001, + "loss": 69.1171, + "step": 9350 + }, + { + "epoch": 1.5114147877515594, + "grad_norm": 0.6324070692062378, + "learning_rate": 0.0001, + "loss": 69.7498, + "step": 9360 + }, + { + "epoch": 1.5130296118366604, + "grad_norm": 0.9814477562904358, + "learning_rate": 0.0001, + "loss": 69.1426, + "step": 9370 + }, + { + "epoch": 1.514644435921762, + "grad_norm": 0.5883511304855347, + "learning_rate": 0.0001, + "loss": 68.5363, + "step": 9380 + }, + { + "epoch": 1.516259260006863, + "grad_norm": 0.8325281739234924, + "learning_rate": 0.0001, + "loss": 67.5951, + "step": 9390 + }, + { + "epoch": 1.5178740840919642, + "grad_norm": 0.7456872463226318, + "learning_rate": 0.0001, + "loss": 68.1876, + "step": 9400 + }, + { + "epoch": 1.5194889081770655, + "grad_norm": 0.7692755460739136, + "learning_rate": 0.0001, + "loss": 69.1677, + "step": 9410 + }, + { + "epoch": 1.5211037322621666, + "grad_norm": 0.7346193194389343, + "learning_rate": 0.0001, + "loss": 69.8608, + "step": 9420 + }, + { + "epoch": 1.522718556347268, + "grad_norm": 0.7658166289329529, + "learning_rate": 0.0001, + "loss": 68.8584, + "step": 9430 + }, + { + "epoch": 1.524333380432369, + "grad_norm": 0.7892371416091919, + "learning_rate": 0.0001, + "loss": 68.3882, + "step": 9440 + }, + { + "epoch": 1.5259482045174704, + "grad_norm": 0.5943968296051025, + "learning_rate": 0.0001, + "loss": 68.7935, + "step": 9450 + }, + { + "epoch": 1.5275630286025716, + "grad_norm": 0.6611922383308411, + "learning_rate": 0.0001, + "loss": 69.0865, + "step": 9460 + }, + { + "epoch": 1.5291778526876727, + "grad_norm": 0.6697778105735779, + "learning_rate": 0.0001, + "loss": 68.6131, + "step": 9470 + }, + { + "epoch": 1.5307926767727742, + "grad_norm": 0.862293541431427, + "learning_rate": 0.0001, + "loss": 71.0167, + "step": 9480 + }, + { + "epoch": 1.5324075008578752, + "grad_norm": 0.7098677754402161, + "learning_rate": 0.0001, + "loss": 67.7216, + "step": 9490 + }, + { + "epoch": 1.5340223249429765, + "grad_norm": 0.6578338742256165, + "learning_rate": 0.0001, + "loss": 69.872, + "step": 9500 + }, + { + "epoch": 1.5356371490280778, + "grad_norm": 0.7272375822067261, + "learning_rate": 0.0001, + "loss": 70.0971, + "step": 9510 + }, + { + "epoch": 1.5372519731131788, + "grad_norm": 0.6497064828872681, + "learning_rate": 0.0001, + "loss": 70.5921, + "step": 9520 + }, + { + "epoch": 1.5388667971982803, + "grad_norm": 0.6691288352012634, + "learning_rate": 0.0001, + "loss": 68.5213, + "step": 9530 + }, + { + "epoch": 1.5404816212833814, + "grad_norm": 0.9180259108543396, + "learning_rate": 0.0001, + "loss": 67.0699, + "step": 9540 + }, + { + "epoch": 1.5420964453684827, + "grad_norm": 0.7062082886695862, + "learning_rate": 0.0001, + "loss": 69.3965, + "step": 9550 + }, + { + "epoch": 1.543711269453584, + "grad_norm": 0.711613655090332, + "learning_rate": 0.0001, + "loss": 67.8276, + "step": 9560 + }, + { + "epoch": 1.545326093538685, + "grad_norm": 0.670032799243927, + "learning_rate": 0.0001, + "loss": 68.1687, + "step": 9570 + }, + { + "epoch": 1.5469409176237865, + "grad_norm": 0.7531955242156982, + "learning_rate": 0.0001, + "loss": 69.2144, + "step": 9580 + }, + { + "epoch": 1.5485557417088875, + "grad_norm": 0.6759967803955078, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 9590 + }, + { + "epoch": 1.5501705657939888, + "grad_norm": 0.6134868860244751, + "learning_rate": 0.0001, + "loss": 67.4348, + "step": 9600 + }, + { + "epoch": 1.55178538987909, + "grad_norm": 0.638785719871521, + "learning_rate": 0.0001, + "loss": 68.4635, + "step": 9610 + }, + { + "epoch": 1.5534002139641911, + "grad_norm": 0.7901169657707214, + "learning_rate": 0.0001, + "loss": 68.1079, + "step": 9620 + }, + { + "epoch": 1.5550150380492926, + "grad_norm": 0.6970444917678833, + "learning_rate": 0.0001, + "loss": 69.3883, + "step": 9630 + }, + { + "epoch": 1.5566298621343937, + "grad_norm": 0.6954355239868164, + "learning_rate": 0.0001, + "loss": 69.3841, + "step": 9640 + }, + { + "epoch": 1.558244686219495, + "grad_norm": 0.7097641825675964, + "learning_rate": 0.0001, + "loss": 68.3417, + "step": 9650 + }, + { + "epoch": 1.5598595103045962, + "grad_norm": 0.7082354426383972, + "learning_rate": 0.0001, + "loss": 69.9314, + "step": 9660 + }, + { + "epoch": 1.5614743343896973, + "grad_norm": 0.6570103764533997, + "learning_rate": 0.0001, + "loss": 69.7219, + "step": 9670 + }, + { + "epoch": 1.5630891584747988, + "grad_norm": 0.5705522894859314, + "learning_rate": 0.0001, + "loss": 67.5177, + "step": 9680 + }, + { + "epoch": 1.5647039825598998, + "grad_norm": 0.6992695331573486, + "learning_rate": 0.0001, + "loss": 67.6538, + "step": 9690 + }, + { + "epoch": 1.5663188066450011, + "grad_norm": 0.7523599863052368, + "learning_rate": 0.0001, + "loss": 66.869, + "step": 9700 + }, + { + "epoch": 1.5679336307301024, + "grad_norm": 0.6267555356025696, + "learning_rate": 0.0001, + "loss": 66.6805, + "step": 9710 + }, + { + "epoch": 1.5695484548152034, + "grad_norm": 0.6138116717338562, + "learning_rate": 0.0001, + "loss": 71.3217, + "step": 9720 + }, + { + "epoch": 1.571163278900305, + "grad_norm": 0.7408527731895447, + "learning_rate": 0.0001, + "loss": 69.2759, + "step": 9730 + }, + { + "epoch": 1.572778102985406, + "grad_norm": 0.813761830329895, + "learning_rate": 0.0001, + "loss": 70.4959, + "step": 9740 + }, + { + "epoch": 1.5743929270705073, + "grad_norm": 1.0363599061965942, + "learning_rate": 0.0001, + "loss": 68.1996, + "step": 9750 + }, + { + "epoch": 1.5760077511556085, + "grad_norm": 0.7773903012275696, + "learning_rate": 0.0001, + "loss": 67.9354, + "step": 9760 + }, + { + "epoch": 1.5776225752407096, + "grad_norm": 0.6465738415718079, + "learning_rate": 0.0001, + "loss": 69.5247, + "step": 9770 + }, + { + "epoch": 1.579237399325811, + "grad_norm": 0.8433385491371155, + "learning_rate": 0.0001, + "loss": 66.9448, + "step": 9780 + }, + { + "epoch": 1.5808522234109121, + "grad_norm": 0.8651545643806458, + "learning_rate": 0.0001, + "loss": 68.6994, + "step": 9790 + }, + { + "epoch": 1.5824670474960134, + "grad_norm": 0.6940448880195618, + "learning_rate": 0.0001, + "loss": 67.622, + "step": 9800 + }, + { + "epoch": 1.5840818715811147, + "grad_norm": 0.622756838798523, + "learning_rate": 0.0001, + "loss": 69.0994, + "step": 9810 + }, + { + "epoch": 1.5856966956662157, + "grad_norm": 0.6119604110717773, + "learning_rate": 0.0001, + "loss": 67.7682, + "step": 9820 + }, + { + "epoch": 1.5873115197513172, + "grad_norm": 0.5729681849479675, + "learning_rate": 0.0001, + "loss": 67.3113, + "step": 9830 + }, + { + "epoch": 1.5889263438364183, + "grad_norm": 0.7143844366073608, + "learning_rate": 0.0001, + "loss": 68.8128, + "step": 9840 + }, + { + "epoch": 1.5905411679215196, + "grad_norm": 0.9814873337745667, + "learning_rate": 0.0001, + "loss": 69.3099, + "step": 9850 + }, + { + "epoch": 1.5921559920066208, + "grad_norm": 0.8044945001602173, + "learning_rate": 0.0001, + "loss": 69.3939, + "step": 9860 + }, + { + "epoch": 1.5937708160917219, + "grad_norm": 0.8494839668273926, + "learning_rate": 0.0001, + "loss": 69.3889, + "step": 9870 + }, + { + "epoch": 1.5953856401768234, + "grad_norm": 0.6468870639801025, + "learning_rate": 0.0001, + "loss": 67.2047, + "step": 9880 + }, + { + "epoch": 1.5970004642619244, + "grad_norm": 0.6270474791526794, + "learning_rate": 0.0001, + "loss": 69.6813, + "step": 9890 + }, + { + "epoch": 1.5986152883470257, + "grad_norm": 0.7102645039558411, + "learning_rate": 0.0001, + "loss": 69.1546, + "step": 9900 + }, + { + "epoch": 1.600230112432127, + "grad_norm": 0.5929123759269714, + "learning_rate": 0.0001, + "loss": 67.2187, + "step": 9910 + }, + { + "epoch": 1.601844936517228, + "grad_norm": 0.6466200947761536, + "learning_rate": 0.0001, + "loss": 68.6876, + "step": 9920 + }, + { + "epoch": 1.6034597606023295, + "grad_norm": 0.6441522836685181, + "learning_rate": 0.0001, + "loss": 68.7955, + "step": 9930 + }, + { + "epoch": 1.6050745846874306, + "grad_norm": 0.7264062166213989, + "learning_rate": 0.0001, + "loss": 67.8876, + "step": 9940 + }, + { + "epoch": 1.6066894087725319, + "grad_norm": 0.6973199248313904, + "learning_rate": 0.0001, + "loss": 69.0904, + "step": 9950 + }, + { + "epoch": 1.6083042328576331, + "grad_norm": 0.582053005695343, + "learning_rate": 0.0001, + "loss": 69.3876, + "step": 9960 + }, + { + "epoch": 1.6099190569427342, + "grad_norm": 0.6431038975715637, + "learning_rate": 0.0001, + "loss": 67.4232, + "step": 9970 + }, + { + "epoch": 1.6115338810278357, + "grad_norm": 0.6313247084617615, + "learning_rate": 0.0001, + "loss": 68.3759, + "step": 9980 + }, + { + "epoch": 1.6131487051129367, + "grad_norm": 1.2003700733184814, + "learning_rate": 0.0001, + "loss": 68.3413, + "step": 9990 + }, + { + "epoch": 1.614763529198038, + "grad_norm": 1.0262068510055542, + "learning_rate": 0.0001, + "loss": 66.7308, + "step": 10000 + }, + { + "epoch": 1.6163783532831393, + "grad_norm": 0.5975973010063171, + "learning_rate": 0.0001, + "loss": 68.6893, + "step": 10010 + }, + { + "epoch": 1.6179931773682403, + "grad_norm": 0.6045264601707458, + "learning_rate": 0.0001, + "loss": 69.5241, + "step": 10020 + }, + { + "epoch": 1.6196080014533418, + "grad_norm": 0.654019832611084, + "learning_rate": 0.0001, + "loss": 66.1208, + "step": 10030 + }, + { + "epoch": 1.6212228255384429, + "grad_norm": 0.533981204032898, + "learning_rate": 0.0001, + "loss": 68.2129, + "step": 10040 + }, + { + "epoch": 1.6228376496235442, + "grad_norm": 1.184734582901001, + "learning_rate": 0.0001, + "loss": 70.0547, + "step": 10050 + }, + { + "epoch": 1.6244524737086454, + "grad_norm": 0.5836975574493408, + "learning_rate": 0.0001, + "loss": 68.1098, + "step": 10060 + }, + { + "epoch": 1.6260672977937465, + "grad_norm": 0.5856032967567444, + "learning_rate": 0.0001, + "loss": 67.483, + "step": 10070 + }, + { + "epoch": 1.627682121878848, + "grad_norm": 0.6735382080078125, + "learning_rate": 0.0001, + "loss": 68.8628, + "step": 10080 + }, + { + "epoch": 1.629296945963949, + "grad_norm": 0.5753204226493835, + "learning_rate": 0.0001, + "loss": 66.2777, + "step": 10090 + }, + { + "epoch": 1.6309117700490503, + "grad_norm": 0.8327603340148926, + "learning_rate": 0.0001, + "loss": 67.9309, + "step": 10100 + }, + { + "epoch": 1.6325265941341516, + "grad_norm": 0.6873740553855896, + "learning_rate": 0.0001, + "loss": 68.7627, + "step": 10110 + }, + { + "epoch": 1.6341414182192526, + "grad_norm": 0.7218474745750427, + "learning_rate": 0.0001, + "loss": 69.3637, + "step": 10120 + }, + { + "epoch": 1.6357562423043541, + "grad_norm": 0.8393228650093079, + "learning_rate": 0.0001, + "loss": 66.7686, + "step": 10130 + }, + { + "epoch": 1.6373710663894552, + "grad_norm": 0.6433869004249573, + "learning_rate": 0.0001, + "loss": 68.0896, + "step": 10140 + }, + { + "epoch": 1.6389858904745565, + "grad_norm": 0.7419982552528381, + "learning_rate": 0.0001, + "loss": 67.9707, + "step": 10150 + }, + { + "epoch": 1.6406007145596577, + "grad_norm": 0.5884562730789185, + "learning_rate": 0.0001, + "loss": 68.0287, + "step": 10160 + }, + { + "epoch": 1.6422155386447588, + "grad_norm": 0.729557454586029, + "learning_rate": 0.0001, + "loss": 69.942, + "step": 10170 + }, + { + "epoch": 1.6438303627298603, + "grad_norm": 0.6026669144630432, + "learning_rate": 0.0001, + "loss": 66.8192, + "step": 10180 + }, + { + "epoch": 1.6454451868149613, + "grad_norm": 0.6921732425689697, + "learning_rate": 0.0001, + "loss": 66.7087, + "step": 10190 + }, + { + "epoch": 1.6470600109000626, + "grad_norm": 0.755172848701477, + "learning_rate": 0.0001, + "loss": 70.799, + "step": 10200 + }, + { + "epoch": 1.6486748349851639, + "grad_norm": 0.7318310737609863, + "learning_rate": 0.0001, + "loss": 67.6957, + "step": 10210 + }, + { + "epoch": 1.650289659070265, + "grad_norm": 0.6915041208267212, + "learning_rate": 0.0001, + "loss": 69.3388, + "step": 10220 + }, + { + "epoch": 1.6519044831553664, + "grad_norm": 0.7173444628715515, + "learning_rate": 0.0001, + "loss": 68.3612, + "step": 10230 + }, + { + "epoch": 1.6535193072404675, + "grad_norm": 0.7485487461090088, + "learning_rate": 0.0001, + "loss": 68.5366, + "step": 10240 + }, + { + "epoch": 1.6551341313255687, + "grad_norm": 0.5838190913200378, + "learning_rate": 0.0001, + "loss": 68.3841, + "step": 10250 + }, + { + "epoch": 1.65674895541067, + "grad_norm": 0.5908819437026978, + "learning_rate": 0.0001, + "loss": 67.8741, + "step": 10260 + }, + { + "epoch": 1.658363779495771, + "grad_norm": 0.7475612163543701, + "learning_rate": 0.0001, + "loss": 69.0636, + "step": 10270 + }, + { + "epoch": 1.6599786035808726, + "grad_norm": 0.5451382994651794, + "learning_rate": 0.0001, + "loss": 68.6695, + "step": 10280 + }, + { + "epoch": 1.6615934276659736, + "grad_norm": 0.6277157068252563, + "learning_rate": 0.0001, + "loss": 67.5432, + "step": 10290 + }, + { + "epoch": 1.663208251751075, + "grad_norm": 0.8144198060035706, + "learning_rate": 0.0001, + "loss": 66.7892, + "step": 10300 + }, + { + "epoch": 1.6648230758361762, + "grad_norm": 0.7400529980659485, + "learning_rate": 0.0001, + "loss": 66.2653, + "step": 10310 + }, + { + "epoch": 1.6664378999212772, + "grad_norm": 0.74524986743927, + "learning_rate": 0.0001, + "loss": 67.8252, + "step": 10320 + }, + { + "epoch": 1.6680527240063787, + "grad_norm": 0.5554162263870239, + "learning_rate": 0.0001, + "loss": 69.7704, + "step": 10330 + }, + { + "epoch": 1.6696675480914798, + "grad_norm": 0.8194338083267212, + "learning_rate": 0.0001, + "loss": 69.4668, + "step": 10340 + }, + { + "epoch": 1.671282372176581, + "grad_norm": 0.7947783470153809, + "learning_rate": 0.0001, + "loss": 64.9276, + "step": 10350 + }, + { + "epoch": 1.6728971962616823, + "grad_norm": 0.6145230531692505, + "learning_rate": 0.0001, + "loss": 68.1815, + "step": 10360 + }, + { + "epoch": 1.6745120203467834, + "grad_norm": 0.6293516755104065, + "learning_rate": 0.0001, + "loss": 67.8994, + "step": 10370 + }, + { + "epoch": 1.6761268444318849, + "grad_norm": 0.6160871386528015, + "learning_rate": 0.0001, + "loss": 66.7463, + "step": 10380 + }, + { + "epoch": 1.677741668516986, + "grad_norm": 0.6925204992294312, + "learning_rate": 0.0001, + "loss": 68.5261, + "step": 10390 + }, + { + "epoch": 1.6793564926020872, + "grad_norm": 0.6892788410186768, + "learning_rate": 0.0001, + "loss": 66.8928, + "step": 10400 + }, + { + "epoch": 1.6809713166871885, + "grad_norm": 0.9066945910453796, + "learning_rate": 0.0001, + "loss": 66.2556, + "step": 10410 + }, + { + "epoch": 1.6825861407722895, + "grad_norm": 0.5857477784156799, + "learning_rate": 0.0001, + "loss": 67.1866, + "step": 10420 + }, + { + "epoch": 1.684200964857391, + "grad_norm": 0.7989841103553772, + "learning_rate": 0.0001, + "loss": 67.7762, + "step": 10430 + }, + { + "epoch": 1.685815788942492, + "grad_norm": 0.559428870677948, + "learning_rate": 0.0001, + "loss": 67.5628, + "step": 10440 + }, + { + "epoch": 1.6874306130275933, + "grad_norm": 0.6718077659606934, + "learning_rate": 0.0001, + "loss": 69.2877, + "step": 10450 + }, + { + "epoch": 1.6890454371126946, + "grad_norm": 0.6952136754989624, + "learning_rate": 0.0001, + "loss": 67.2734, + "step": 10460 + }, + { + "epoch": 1.6906602611977957, + "grad_norm": 0.6079897880554199, + "learning_rate": 0.0001, + "loss": 67.4677, + "step": 10470 + }, + { + "epoch": 1.692275085282897, + "grad_norm": 0.6139684915542603, + "learning_rate": 0.0001, + "loss": 69.2464, + "step": 10480 + }, + { + "epoch": 1.6938899093679982, + "grad_norm": 0.7009351253509521, + "learning_rate": 0.0001, + "loss": 67.5424, + "step": 10490 + }, + { + "epoch": 1.6955047334530995, + "grad_norm": 0.8567383885383606, + "learning_rate": 0.0001, + "loss": 65.6573, + "step": 10500 + }, + { + "epoch": 1.6971195575382008, + "grad_norm": 0.6903256177902222, + "learning_rate": 0.0001, + "loss": 69.2218, + "step": 10510 + }, + { + "epoch": 1.6987343816233018, + "grad_norm": 0.5958806872367859, + "learning_rate": 0.0001, + "loss": 67.3718, + "step": 10520 + }, + { + "epoch": 1.700349205708403, + "grad_norm": 0.5983722805976868, + "learning_rate": 0.0001, + "loss": 68.7944, + "step": 10530 + }, + { + "epoch": 1.7019640297935044, + "grad_norm": 0.8119397759437561, + "learning_rate": 0.0001, + "loss": 66.7936, + "step": 10540 + }, + { + "epoch": 1.7035788538786056, + "grad_norm": 0.6383484601974487, + "learning_rate": 0.0001, + "loss": 67.8636, + "step": 10550 + }, + { + "epoch": 1.705193677963707, + "grad_norm": 0.6574084162712097, + "learning_rate": 0.0001, + "loss": 67.26, + "step": 10560 + }, + { + "epoch": 1.706808502048808, + "grad_norm": 0.6716108918190002, + "learning_rate": 0.0001, + "loss": 66.485, + "step": 10570 + }, + { + "epoch": 1.7084233261339092, + "grad_norm": 0.6998292803764343, + "learning_rate": 0.0001, + "loss": 66.4201, + "step": 10580 + }, + { + "epoch": 1.7100381502190105, + "grad_norm": 0.7300788760185242, + "learning_rate": 0.0001, + "loss": 67.1446, + "step": 10590 + }, + { + "epoch": 1.7116529743041118, + "grad_norm": 0.7066853642463684, + "learning_rate": 0.0001, + "loss": 70.8954, + "step": 10600 + }, + { + "epoch": 1.713267798389213, + "grad_norm": 0.6933701038360596, + "learning_rate": 0.0001, + "loss": 67.8473, + "step": 10610 + }, + { + "epoch": 1.7148826224743141, + "grad_norm": 0.6885779500007629, + "learning_rate": 0.0001, + "loss": 65.3276, + "step": 10620 + }, + { + "epoch": 1.7164974465594154, + "grad_norm": 0.6542202830314636, + "learning_rate": 0.0001, + "loss": 69.2596, + "step": 10630 + }, + { + "epoch": 1.7181122706445167, + "grad_norm": 0.5719691514968872, + "learning_rate": 0.0001, + "loss": 69.1907, + "step": 10640 + }, + { + "epoch": 1.719727094729618, + "grad_norm": 0.6596891283988953, + "learning_rate": 0.0001, + "loss": 69.0778, + "step": 10650 + }, + { + "epoch": 1.7213419188147192, + "grad_norm": 0.6860799193382263, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 10660 + }, + { + "epoch": 1.7229567428998203, + "grad_norm": 0.7912564277648926, + "learning_rate": 0.0001, + "loss": 68.2091, + "step": 10670 + }, + { + "epoch": 1.7245715669849215, + "grad_norm": 0.6673903465270996, + "learning_rate": 0.0001, + "loss": 68.24, + "step": 10680 + }, + { + "epoch": 1.7261863910700228, + "grad_norm": 0.6243769526481628, + "learning_rate": 0.0001, + "loss": 68.6396, + "step": 10690 + }, + { + "epoch": 1.727801215155124, + "grad_norm": 0.6622587442398071, + "learning_rate": 0.0001, + "loss": 67.9406, + "step": 10700 + }, + { + "epoch": 1.7294160392402254, + "grad_norm": 0.6808138489723206, + "learning_rate": 0.0001, + "loss": 69.1403, + "step": 10710 + }, + { + "epoch": 1.7310308633253264, + "grad_norm": 0.8498415350914001, + "learning_rate": 0.0001, + "loss": 70.1438, + "step": 10720 + }, + { + "epoch": 1.7326456874104277, + "grad_norm": 1.0091742277145386, + "learning_rate": 0.0001, + "loss": 67.0196, + "step": 10730 + }, + { + "epoch": 1.734260511495529, + "grad_norm": 0.6997767686843872, + "learning_rate": 0.0001, + "loss": 67.0307, + "step": 10740 + }, + { + "epoch": 1.7358753355806302, + "grad_norm": 0.6618793606758118, + "learning_rate": 0.0001, + "loss": 67.906, + "step": 10750 + }, + { + "epoch": 1.7374901596657315, + "grad_norm": 0.7082014679908752, + "learning_rate": 0.0001, + "loss": 68.0665, + "step": 10760 + }, + { + "epoch": 1.7391049837508326, + "grad_norm": 0.8494391441345215, + "learning_rate": 0.0001, + "loss": 68.4202, + "step": 10770 + }, + { + "epoch": 1.7407198078359338, + "grad_norm": 0.6247998476028442, + "learning_rate": 0.0001, + "loss": 66.4399, + "step": 10780 + }, + { + "epoch": 1.742334631921035, + "grad_norm": 0.8239936828613281, + "learning_rate": 0.0001, + "loss": 68.4374, + "step": 10790 + }, + { + "epoch": 1.7439494560061364, + "grad_norm": 0.6734293103218079, + "learning_rate": 0.0001, + "loss": 66.3411, + "step": 10800 + }, + { + "epoch": 1.7455642800912377, + "grad_norm": 0.7066450119018555, + "learning_rate": 0.0001, + "loss": 67.7102, + "step": 10810 + }, + { + "epoch": 1.7471791041763387, + "grad_norm": 0.7440647482872009, + "learning_rate": 0.0001, + "loss": 67.7221, + "step": 10820 + }, + { + "epoch": 1.74879392826144, + "grad_norm": 0.7771329283714294, + "learning_rate": 0.0001, + "loss": 67.5907, + "step": 10830 + }, + { + "epoch": 1.7504087523465413, + "grad_norm": 1.2484230995178223, + "learning_rate": 0.0001, + "loss": 66.3242, + "step": 10840 + }, + { + "epoch": 1.7520235764316425, + "grad_norm": 0.6946772933006287, + "learning_rate": 0.0001, + "loss": 67.8741, + "step": 10850 + }, + { + "epoch": 1.7536384005167438, + "grad_norm": 0.7676035761833191, + "learning_rate": 0.0001, + "loss": 67.3776, + "step": 10860 + }, + { + "epoch": 1.7552532246018449, + "grad_norm": 0.5534589290618896, + "learning_rate": 0.0001, + "loss": 67.6656, + "step": 10870 + }, + { + "epoch": 1.7568680486869461, + "grad_norm": 1.2745810747146606, + "learning_rate": 0.0001, + "loss": 68.5959, + "step": 10880 + }, + { + "epoch": 1.7584828727720474, + "grad_norm": 0.6241582036018372, + "learning_rate": 0.0001, + "loss": 67.7142, + "step": 10890 + }, + { + "epoch": 1.7600976968571487, + "grad_norm": 0.9308313131332397, + "learning_rate": 0.0001, + "loss": 67.0322, + "step": 10900 + }, + { + "epoch": 1.76171252094225, + "grad_norm": 0.5639570951461792, + "learning_rate": 0.0001, + "loss": 67.928, + "step": 10910 + }, + { + "epoch": 1.763327345027351, + "grad_norm": 0.6692782044410706, + "learning_rate": 0.0001, + "loss": 67.625, + "step": 10920 + }, + { + "epoch": 1.7649421691124523, + "grad_norm": 0.7322304248809814, + "learning_rate": 0.0001, + "loss": 66.2372, + "step": 10930 + }, + { + "epoch": 1.7665569931975535, + "grad_norm": 0.6275238394737244, + "learning_rate": 0.0001, + "loss": 67.7628, + "step": 10940 + }, + { + "epoch": 1.7681718172826548, + "grad_norm": 0.6832380890846252, + "learning_rate": 0.0001, + "loss": 67.6587, + "step": 10950 + }, + { + "epoch": 1.769786641367756, + "grad_norm": 0.6934792399406433, + "learning_rate": 0.0001, + "loss": 67.1411, + "step": 10960 + }, + { + "epoch": 1.7714014654528571, + "grad_norm": 0.7031943202018738, + "learning_rate": 0.0001, + "loss": 69.2734, + "step": 10970 + }, + { + "epoch": 1.7730162895379584, + "grad_norm": 0.91269451379776, + "learning_rate": 0.0001, + "loss": 68.1353, + "step": 10980 + }, + { + "epoch": 1.7746311136230597, + "grad_norm": 0.8927266001701355, + "learning_rate": 0.0001, + "loss": 68.003, + "step": 10990 + }, + { + "epoch": 1.776245937708161, + "grad_norm": 0.9115116596221924, + "learning_rate": 0.0001, + "loss": 69.1506, + "step": 11000 + }, + { + "epoch": 1.7778607617932622, + "grad_norm": 0.6055391430854797, + "learning_rate": 0.0001, + "loss": 68.9785, + "step": 11010 + }, + { + "epoch": 1.7794755858783633, + "grad_norm": 0.872809112071991, + "learning_rate": 0.0001, + "loss": 66.373, + "step": 11020 + }, + { + "epoch": 1.7810904099634646, + "grad_norm": 0.5838716626167297, + "learning_rate": 0.0001, + "loss": 69.507, + "step": 11030 + }, + { + "epoch": 1.7827052340485658, + "grad_norm": 0.6108883619308472, + "learning_rate": 0.0001, + "loss": 67.3743, + "step": 11040 + }, + { + "epoch": 1.784320058133667, + "grad_norm": 0.5457114577293396, + "learning_rate": 0.0001, + "loss": 66.1342, + "step": 11050 + }, + { + "epoch": 1.7859348822187684, + "grad_norm": 0.7696211338043213, + "learning_rate": 0.0001, + "loss": 67.9779, + "step": 11060 + }, + { + "epoch": 1.7875497063038694, + "grad_norm": 0.5795412659645081, + "learning_rate": 0.0001, + "loss": 67.1763, + "step": 11070 + }, + { + "epoch": 1.7891645303889707, + "grad_norm": 0.7401878833770752, + "learning_rate": 0.0001, + "loss": 67.4972, + "step": 11080 + }, + { + "epoch": 1.790779354474072, + "grad_norm": 0.666051983833313, + "learning_rate": 0.0001, + "loss": 68.7621, + "step": 11090 + }, + { + "epoch": 1.792394178559173, + "grad_norm": 0.9650185108184814, + "learning_rate": 0.0001, + "loss": 66.7721, + "step": 11100 + }, + { + "epoch": 1.7940090026442745, + "grad_norm": 0.6026908159255981, + "learning_rate": 0.0001, + "loss": 66.2993, + "step": 11110 + }, + { + "epoch": 1.7956238267293756, + "grad_norm": 0.6560576558113098, + "learning_rate": 0.0001, + "loss": 67.8543, + "step": 11120 + }, + { + "epoch": 1.7972386508144769, + "grad_norm": 0.9523794054985046, + "learning_rate": 0.0001, + "loss": 66.7881, + "step": 11130 + }, + { + "epoch": 1.7988534748995781, + "grad_norm": 0.7241806983947754, + "learning_rate": 0.0001, + "loss": 68.1071, + "step": 11140 + }, + { + "epoch": 1.8004682989846792, + "grad_norm": 0.639866292476654, + "learning_rate": 0.0001, + "loss": 68.992, + "step": 11150 + }, + { + "epoch": 1.8020831230697807, + "grad_norm": 0.6050712466239929, + "learning_rate": 0.0001, + "loss": 67.7993, + "step": 11160 + }, + { + "epoch": 1.8036979471548817, + "grad_norm": 0.5777723789215088, + "learning_rate": 0.0001, + "loss": 66.9515, + "step": 11170 + }, + { + "epoch": 1.805312771239983, + "grad_norm": 0.6153426766395569, + "learning_rate": 0.0001, + "loss": 68.3413, + "step": 11180 + }, + { + "epoch": 1.8069275953250843, + "grad_norm": 0.6118552088737488, + "learning_rate": 0.0001, + "loss": 67.7853, + "step": 11190 + }, + { + "epoch": 1.8085424194101853, + "grad_norm": 0.6520341038703918, + "learning_rate": 0.0001, + "loss": 67.3511, + "step": 11200 + }, + { + "epoch": 1.8101572434952868, + "grad_norm": 0.6189568638801575, + "learning_rate": 0.0001, + "loss": 69.28, + "step": 11210 + }, + { + "epoch": 1.811772067580388, + "grad_norm": 0.602411150932312, + "learning_rate": 0.0001, + "loss": 67.5133, + "step": 11220 + }, + { + "epoch": 1.8133868916654892, + "grad_norm": 0.7359414100646973, + "learning_rate": 0.0001, + "loss": 68.631, + "step": 11230 + }, + { + "epoch": 1.8150017157505904, + "grad_norm": 0.6216995120048523, + "learning_rate": 0.0001, + "loss": 68.3588, + "step": 11240 + }, + { + "epoch": 1.8166165398356915, + "grad_norm": 0.7683486938476562, + "learning_rate": 0.0001, + "loss": 67.1434, + "step": 11250 + }, + { + "epoch": 1.818231363920793, + "grad_norm": 0.5565283894538879, + "learning_rate": 0.0001, + "loss": 66.9995, + "step": 11260 + }, + { + "epoch": 1.819846188005894, + "grad_norm": 0.5623223781585693, + "learning_rate": 0.0001, + "loss": 68.1316, + "step": 11270 + }, + { + "epoch": 1.8214610120909953, + "grad_norm": 0.532821774482727, + "learning_rate": 0.0001, + "loss": 69.1903, + "step": 11280 + }, + { + "epoch": 1.8230758361760966, + "grad_norm": 0.6297196745872498, + "learning_rate": 0.0001, + "loss": 68.4965, + "step": 11290 + }, + { + "epoch": 1.8246906602611976, + "grad_norm": 0.839862048625946, + "learning_rate": 0.0001, + "loss": 67.1782, + "step": 11300 + }, + { + "epoch": 1.8263054843462991, + "grad_norm": 0.7021651864051819, + "learning_rate": 0.0001, + "loss": 67.4355, + "step": 11310 + }, + { + "epoch": 1.8279203084314002, + "grad_norm": 0.676034152507782, + "learning_rate": 0.0001, + "loss": 68.3721, + "step": 11320 + }, + { + "epoch": 1.8295351325165015, + "grad_norm": 0.6621078252792358, + "learning_rate": 0.0001, + "loss": 67.3255, + "step": 11330 + }, + { + "epoch": 1.8311499566016027, + "grad_norm": 0.5902897119522095, + "learning_rate": 0.0001, + "loss": 68.0641, + "step": 11340 + }, + { + "epoch": 1.8327647806867038, + "grad_norm": 0.6334371566772461, + "learning_rate": 0.0001, + "loss": 68.2682, + "step": 11350 + }, + { + "epoch": 1.8343796047718053, + "grad_norm": 0.6753348708152771, + "learning_rate": 0.0001, + "loss": 66.6749, + "step": 11360 + }, + { + "epoch": 1.8359944288569063, + "grad_norm": 0.7221750617027283, + "learning_rate": 0.0001, + "loss": 68.4142, + "step": 11370 + }, + { + "epoch": 1.8376092529420076, + "grad_norm": 0.7476344108581543, + "learning_rate": 0.0001, + "loss": 67.1416, + "step": 11380 + }, + { + "epoch": 1.8392240770271089, + "grad_norm": 0.5927479863166809, + "learning_rate": 0.0001, + "loss": 67.6368, + "step": 11390 + }, + { + "epoch": 1.84083890111221, + "grad_norm": 0.6753717660903931, + "learning_rate": 0.0001, + "loss": 67.1026, + "step": 11400 + }, + { + "epoch": 1.8424537251973114, + "grad_norm": 0.6500268578529358, + "learning_rate": 0.0001, + "loss": 68.7537, + "step": 11410 + }, + { + "epoch": 1.8440685492824125, + "grad_norm": 0.5930709838867188, + "learning_rate": 0.0001, + "loss": 65.7429, + "step": 11420 + }, + { + "epoch": 1.8456833733675138, + "grad_norm": 0.6467777490615845, + "learning_rate": 0.0001, + "loss": 68.1333, + "step": 11430 + }, + { + "epoch": 1.847298197452615, + "grad_norm": 0.6935669183731079, + "learning_rate": 0.0001, + "loss": 66.7728, + "step": 11440 + }, + { + "epoch": 1.848913021537716, + "grad_norm": 0.5838116407394409, + "learning_rate": 0.0001, + "loss": 66.6005, + "step": 11450 + }, + { + "epoch": 1.8505278456228176, + "grad_norm": 0.8621205687522888, + "learning_rate": 0.0001, + "loss": 66.0859, + "step": 11460 + }, + { + "epoch": 1.8521426697079186, + "grad_norm": 0.7119213342666626, + "learning_rate": 0.0001, + "loss": 66.6198, + "step": 11470 + }, + { + "epoch": 1.85375749379302, + "grad_norm": 0.6861911416053772, + "learning_rate": 0.0001, + "loss": 67.2275, + "step": 11480 + }, + { + "epoch": 1.8553723178781212, + "grad_norm": 0.5653937458992004, + "learning_rate": 0.0001, + "loss": 67.827, + "step": 11490 + }, + { + "epoch": 1.8569871419632222, + "grad_norm": 0.6458482146263123, + "learning_rate": 0.0001, + "loss": 66.3906, + "step": 11500 + }, + { + "epoch": 1.8586019660483237, + "grad_norm": 0.6701959371566772, + "learning_rate": 0.0001, + "loss": 67.88, + "step": 11510 + }, + { + "epoch": 1.8602167901334248, + "grad_norm": 0.6598711609840393, + "learning_rate": 0.0001, + "loss": 66.8373, + "step": 11520 + }, + { + "epoch": 1.861831614218526, + "grad_norm": 0.6095236539840698, + "learning_rate": 0.0001, + "loss": 67.6478, + "step": 11530 + }, + { + "epoch": 1.8634464383036273, + "grad_norm": 0.8192022442817688, + "learning_rate": 0.0001, + "loss": 67.7343, + "step": 11540 + }, + { + "epoch": 1.8650612623887284, + "grad_norm": 0.5788753628730774, + "learning_rate": 0.0001, + "loss": 66.7909, + "step": 11550 + }, + { + "epoch": 1.8666760864738299, + "grad_norm": 0.5383989214897156, + "learning_rate": 0.0001, + "loss": 68.5349, + "step": 11560 + }, + { + "epoch": 1.868290910558931, + "grad_norm": 0.6224480271339417, + "learning_rate": 0.0001, + "loss": 68.1938, + "step": 11570 + }, + { + "epoch": 1.8699057346440322, + "grad_norm": 0.5852788686752319, + "learning_rate": 0.0001, + "loss": 66.4678, + "step": 11580 + }, + { + "epoch": 1.8715205587291335, + "grad_norm": 0.730378270149231, + "learning_rate": 0.0001, + "loss": 67.57, + "step": 11590 + }, + { + "epoch": 1.8731353828142345, + "grad_norm": 0.6197090744972229, + "learning_rate": 0.0001, + "loss": 65.7801, + "step": 11600 + }, + { + "epoch": 1.874750206899336, + "grad_norm": 0.518974781036377, + "learning_rate": 0.0001, + "loss": 68.9308, + "step": 11610 + }, + { + "epoch": 1.876365030984437, + "grad_norm": 0.6601742506027222, + "learning_rate": 0.0001, + "loss": 66.821, + "step": 11620 + }, + { + "epoch": 1.8779798550695384, + "grad_norm": 0.7409974932670593, + "learning_rate": 0.0001, + "loss": 66.6883, + "step": 11630 + }, + { + "epoch": 1.8795946791546396, + "grad_norm": 0.6372189521789551, + "learning_rate": 0.0001, + "loss": 67.2109, + "step": 11640 + }, + { + "epoch": 1.8812095032397407, + "grad_norm": 0.6107574701309204, + "learning_rate": 0.0001, + "loss": 67.029, + "step": 11650 + }, + { + "epoch": 1.8828243273248422, + "grad_norm": 0.6543708443641663, + "learning_rate": 0.0001, + "loss": 64.2133, + "step": 11660 + }, + { + "epoch": 1.8844391514099432, + "grad_norm": 0.5161535739898682, + "learning_rate": 0.0001, + "loss": 66.7091, + "step": 11670 + }, + { + "epoch": 1.8860539754950445, + "grad_norm": 0.6089748740196228, + "learning_rate": 0.0001, + "loss": 69.2073, + "step": 11680 + }, + { + "epoch": 1.8876687995801458, + "grad_norm": 0.6930866837501526, + "learning_rate": 0.0001, + "loss": 68.7227, + "step": 11690 + }, + { + "epoch": 1.8892836236652468, + "grad_norm": 0.6563196182250977, + "learning_rate": 0.0001, + "loss": 68.27, + "step": 11700 + }, + { + "epoch": 1.8908984477503483, + "grad_norm": 0.6173767447471619, + "learning_rate": 0.0001, + "loss": 68.5535, + "step": 11710 + }, + { + "epoch": 1.8925132718354494, + "grad_norm": 0.5498561263084412, + "learning_rate": 0.0001, + "loss": 67.7681, + "step": 11720 + }, + { + "epoch": 1.8941280959205506, + "grad_norm": 0.584891676902771, + "learning_rate": 0.0001, + "loss": 68.1709, + "step": 11730 + }, + { + "epoch": 1.895742920005652, + "grad_norm": 0.7491183280944824, + "learning_rate": 0.0001, + "loss": 65.7435, + "step": 11740 + }, + { + "epoch": 1.897357744090753, + "grad_norm": 0.6519315242767334, + "learning_rate": 0.0001, + "loss": 67.5436, + "step": 11750 + }, + { + "epoch": 1.8989725681758545, + "grad_norm": 0.5740950107574463, + "learning_rate": 0.0001, + "loss": 66.3092, + "step": 11760 + }, + { + "epoch": 1.9005873922609555, + "grad_norm": 0.7978914380073547, + "learning_rate": 0.0001, + "loss": 66.8598, + "step": 11770 + }, + { + "epoch": 1.9022022163460568, + "grad_norm": 0.5536732077598572, + "learning_rate": 0.0001, + "loss": 68.1609, + "step": 11780 + }, + { + "epoch": 1.903817040431158, + "grad_norm": 0.6358754634857178, + "learning_rate": 0.0001, + "loss": 68.2948, + "step": 11790 + }, + { + "epoch": 1.9054318645162591, + "grad_norm": 0.8400535583496094, + "learning_rate": 0.0001, + "loss": 67.7722, + "step": 11800 + }, + { + "epoch": 1.9070466886013606, + "grad_norm": 0.8256928324699402, + "learning_rate": 0.0001, + "loss": 65.8444, + "step": 11810 + }, + { + "epoch": 1.9086615126864617, + "grad_norm": 0.9160122871398926, + "learning_rate": 0.0001, + "loss": 67.4363, + "step": 11820 + }, + { + "epoch": 1.910276336771563, + "grad_norm": 0.6507587432861328, + "learning_rate": 0.0001, + "loss": 65.9515, + "step": 11830 + }, + { + "epoch": 1.9118911608566642, + "grad_norm": 0.5940394401550293, + "learning_rate": 0.0001, + "loss": 65.8083, + "step": 11840 + }, + { + "epoch": 1.9135059849417653, + "grad_norm": 0.7403995990753174, + "learning_rate": 0.0001, + "loss": 66.7136, + "step": 11850 + }, + { + "epoch": 1.9151208090268668, + "grad_norm": 0.6039106845855713, + "learning_rate": 0.0001, + "loss": 67.6913, + "step": 11860 + }, + { + "epoch": 1.9167356331119678, + "grad_norm": 0.9714781641960144, + "learning_rate": 0.0001, + "loss": 67.9658, + "step": 11870 + }, + { + "epoch": 1.918350457197069, + "grad_norm": 0.6880549192428589, + "learning_rate": 0.0001, + "loss": 68.4463, + "step": 11880 + }, + { + "epoch": 1.9199652812821704, + "grad_norm": 0.5718733668327332, + "learning_rate": 0.0001, + "loss": 66.9625, + "step": 11890 + }, + { + "epoch": 1.9215801053672714, + "grad_norm": 0.6208164691925049, + "learning_rate": 0.0001, + "loss": 65.9462, + "step": 11900 + }, + { + "epoch": 1.923194929452373, + "grad_norm": 0.5957098007202148, + "learning_rate": 0.0001, + "loss": 66.9398, + "step": 11910 + }, + { + "epoch": 1.924809753537474, + "grad_norm": 0.6295427680015564, + "learning_rate": 0.0001, + "loss": 66.2246, + "step": 11920 + }, + { + "epoch": 1.9264245776225752, + "grad_norm": 0.8903796672821045, + "learning_rate": 0.0001, + "loss": 66.2999, + "step": 11930 + }, + { + "epoch": 1.9280394017076765, + "grad_norm": 0.6410874128341675, + "learning_rate": 0.0001, + "loss": 67.6857, + "step": 11940 + }, + { + "epoch": 1.9296542257927776, + "grad_norm": 0.5800307393074036, + "learning_rate": 0.0001, + "loss": 66.0943, + "step": 11950 + }, + { + "epoch": 1.931269049877879, + "grad_norm": 0.6834051609039307, + "learning_rate": 0.0001, + "loss": 67.6365, + "step": 11960 + }, + { + "epoch": 1.9328838739629801, + "grad_norm": 0.6112056970596313, + "learning_rate": 0.0001, + "loss": 66.8131, + "step": 11970 + }, + { + "epoch": 1.9344986980480814, + "grad_norm": 0.6657601594924927, + "learning_rate": 0.0001, + "loss": 65.7584, + "step": 11980 + }, + { + "epoch": 1.9361135221331827, + "grad_norm": 0.7570573091506958, + "learning_rate": 0.0001, + "loss": 68.6683, + "step": 11990 + }, + { + "epoch": 1.9377283462182837, + "grad_norm": 0.6231463551521301, + "learning_rate": 0.0001, + "loss": 67.71, + "step": 12000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4693555080687256e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6a151f546555c78ebc502dd7b703b665f0672d6f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35235df8408202e91954a585851d27187bafaf4e1d75c4e63c51fbf721136ff5 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-12000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/latest b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/latest new file mode 100644 index 0000000000000000000000000000000000000000..6c2e99ab39c9aa1d68dfdcb53dc9234015ced784 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/latest @@ -0,0 +1 @@ +global_step14000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3a893343b5aca926914d51ef2577fa67527857b1 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:546e27657ba97cbe60d3b94694b0b09d16a761577e5071386aedb7a9360f1ffb +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..fb29deed64b9191873e7866b459b970c4db99a47 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cea9d7eaa9ca9c183be5d9c8abde204f22439e196b5efe1c1023c9c38923604 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..066d30c64e2db4e5cf91bbd8b72b2c4d66b95a0f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb6e1774507dc1bc4cd3029ec8625a3faab8b9363f7fa3198f30ce5250759eda +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e24d95a2d0a15d4d30168d9cd7db90d254bf0f95 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba3318165394df9c71812b3f79d756ab018597b25159cc0cb85fea648f6654b9 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..2479e3be9f751c18da554e5e431c93dcbf056bb1 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c609ddbc84536db005191627008c6135e34a505e9630534bffb72ee24d2c401 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..40661fd48256f5695497ecc1ecf1571d26139f48 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92f1c3b48f4c3dab77c976f96201ade34129406849ed18dead4615d85b0a56a7 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..a184e360796cc83fc7e8e19d3f845b89d818965d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fc1f74092e62fc07dc8279b4f52d3c345854a878d2bf8c8fd5a7586a8a0d8e1 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..7a01818e692afb83063fed797f4bd5de658daaa8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1555e38bd3a9c0eb1366945b6cdafff7a16bbc85caadbeeaa990edb52d7b99b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..2eb6767b7ddf0ba5e1a750b5e444b8c443c460f7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec753e28c5473428d0b7eb60c466a25f5e3fe66c67c49b72595ac68f8e0d2f48 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..f41b87afdce1fcc0b7b9eff4e6b9589700d352b3 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77cff55491dcd98e210e6c07623399a07e8a123042ff2c25bda661be0e7132f3 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d2a18d45f008e5eb431505284f12f2f98ec83986 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/trainer_state.json @@ -0,0 +1,9834 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.260632607335338, + "eval_steps": 2000, + "global_step": 14000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0016148240851012292, + "grad_norm": 43.90101623535156, + "learning_rate": 0.0001, + "loss": 307.2084, + "step": 10 + }, + { + "epoch": 0.0032296481702024584, + "grad_norm": 19.106313705444336, + "learning_rate": 0.0001, + "loss": 320.2794, + "step": 20 + }, + { + "epoch": 0.004844472255303688, + "grad_norm": 5.771847248077393, + "learning_rate": 0.0001, + "loss": 247.4977, + "step": 30 + }, + { + "epoch": 0.006459296340404917, + "grad_norm": 2.630659580230713, + "learning_rate": 0.0001, + "loss": 229.9593, + "step": 40 + }, + { + "epoch": 0.008074120425506147, + "grad_norm": 2.487499475479126, + "learning_rate": 0.0001, + "loss": 228.65, + "step": 50 + }, + { + "epoch": 0.009688944510607376, + "grad_norm": 2.078263521194458, + "learning_rate": 0.0001, + "loss": 227.648, + "step": 60 + }, + { + "epoch": 0.011303768595708604, + "grad_norm": 3.105417490005493, + "learning_rate": 0.0001, + "loss": 227.9345, + "step": 70 + }, + { + "epoch": 0.012918592680809833, + "grad_norm": 2.2145848274230957, + "learning_rate": 0.0001, + "loss": 227.3662, + "step": 80 + }, + { + "epoch": 0.014533416765911064, + "grad_norm": 1.9644615650177002, + "learning_rate": 0.0001, + "loss": 226.9145, + "step": 90 + }, + { + "epoch": 0.016148240851012293, + "grad_norm": 1.867382287979126, + "learning_rate": 0.0001, + "loss": 227.5703, + "step": 100 + }, + { + "epoch": 0.017763064936113522, + "grad_norm": 1.4613710641860962, + "learning_rate": 0.0001, + "loss": 227.1107, + "step": 110 + }, + { + "epoch": 0.01937788902121475, + "grad_norm": 2.8953869342803955, + "learning_rate": 0.0001, + "loss": 226.8185, + "step": 120 + }, + { + "epoch": 0.02099271310631598, + "grad_norm": 1.8446482419967651, + "learning_rate": 0.0001, + "loss": 226.9624, + "step": 130 + }, + { + "epoch": 0.02260753719141721, + "grad_norm": 1.3048362731933594, + "learning_rate": 0.0001, + "loss": 226.8404, + "step": 140 + }, + { + "epoch": 0.024222361276518438, + "grad_norm": 1.7100074291229248, + "learning_rate": 0.0001, + "loss": 226.1071, + "step": 150 + }, + { + "epoch": 0.025837185361619667, + "grad_norm": 1.7875330448150635, + "learning_rate": 0.0001, + "loss": 226.0764, + "step": 160 + }, + { + "epoch": 0.0274520094467209, + "grad_norm": 1.7503769397735596, + "learning_rate": 0.0001, + "loss": 226.0772, + "step": 170 + }, + { + "epoch": 0.029066833531822128, + "grad_norm": 2.7083983421325684, + "learning_rate": 0.0001, + "loss": 224.7667, + "step": 180 + }, + { + "epoch": 0.030681657616923357, + "grad_norm": 1.8192371129989624, + "learning_rate": 0.0001, + "loss": 223.6494, + "step": 190 + }, + { + "epoch": 0.032296481702024586, + "grad_norm": 3.210358142852783, + "learning_rate": 0.0001, + "loss": 223.3545, + "step": 200 + }, + { + "epoch": 0.03391130578712581, + "grad_norm": 5.819669246673584, + "learning_rate": 0.0001, + "loss": 222.1732, + "step": 210 + }, + { + "epoch": 0.035526129872227044, + "grad_norm": 4.185793399810791, + "learning_rate": 0.0001, + "loss": 221.6866, + "step": 220 + }, + { + "epoch": 0.03714095395732828, + "grad_norm": 2.7514216899871826, + "learning_rate": 0.0001, + "loss": 221.0769, + "step": 230 + }, + { + "epoch": 0.0387557780424295, + "grad_norm": 1.5478988885879517, + "learning_rate": 0.0001, + "loss": 219.7232, + "step": 240 + }, + { + "epoch": 0.040370602127530734, + "grad_norm": 1.4958223104476929, + "learning_rate": 0.0001, + "loss": 218.5454, + "step": 250 + }, + { + "epoch": 0.04198542621263196, + "grad_norm": 6.36635684967041, + "learning_rate": 0.0001, + "loss": 218.5061, + "step": 260 + }, + { + "epoch": 0.04360025029773319, + "grad_norm": 3.029601812362671, + "learning_rate": 0.0001, + "loss": 216.3466, + "step": 270 + }, + { + "epoch": 0.04521507438283442, + "grad_norm": 2.6436100006103516, + "learning_rate": 0.0001, + "loss": 214.0003, + "step": 280 + }, + { + "epoch": 0.04682989846793565, + "grad_norm": 5.329655647277832, + "learning_rate": 0.0001, + "loss": 212.9169, + "step": 290 + }, + { + "epoch": 0.048444722553036876, + "grad_norm": 4.930875301361084, + "learning_rate": 0.0001, + "loss": 209.7737, + "step": 300 + }, + { + "epoch": 0.05005954663813811, + "grad_norm": 18.902585983276367, + "learning_rate": 0.0001, + "loss": 209.4267, + "step": 310 + }, + { + "epoch": 0.051674370723239334, + "grad_norm": 5.364975929260254, + "learning_rate": 0.0001, + "loss": 206.49, + "step": 320 + }, + { + "epoch": 0.053289194808340566, + "grad_norm": 3.1421146392822266, + "learning_rate": 0.0001, + "loss": 204.8132, + "step": 330 + }, + { + "epoch": 0.0549040188934418, + "grad_norm": 3.399320602416992, + "learning_rate": 0.0001, + "loss": 202.4785, + "step": 340 + }, + { + "epoch": 0.056518842978543024, + "grad_norm": 5.137377738952637, + "learning_rate": 0.0001, + "loss": 199.5143, + "step": 350 + }, + { + "epoch": 0.058133667063644257, + "grad_norm": 5.4967522621154785, + "learning_rate": 0.0001, + "loss": 196.2809, + "step": 360 + }, + { + "epoch": 0.05974849114874548, + "grad_norm": 6.435070037841797, + "learning_rate": 0.0001, + "loss": 196.8831, + "step": 370 + }, + { + "epoch": 0.061363315233846714, + "grad_norm": 5.7334370613098145, + "learning_rate": 0.0001, + "loss": 193.141, + "step": 380 + }, + { + "epoch": 0.06297813931894794, + "grad_norm": 3.6522979736328125, + "learning_rate": 0.0001, + "loss": 191.5819, + "step": 390 + }, + { + "epoch": 0.06459296340404917, + "grad_norm": 2.614051103591919, + "learning_rate": 0.0001, + "loss": 190.046, + "step": 400 + }, + { + "epoch": 0.0662077874891504, + "grad_norm": 4.845924377441406, + "learning_rate": 0.0001, + "loss": 188.7121, + "step": 410 + }, + { + "epoch": 0.06782261157425162, + "grad_norm": 4.319681644439697, + "learning_rate": 0.0001, + "loss": 185.3697, + "step": 420 + }, + { + "epoch": 0.06943743565935286, + "grad_norm": 2.7018792629241943, + "learning_rate": 0.0001, + "loss": 183.5171, + "step": 430 + }, + { + "epoch": 0.07105225974445409, + "grad_norm": 2.6407113075256348, + "learning_rate": 0.0001, + "loss": 181.245, + "step": 440 + }, + { + "epoch": 0.07266708382955532, + "grad_norm": 3.314251184463501, + "learning_rate": 0.0001, + "loss": 179.3874, + "step": 450 + }, + { + "epoch": 0.07428190791465655, + "grad_norm": 2.635136365890503, + "learning_rate": 0.0001, + "loss": 177.8257, + "step": 460 + }, + { + "epoch": 0.07589673199975777, + "grad_norm": 5.227738857269287, + "learning_rate": 0.0001, + "loss": 176.4699, + "step": 470 + }, + { + "epoch": 0.077511556084859, + "grad_norm": 4.998415470123291, + "learning_rate": 0.0001, + "loss": 175.7601, + "step": 480 + }, + { + "epoch": 0.07912638016996024, + "grad_norm": 2.946892261505127, + "learning_rate": 0.0001, + "loss": 173.936, + "step": 490 + }, + { + "epoch": 0.08074120425506147, + "grad_norm": 4.20358943939209, + "learning_rate": 0.0001, + "loss": 172.065, + "step": 500 + }, + { + "epoch": 0.08235602834016269, + "grad_norm": 3.125765562057495, + "learning_rate": 0.0001, + "loss": 171.3727, + "step": 510 + }, + { + "epoch": 0.08397085242526392, + "grad_norm": 2.4870612621307373, + "learning_rate": 0.0001, + "loss": 169.9, + "step": 520 + }, + { + "epoch": 0.08558567651036515, + "grad_norm": 3.917384147644043, + "learning_rate": 0.0001, + "loss": 168.5049, + "step": 530 + }, + { + "epoch": 0.08720050059546638, + "grad_norm": 2.302142858505249, + "learning_rate": 0.0001, + "loss": 167.2027, + "step": 540 + }, + { + "epoch": 0.08881532468056762, + "grad_norm": 4.794841289520264, + "learning_rate": 0.0001, + "loss": 166.0887, + "step": 550 + }, + { + "epoch": 0.09043014876566884, + "grad_norm": 2.200291395187378, + "learning_rate": 0.0001, + "loss": 164.5882, + "step": 560 + }, + { + "epoch": 0.09204497285077007, + "grad_norm": 1.9572436809539795, + "learning_rate": 0.0001, + "loss": 163.3007, + "step": 570 + }, + { + "epoch": 0.0936597969358713, + "grad_norm": 3.443633556365967, + "learning_rate": 0.0001, + "loss": 160.6805, + "step": 580 + }, + { + "epoch": 0.09527462102097253, + "grad_norm": 2.6079020500183105, + "learning_rate": 0.0001, + "loss": 160.3372, + "step": 590 + }, + { + "epoch": 0.09688944510607375, + "grad_norm": 2.4401192665100098, + "learning_rate": 0.0001, + "loss": 160.0956, + "step": 600 + }, + { + "epoch": 0.09850426919117498, + "grad_norm": 2.686101198196411, + "learning_rate": 0.0001, + "loss": 157.6138, + "step": 610 + }, + { + "epoch": 0.10011909327627622, + "grad_norm": 2.594151496887207, + "learning_rate": 0.0001, + "loss": 156.0726, + "step": 620 + }, + { + "epoch": 0.10173391736137745, + "grad_norm": 2.509141683578491, + "learning_rate": 0.0001, + "loss": 155.9923, + "step": 630 + }, + { + "epoch": 0.10334874144647867, + "grad_norm": 2.4077377319335938, + "learning_rate": 0.0001, + "loss": 155.214, + "step": 640 + }, + { + "epoch": 0.1049635655315799, + "grad_norm": 4.083552837371826, + "learning_rate": 0.0001, + "loss": 152.8257, + "step": 650 + }, + { + "epoch": 0.10657838961668113, + "grad_norm": 2.7188713550567627, + "learning_rate": 0.0001, + "loss": 151.875, + "step": 660 + }, + { + "epoch": 0.10819321370178236, + "grad_norm": 2.41287899017334, + "learning_rate": 0.0001, + "loss": 151.935, + "step": 670 + }, + { + "epoch": 0.1098080377868836, + "grad_norm": 3.995708465576172, + "learning_rate": 0.0001, + "loss": 151.0069, + "step": 680 + }, + { + "epoch": 0.11142286187198482, + "grad_norm": 3.217705488204956, + "learning_rate": 0.0001, + "loss": 149.5922, + "step": 690 + }, + { + "epoch": 0.11303768595708605, + "grad_norm": 3.4349842071533203, + "learning_rate": 0.0001, + "loss": 148.0044, + "step": 700 + }, + { + "epoch": 0.11465251004218728, + "grad_norm": 2.106865167617798, + "learning_rate": 0.0001, + "loss": 147.8404, + "step": 710 + }, + { + "epoch": 0.11626733412728851, + "grad_norm": 1.8729510307312012, + "learning_rate": 0.0001, + "loss": 145.1299, + "step": 720 + }, + { + "epoch": 0.11788215821238973, + "grad_norm": 4.14620304107666, + "learning_rate": 0.0001, + "loss": 144.3928, + "step": 730 + }, + { + "epoch": 0.11949698229749096, + "grad_norm": 2.202505350112915, + "learning_rate": 0.0001, + "loss": 144.8357, + "step": 740 + }, + { + "epoch": 0.1211118063825922, + "grad_norm": 2.201470375061035, + "learning_rate": 0.0001, + "loss": 143.2202, + "step": 750 + }, + { + "epoch": 0.12272663046769343, + "grad_norm": 1.5977853536605835, + "learning_rate": 0.0001, + "loss": 143.3537, + "step": 760 + }, + { + "epoch": 0.12434145455279466, + "grad_norm": 2.4403927326202393, + "learning_rate": 0.0001, + "loss": 141.2914, + "step": 770 + }, + { + "epoch": 0.12595627863789588, + "grad_norm": 1.6895846128463745, + "learning_rate": 0.0001, + "loss": 139.9639, + "step": 780 + }, + { + "epoch": 0.12757110272299713, + "grad_norm": 3.4370665550231934, + "learning_rate": 0.0001, + "loss": 140.715, + "step": 790 + }, + { + "epoch": 0.12918592680809834, + "grad_norm": 2.1449077129364014, + "learning_rate": 0.0001, + "loss": 139.2094, + "step": 800 + }, + { + "epoch": 0.13080075089319956, + "grad_norm": 2.4578912258148193, + "learning_rate": 0.0001, + "loss": 137.6724, + "step": 810 + }, + { + "epoch": 0.1324155749783008, + "grad_norm": 1.8994814157485962, + "learning_rate": 0.0001, + "loss": 136.9551, + "step": 820 + }, + { + "epoch": 0.13403039906340203, + "grad_norm": 2.6531031131744385, + "learning_rate": 0.0001, + "loss": 136.5812, + "step": 830 + }, + { + "epoch": 0.13564522314850325, + "grad_norm": 3.8680083751678467, + "learning_rate": 0.0001, + "loss": 134.3655, + "step": 840 + }, + { + "epoch": 0.1372600472336045, + "grad_norm": 1.9594208002090454, + "learning_rate": 0.0001, + "loss": 135.4958, + "step": 850 + }, + { + "epoch": 0.1388748713187057, + "grad_norm": 1.5475000143051147, + "learning_rate": 0.0001, + "loss": 135.2142, + "step": 860 + }, + { + "epoch": 0.14048969540380696, + "grad_norm": 1.6569288969039917, + "learning_rate": 0.0001, + "loss": 133.4162, + "step": 870 + }, + { + "epoch": 0.14210451948890818, + "grad_norm": 1.6724224090576172, + "learning_rate": 0.0001, + "loss": 133.7519, + "step": 880 + }, + { + "epoch": 0.1437193435740094, + "grad_norm": 1.7299500703811646, + "learning_rate": 0.0001, + "loss": 131.471, + "step": 890 + }, + { + "epoch": 0.14533416765911064, + "grad_norm": 1.9444470405578613, + "learning_rate": 0.0001, + "loss": 128.5966, + "step": 900 + }, + { + "epoch": 0.14694899174421186, + "grad_norm": 2.404723644256592, + "learning_rate": 0.0001, + "loss": 129.9824, + "step": 910 + }, + { + "epoch": 0.1485638158293131, + "grad_norm": 2.1493897438049316, + "learning_rate": 0.0001, + "loss": 128.0219, + "step": 920 + }, + { + "epoch": 0.15017863991441432, + "grad_norm": 2.8861117362976074, + "learning_rate": 0.0001, + "loss": 128.5297, + "step": 930 + }, + { + "epoch": 0.15179346399951554, + "grad_norm": 2.546415090560913, + "learning_rate": 0.0001, + "loss": 126.6817, + "step": 940 + }, + { + "epoch": 0.1534082880846168, + "grad_norm": 1.727663278579712, + "learning_rate": 0.0001, + "loss": 125.0413, + "step": 950 + }, + { + "epoch": 0.155023112169718, + "grad_norm": 3.7156920433044434, + "learning_rate": 0.0001, + "loss": 124.8085, + "step": 960 + }, + { + "epoch": 0.15663793625481925, + "grad_norm": 3.0576517581939697, + "learning_rate": 0.0001, + "loss": 125.9107, + "step": 970 + }, + { + "epoch": 0.15825276033992047, + "grad_norm": 1.9951574802398682, + "learning_rate": 0.0001, + "loss": 123.5359, + "step": 980 + }, + { + "epoch": 0.1598675844250217, + "grad_norm": 1.4620885848999023, + "learning_rate": 0.0001, + "loss": 121.9363, + "step": 990 + }, + { + "epoch": 0.16148240851012294, + "grad_norm": 1.7640169858932495, + "learning_rate": 0.0001, + "loss": 121.1764, + "step": 1000 + }, + { + "epoch": 0.16309723259522416, + "grad_norm": 2.921750545501709, + "learning_rate": 0.0001, + "loss": 119.6931, + "step": 1010 + }, + { + "epoch": 0.16471205668032537, + "grad_norm": 1.5623588562011719, + "learning_rate": 0.0001, + "loss": 119.097, + "step": 1020 + }, + { + "epoch": 0.16632688076542662, + "grad_norm": 2.2386627197265625, + "learning_rate": 0.0001, + "loss": 120.9224, + "step": 1030 + }, + { + "epoch": 0.16794170485052784, + "grad_norm": 1.3213791847229004, + "learning_rate": 0.0001, + "loss": 119.5649, + "step": 1040 + }, + { + "epoch": 0.16955652893562909, + "grad_norm": 1.5255440473556519, + "learning_rate": 0.0001, + "loss": 118.1524, + "step": 1050 + }, + { + "epoch": 0.1711713530207303, + "grad_norm": 1.6038514375686646, + "learning_rate": 0.0001, + "loss": 118.6307, + "step": 1060 + }, + { + "epoch": 0.17278617710583152, + "grad_norm": 2.356739044189453, + "learning_rate": 0.0001, + "loss": 116.8472, + "step": 1070 + }, + { + "epoch": 0.17440100119093277, + "grad_norm": 2.5326743125915527, + "learning_rate": 0.0001, + "loss": 115.6625, + "step": 1080 + }, + { + "epoch": 0.176015825276034, + "grad_norm": 1.2823396921157837, + "learning_rate": 0.0001, + "loss": 115.3312, + "step": 1090 + }, + { + "epoch": 0.17763064936113523, + "grad_norm": 1.981788158416748, + "learning_rate": 0.0001, + "loss": 116.2257, + "step": 1100 + }, + { + "epoch": 0.17924547344623645, + "grad_norm": 1.540440559387207, + "learning_rate": 0.0001, + "loss": 114.5031, + "step": 1110 + }, + { + "epoch": 0.18086029753133767, + "grad_norm": 1.7667800188064575, + "learning_rate": 0.0001, + "loss": 113.7663, + "step": 1120 + }, + { + "epoch": 0.18247512161643892, + "grad_norm": 2.2793102264404297, + "learning_rate": 0.0001, + "loss": 115.219, + "step": 1130 + }, + { + "epoch": 0.18408994570154014, + "grad_norm": 2.132046699523926, + "learning_rate": 0.0001, + "loss": 112.8482, + "step": 1140 + }, + { + "epoch": 0.18570476978664135, + "grad_norm": 1.3153988122940063, + "learning_rate": 0.0001, + "loss": 111.4546, + "step": 1150 + }, + { + "epoch": 0.1873195938717426, + "grad_norm": 1.5246292352676392, + "learning_rate": 0.0001, + "loss": 112.4348, + "step": 1160 + }, + { + "epoch": 0.18893441795684382, + "grad_norm": 1.365096092224121, + "learning_rate": 0.0001, + "loss": 109.755, + "step": 1170 + }, + { + "epoch": 0.19054924204194507, + "grad_norm": 1.2777953147888184, + "learning_rate": 0.0001, + "loss": 111.0318, + "step": 1180 + }, + { + "epoch": 0.19216406612704628, + "grad_norm": 1.8672829866409302, + "learning_rate": 0.0001, + "loss": 109.1648, + "step": 1190 + }, + { + "epoch": 0.1937788902121475, + "grad_norm": 1.709987759590149, + "learning_rate": 0.0001, + "loss": 108.1616, + "step": 1200 + }, + { + "epoch": 0.19539371429724875, + "grad_norm": 1.176277995109558, + "learning_rate": 0.0001, + "loss": 109.3591, + "step": 1210 + }, + { + "epoch": 0.19700853838234997, + "grad_norm": 1.4181180000305176, + "learning_rate": 0.0001, + "loss": 110.0216, + "step": 1220 + }, + { + "epoch": 0.19862336246745121, + "grad_norm": 1.3246116638183594, + "learning_rate": 0.0001, + "loss": 107.9769, + "step": 1230 + }, + { + "epoch": 0.20023818655255243, + "grad_norm": 1.2501498460769653, + "learning_rate": 0.0001, + "loss": 108.2601, + "step": 1240 + }, + { + "epoch": 0.20185301063765365, + "grad_norm": 1.2277404069900513, + "learning_rate": 0.0001, + "loss": 107.1258, + "step": 1250 + }, + { + "epoch": 0.2034678347227549, + "grad_norm": 1.8064894676208496, + "learning_rate": 0.0001, + "loss": 105.6278, + "step": 1260 + }, + { + "epoch": 0.20508265880785612, + "grad_norm": 1.708897352218628, + "learning_rate": 0.0001, + "loss": 106.9269, + "step": 1270 + }, + { + "epoch": 0.20669748289295733, + "grad_norm": 1.4507478475570679, + "learning_rate": 0.0001, + "loss": 106.9236, + "step": 1280 + }, + { + "epoch": 0.20831230697805858, + "grad_norm": 1.4904310703277588, + "learning_rate": 0.0001, + "loss": 106.0653, + "step": 1290 + }, + { + "epoch": 0.2099271310631598, + "grad_norm": 1.2637165784835815, + "learning_rate": 0.0001, + "loss": 104.8665, + "step": 1300 + }, + { + "epoch": 0.21154195514826105, + "grad_norm": 1.4085133075714111, + "learning_rate": 0.0001, + "loss": 104.8891, + "step": 1310 + }, + { + "epoch": 0.21315677923336226, + "grad_norm": 1.5710629224777222, + "learning_rate": 0.0001, + "loss": 104.3009, + "step": 1320 + }, + { + "epoch": 0.21477160331846348, + "grad_norm": 1.3634663820266724, + "learning_rate": 0.0001, + "loss": 104.4545, + "step": 1330 + }, + { + "epoch": 0.21638642740356473, + "grad_norm": 1.5565859079360962, + "learning_rate": 0.0001, + "loss": 103.6205, + "step": 1340 + }, + { + "epoch": 0.21800125148866595, + "grad_norm": 1.4491888284683228, + "learning_rate": 0.0001, + "loss": 104.0892, + "step": 1350 + }, + { + "epoch": 0.2196160755737672, + "grad_norm": 1.7609063386917114, + "learning_rate": 0.0001, + "loss": 104.5177, + "step": 1360 + }, + { + "epoch": 0.2212308996588684, + "grad_norm": 1.6723753213882446, + "learning_rate": 0.0001, + "loss": 104.5552, + "step": 1370 + }, + { + "epoch": 0.22284572374396963, + "grad_norm": 1.3781665563583374, + "learning_rate": 0.0001, + "loss": 102.1022, + "step": 1380 + }, + { + "epoch": 0.22446054782907088, + "grad_norm": 1.5397366285324097, + "learning_rate": 0.0001, + "loss": 102.5882, + "step": 1390 + }, + { + "epoch": 0.2260753719141721, + "grad_norm": 1.4567879438400269, + "learning_rate": 0.0001, + "loss": 101.3781, + "step": 1400 + }, + { + "epoch": 0.22769019599927334, + "grad_norm": 1.2882301807403564, + "learning_rate": 0.0001, + "loss": 100.9378, + "step": 1410 + }, + { + "epoch": 0.22930502008437456, + "grad_norm": 1.4510424137115479, + "learning_rate": 0.0001, + "loss": 101.5383, + "step": 1420 + }, + { + "epoch": 0.23091984416947578, + "grad_norm": 2.8331449031829834, + "learning_rate": 0.0001, + "loss": 100.2773, + "step": 1430 + }, + { + "epoch": 0.23253466825457703, + "grad_norm": 1.363616943359375, + "learning_rate": 0.0001, + "loss": 101.646, + "step": 1440 + }, + { + "epoch": 0.23414949233967824, + "grad_norm": 1.2184548377990723, + "learning_rate": 0.0001, + "loss": 101.1133, + "step": 1450 + }, + { + "epoch": 0.23576431642477946, + "grad_norm": 2.235656976699829, + "learning_rate": 0.0001, + "loss": 100.2044, + "step": 1460 + }, + { + "epoch": 0.2373791405098807, + "grad_norm": 1.2334418296813965, + "learning_rate": 0.0001, + "loss": 99.4031, + "step": 1470 + }, + { + "epoch": 0.23899396459498193, + "grad_norm": 1.2394684553146362, + "learning_rate": 0.0001, + "loss": 101.1247, + "step": 1480 + }, + { + "epoch": 0.24060878868008317, + "grad_norm": 1.5218186378479004, + "learning_rate": 0.0001, + "loss": 100.4413, + "step": 1490 + }, + { + "epoch": 0.2422236127651844, + "grad_norm": 1.2112364768981934, + "learning_rate": 0.0001, + "loss": 99.8722, + "step": 1500 + }, + { + "epoch": 0.2438384368502856, + "grad_norm": 1.2277610301971436, + "learning_rate": 0.0001, + "loss": 99.0276, + "step": 1510 + }, + { + "epoch": 0.24545326093538686, + "grad_norm": 1.1005053520202637, + "learning_rate": 0.0001, + "loss": 98.6764, + "step": 1520 + }, + { + "epoch": 0.24706808502048808, + "grad_norm": 1.0754456520080566, + "learning_rate": 0.0001, + "loss": 97.8968, + "step": 1530 + }, + { + "epoch": 0.24868290910558932, + "grad_norm": 1.0695950984954834, + "learning_rate": 0.0001, + "loss": 98.4032, + "step": 1540 + }, + { + "epoch": 0.25029773319069054, + "grad_norm": 1.5098627805709839, + "learning_rate": 0.0001, + "loss": 97.9391, + "step": 1550 + }, + { + "epoch": 0.25191255727579176, + "grad_norm": 1.006681203842163, + "learning_rate": 0.0001, + "loss": 96.5525, + "step": 1560 + }, + { + "epoch": 0.253527381360893, + "grad_norm": 1.3987332582473755, + "learning_rate": 0.0001, + "loss": 98.383, + "step": 1570 + }, + { + "epoch": 0.25514220544599425, + "grad_norm": 1.290887713432312, + "learning_rate": 0.0001, + "loss": 99.0901, + "step": 1580 + }, + { + "epoch": 0.25675702953109547, + "grad_norm": 0.991177499294281, + "learning_rate": 0.0001, + "loss": 97.7093, + "step": 1590 + }, + { + "epoch": 0.2583718536161967, + "grad_norm": 1.147647738456726, + "learning_rate": 0.0001, + "loss": 96.9181, + "step": 1600 + }, + { + "epoch": 0.2599866777012979, + "grad_norm": 1.1756514310836792, + "learning_rate": 0.0001, + "loss": 98.3935, + "step": 1610 + }, + { + "epoch": 0.2616015017863991, + "grad_norm": 1.1708602905273438, + "learning_rate": 0.0001, + "loss": 96.715, + "step": 1620 + }, + { + "epoch": 0.2632163258715004, + "grad_norm": 1.4577715396881104, + "learning_rate": 0.0001, + "loss": 97.2971, + "step": 1630 + }, + { + "epoch": 0.2648311499566016, + "grad_norm": 1.1172800064086914, + "learning_rate": 0.0001, + "loss": 97.6494, + "step": 1640 + }, + { + "epoch": 0.26644597404170284, + "grad_norm": 1.1491258144378662, + "learning_rate": 0.0001, + "loss": 97.5503, + "step": 1650 + }, + { + "epoch": 0.26806079812680406, + "grad_norm": 1.2188555002212524, + "learning_rate": 0.0001, + "loss": 95.3423, + "step": 1660 + }, + { + "epoch": 0.2696756222119053, + "grad_norm": 1.8595614433288574, + "learning_rate": 0.0001, + "loss": 96.0112, + "step": 1670 + }, + { + "epoch": 0.2712904462970065, + "grad_norm": 1.1887950897216797, + "learning_rate": 0.0001, + "loss": 95.4939, + "step": 1680 + }, + { + "epoch": 0.27290527038210777, + "grad_norm": 1.1777839660644531, + "learning_rate": 0.0001, + "loss": 95.9619, + "step": 1690 + }, + { + "epoch": 0.274520094467209, + "grad_norm": 1.1241233348846436, + "learning_rate": 0.0001, + "loss": 96.2834, + "step": 1700 + }, + { + "epoch": 0.2761349185523102, + "grad_norm": 1.0900121927261353, + "learning_rate": 0.0001, + "loss": 94.7518, + "step": 1710 + }, + { + "epoch": 0.2777497426374114, + "grad_norm": 1.0615205764770508, + "learning_rate": 0.0001, + "loss": 95.8032, + "step": 1720 + }, + { + "epoch": 0.27936456672251264, + "grad_norm": 1.2626168727874756, + "learning_rate": 0.0001, + "loss": 93.9454, + "step": 1730 + }, + { + "epoch": 0.2809793908076139, + "grad_norm": 0.8869941830635071, + "learning_rate": 0.0001, + "loss": 95.4085, + "step": 1740 + }, + { + "epoch": 0.28259421489271513, + "grad_norm": 1.0350773334503174, + "learning_rate": 0.0001, + "loss": 95.7954, + "step": 1750 + }, + { + "epoch": 0.28420903897781635, + "grad_norm": 1.364768147468567, + "learning_rate": 0.0001, + "loss": 93.6447, + "step": 1760 + }, + { + "epoch": 0.28582386306291757, + "grad_norm": 1.0115883350372314, + "learning_rate": 0.0001, + "loss": 93.7905, + "step": 1770 + }, + { + "epoch": 0.2874386871480188, + "grad_norm": 0.9393435120582581, + "learning_rate": 0.0001, + "loss": 94.3108, + "step": 1780 + }, + { + "epoch": 0.28905351123312006, + "grad_norm": 1.0769627094268799, + "learning_rate": 0.0001, + "loss": 92.566, + "step": 1790 + }, + { + "epoch": 0.2906683353182213, + "grad_norm": 2.526201009750366, + "learning_rate": 0.0001, + "loss": 92.8161, + "step": 1800 + }, + { + "epoch": 0.2922831594033225, + "grad_norm": 1.3066468238830566, + "learning_rate": 0.0001, + "loss": 92.0459, + "step": 1810 + }, + { + "epoch": 0.2938979834884237, + "grad_norm": 1.121713638305664, + "learning_rate": 0.0001, + "loss": 93.5032, + "step": 1820 + }, + { + "epoch": 0.29551280757352494, + "grad_norm": 1.3744316101074219, + "learning_rate": 0.0001, + "loss": 94.2623, + "step": 1830 + }, + { + "epoch": 0.2971276316586262, + "grad_norm": 1.1561540365219116, + "learning_rate": 0.0001, + "loss": 94.1611, + "step": 1840 + }, + { + "epoch": 0.29874245574372743, + "grad_norm": 1.3947464227676392, + "learning_rate": 0.0001, + "loss": 93.5962, + "step": 1850 + }, + { + "epoch": 0.30035727982882865, + "grad_norm": 1.0011566877365112, + "learning_rate": 0.0001, + "loss": 92.0199, + "step": 1860 + }, + { + "epoch": 0.30197210391392987, + "grad_norm": 0.9978747963905334, + "learning_rate": 0.0001, + "loss": 91.0945, + "step": 1870 + }, + { + "epoch": 0.3035869279990311, + "grad_norm": 0.92522132396698, + "learning_rate": 0.0001, + "loss": 93.274, + "step": 1880 + }, + { + "epoch": 0.30520175208413236, + "grad_norm": 1.0089120864868164, + "learning_rate": 0.0001, + "loss": 91.9308, + "step": 1890 + }, + { + "epoch": 0.3068165761692336, + "grad_norm": 1.0535134077072144, + "learning_rate": 0.0001, + "loss": 92.563, + "step": 1900 + }, + { + "epoch": 0.3084314002543348, + "grad_norm": 0.9782275557518005, + "learning_rate": 0.0001, + "loss": 90.3486, + "step": 1910 + }, + { + "epoch": 0.310046224339436, + "grad_norm": 1.1433162689208984, + "learning_rate": 0.0001, + "loss": 90.4977, + "step": 1920 + }, + { + "epoch": 0.31166104842453723, + "grad_norm": 0.98392653465271, + "learning_rate": 0.0001, + "loss": 92.6529, + "step": 1930 + }, + { + "epoch": 0.3132758725096385, + "grad_norm": 0.8538148403167725, + "learning_rate": 0.0001, + "loss": 90.6859, + "step": 1940 + }, + { + "epoch": 0.3148906965947397, + "grad_norm": 1.019405484199524, + "learning_rate": 0.0001, + "loss": 91.0028, + "step": 1950 + }, + { + "epoch": 0.31650552067984095, + "grad_norm": 0.9828067421913147, + "learning_rate": 0.0001, + "loss": 89.9826, + "step": 1960 + }, + { + "epoch": 0.31812034476494216, + "grad_norm": 0.9725450277328491, + "learning_rate": 0.0001, + "loss": 91.7342, + "step": 1970 + }, + { + "epoch": 0.3197351688500434, + "grad_norm": 1.0139408111572266, + "learning_rate": 0.0001, + "loss": 90.5362, + "step": 1980 + }, + { + "epoch": 0.3213499929351446, + "grad_norm": 1.2493853569030762, + "learning_rate": 0.0001, + "loss": 92.7222, + "step": 1990 + }, + { + "epoch": 0.3229648170202459, + "grad_norm": 1.0152915716171265, + "learning_rate": 0.0001, + "loss": 92.2271, + "step": 2000 + }, + { + "epoch": 0.3245796411053471, + "grad_norm": 0.9745819568634033, + "learning_rate": 0.0001, + "loss": 88.7034, + "step": 2010 + }, + { + "epoch": 0.3261944651904483, + "grad_norm": 1.009624719619751, + "learning_rate": 0.0001, + "loss": 90.7428, + "step": 2020 + }, + { + "epoch": 0.32780928927554953, + "grad_norm": 0.910858690738678, + "learning_rate": 0.0001, + "loss": 89.8199, + "step": 2030 + }, + { + "epoch": 0.32942411336065075, + "grad_norm": 1.1745878458023071, + "learning_rate": 0.0001, + "loss": 90.8267, + "step": 2040 + }, + { + "epoch": 0.331038937445752, + "grad_norm": 0.9409273266792297, + "learning_rate": 0.0001, + "loss": 90.2937, + "step": 2050 + }, + { + "epoch": 0.33265376153085324, + "grad_norm": 1.4152942895889282, + "learning_rate": 0.0001, + "loss": 90.3965, + "step": 2060 + }, + { + "epoch": 0.33426858561595446, + "grad_norm": 0.9529693722724915, + "learning_rate": 0.0001, + "loss": 90.4739, + "step": 2070 + }, + { + "epoch": 0.3358834097010557, + "grad_norm": 0.8531096577644348, + "learning_rate": 0.0001, + "loss": 90.9301, + "step": 2080 + }, + { + "epoch": 0.3374982337861569, + "grad_norm": 0.9314167499542236, + "learning_rate": 0.0001, + "loss": 90.4235, + "step": 2090 + }, + { + "epoch": 0.33911305787125817, + "grad_norm": 0.9253250360488892, + "learning_rate": 0.0001, + "loss": 88.1893, + "step": 2100 + }, + { + "epoch": 0.3407278819563594, + "grad_norm": 1.773053526878357, + "learning_rate": 0.0001, + "loss": 88.3417, + "step": 2110 + }, + { + "epoch": 0.3423427060414606, + "grad_norm": 0.9432951211929321, + "learning_rate": 0.0001, + "loss": 89.7141, + "step": 2120 + }, + { + "epoch": 0.34395753012656183, + "grad_norm": 3.8847527503967285, + "learning_rate": 0.0001, + "loss": 89.1009, + "step": 2130 + }, + { + "epoch": 0.34557235421166305, + "grad_norm": 1.0309821367263794, + "learning_rate": 0.0001, + "loss": 89.2965, + "step": 2140 + }, + { + "epoch": 0.3471871782967643, + "grad_norm": 0.9590495824813843, + "learning_rate": 0.0001, + "loss": 89.1734, + "step": 2150 + }, + { + "epoch": 0.34880200238186554, + "grad_norm": 0.9943560361862183, + "learning_rate": 0.0001, + "loss": 85.425, + "step": 2160 + }, + { + "epoch": 0.35041682646696676, + "grad_norm": 1.3322339057922363, + "learning_rate": 0.0001, + "loss": 89.7791, + "step": 2170 + }, + { + "epoch": 0.352031650552068, + "grad_norm": 1.0212253332138062, + "learning_rate": 0.0001, + "loss": 89.3771, + "step": 2180 + }, + { + "epoch": 0.3536464746371692, + "grad_norm": 1.0446089506149292, + "learning_rate": 0.0001, + "loss": 88.6894, + "step": 2190 + }, + { + "epoch": 0.35526129872227047, + "grad_norm": 1.05767822265625, + "learning_rate": 0.0001, + "loss": 89.5341, + "step": 2200 + }, + { + "epoch": 0.3568761228073717, + "grad_norm": 1.0373681783676147, + "learning_rate": 0.0001, + "loss": 89.7173, + "step": 2210 + }, + { + "epoch": 0.3584909468924729, + "grad_norm": 0.9098039269447327, + "learning_rate": 0.0001, + "loss": 89.2116, + "step": 2220 + }, + { + "epoch": 0.3601057709775741, + "grad_norm": 1.028130292892456, + "learning_rate": 0.0001, + "loss": 88.6627, + "step": 2230 + }, + { + "epoch": 0.36172059506267534, + "grad_norm": 1.0295171737670898, + "learning_rate": 0.0001, + "loss": 89.608, + "step": 2240 + }, + { + "epoch": 0.36333541914777656, + "grad_norm": 0.9720722436904907, + "learning_rate": 0.0001, + "loss": 88.3624, + "step": 2250 + }, + { + "epoch": 0.36495024323287784, + "grad_norm": 0.839654803276062, + "learning_rate": 0.0001, + "loss": 87.2983, + "step": 2260 + }, + { + "epoch": 0.36656506731797905, + "grad_norm": 1.1600219011306763, + "learning_rate": 0.0001, + "loss": 86.6004, + "step": 2270 + }, + { + "epoch": 0.3681798914030803, + "grad_norm": 1.0513924360275269, + "learning_rate": 0.0001, + "loss": 86.6062, + "step": 2280 + }, + { + "epoch": 0.3697947154881815, + "grad_norm": 0.9728474617004395, + "learning_rate": 0.0001, + "loss": 86.1856, + "step": 2290 + }, + { + "epoch": 0.3714095395732827, + "grad_norm": 1.0196386575698853, + "learning_rate": 0.0001, + "loss": 87.6088, + "step": 2300 + }, + { + "epoch": 0.373024363658384, + "grad_norm": 1.0439444780349731, + "learning_rate": 0.0001, + "loss": 85.2374, + "step": 2310 + }, + { + "epoch": 0.3746391877434852, + "grad_norm": 1.0020275115966797, + "learning_rate": 0.0001, + "loss": 86.0596, + "step": 2320 + }, + { + "epoch": 0.3762540118285864, + "grad_norm": 1.0107691287994385, + "learning_rate": 0.0001, + "loss": 86.9513, + "step": 2330 + }, + { + "epoch": 0.37786883591368764, + "grad_norm": 1.680018663406372, + "learning_rate": 0.0001, + "loss": 87.8885, + "step": 2340 + }, + { + "epoch": 0.37948365999878886, + "grad_norm": 0.9181709885597229, + "learning_rate": 0.0001, + "loss": 86.7662, + "step": 2350 + }, + { + "epoch": 0.38109848408389013, + "grad_norm": 0.9598278403282166, + "learning_rate": 0.0001, + "loss": 88.3775, + "step": 2360 + }, + { + "epoch": 0.38271330816899135, + "grad_norm": 1.0707905292510986, + "learning_rate": 0.0001, + "loss": 86.0997, + "step": 2370 + }, + { + "epoch": 0.38432813225409257, + "grad_norm": 1.125797152519226, + "learning_rate": 0.0001, + "loss": 85.8117, + "step": 2380 + }, + { + "epoch": 0.3859429563391938, + "grad_norm": 1.0251439809799194, + "learning_rate": 0.0001, + "loss": 86.0806, + "step": 2390 + }, + { + "epoch": 0.387557780424295, + "grad_norm": 1.0193125009536743, + "learning_rate": 0.0001, + "loss": 87.264, + "step": 2400 + }, + { + "epoch": 0.3891726045093963, + "grad_norm": 0.9309751391410828, + "learning_rate": 0.0001, + "loss": 85.9187, + "step": 2410 + }, + { + "epoch": 0.3907874285944975, + "grad_norm": 1.1025906801223755, + "learning_rate": 0.0001, + "loss": 86.4365, + "step": 2420 + }, + { + "epoch": 0.3924022526795987, + "grad_norm": 0.9880627393722534, + "learning_rate": 0.0001, + "loss": 86.2303, + "step": 2430 + }, + { + "epoch": 0.39401707676469994, + "grad_norm": 0.9520611763000488, + "learning_rate": 0.0001, + "loss": 84.9035, + "step": 2440 + }, + { + "epoch": 0.39563190084980115, + "grad_norm": 0.8641896843910217, + "learning_rate": 0.0001, + "loss": 85.4754, + "step": 2450 + }, + { + "epoch": 0.39724672493490243, + "grad_norm": 1.1162904500961304, + "learning_rate": 0.0001, + "loss": 85.6767, + "step": 2460 + }, + { + "epoch": 0.39886154902000365, + "grad_norm": 0.9073636531829834, + "learning_rate": 0.0001, + "loss": 85.4198, + "step": 2470 + }, + { + "epoch": 0.40047637310510487, + "grad_norm": 1.1135166883468628, + "learning_rate": 0.0001, + "loss": 86.0884, + "step": 2480 + }, + { + "epoch": 0.4020911971902061, + "grad_norm": 0.9273980259895325, + "learning_rate": 0.0001, + "loss": 85.5473, + "step": 2490 + }, + { + "epoch": 0.4037060212753073, + "grad_norm": 0.8653831481933594, + "learning_rate": 0.0001, + "loss": 87.279, + "step": 2500 + }, + { + "epoch": 0.4053208453604086, + "grad_norm": 0.8535159826278687, + "learning_rate": 0.0001, + "loss": 86.7072, + "step": 2510 + }, + { + "epoch": 0.4069356694455098, + "grad_norm": 0.8864684700965881, + "learning_rate": 0.0001, + "loss": 86.441, + "step": 2520 + }, + { + "epoch": 0.408550493530611, + "grad_norm": 0.8042080998420715, + "learning_rate": 0.0001, + "loss": 83.1706, + "step": 2530 + }, + { + "epoch": 0.41016531761571223, + "grad_norm": 1.0486373901367188, + "learning_rate": 0.0001, + "loss": 83.54, + "step": 2540 + }, + { + "epoch": 0.41178014170081345, + "grad_norm": 0.8087959289550781, + "learning_rate": 0.0001, + "loss": 85.7615, + "step": 2550 + }, + { + "epoch": 0.41339496578591467, + "grad_norm": 1.5032018423080444, + "learning_rate": 0.0001, + "loss": 85.7368, + "step": 2560 + }, + { + "epoch": 0.41500978987101594, + "grad_norm": 1.0786182880401611, + "learning_rate": 0.0001, + "loss": 86.3484, + "step": 2570 + }, + { + "epoch": 0.41662461395611716, + "grad_norm": 0.8744294047355652, + "learning_rate": 0.0001, + "loss": 83.5945, + "step": 2580 + }, + { + "epoch": 0.4182394380412184, + "grad_norm": 0.7840721607208252, + "learning_rate": 0.0001, + "loss": 86.3831, + "step": 2590 + }, + { + "epoch": 0.4198542621263196, + "grad_norm": 0.7977836728096008, + "learning_rate": 0.0001, + "loss": 83.6314, + "step": 2600 + }, + { + "epoch": 0.4214690862114208, + "grad_norm": 0.892484188079834, + "learning_rate": 0.0001, + "loss": 86.4223, + "step": 2610 + }, + { + "epoch": 0.4230839102965221, + "grad_norm": 0.912995457649231, + "learning_rate": 0.0001, + "loss": 83.8971, + "step": 2620 + }, + { + "epoch": 0.4246987343816233, + "grad_norm": 0.9566861987113953, + "learning_rate": 0.0001, + "loss": 85.291, + "step": 2630 + }, + { + "epoch": 0.42631355846672453, + "grad_norm": 0.8324154615402222, + "learning_rate": 0.0001, + "loss": 84.6233, + "step": 2640 + }, + { + "epoch": 0.42792838255182575, + "grad_norm": 0.9490360617637634, + "learning_rate": 0.0001, + "loss": 81.799, + "step": 2650 + }, + { + "epoch": 0.42954320663692697, + "grad_norm": 0.9744566679000854, + "learning_rate": 0.0001, + "loss": 85.4837, + "step": 2660 + }, + { + "epoch": 0.43115803072202824, + "grad_norm": 0.941614031791687, + "learning_rate": 0.0001, + "loss": 82.838, + "step": 2670 + }, + { + "epoch": 0.43277285480712946, + "grad_norm": 1.338364601135254, + "learning_rate": 0.0001, + "loss": 84.4224, + "step": 2680 + }, + { + "epoch": 0.4343876788922307, + "grad_norm": 0.8643816113471985, + "learning_rate": 0.0001, + "loss": 84.3959, + "step": 2690 + }, + { + "epoch": 0.4360025029773319, + "grad_norm": 1.0231473445892334, + "learning_rate": 0.0001, + "loss": 82.3144, + "step": 2700 + }, + { + "epoch": 0.4376173270624331, + "grad_norm": 0.9089650511741638, + "learning_rate": 0.0001, + "loss": 83.0326, + "step": 2710 + }, + { + "epoch": 0.4392321511475344, + "grad_norm": 0.893628716468811, + "learning_rate": 0.0001, + "loss": 83.0981, + "step": 2720 + }, + { + "epoch": 0.4408469752326356, + "grad_norm": 0.869076132774353, + "learning_rate": 0.0001, + "loss": 84.2571, + "step": 2730 + }, + { + "epoch": 0.4424617993177368, + "grad_norm": 0.7964235544204712, + "learning_rate": 0.0001, + "loss": 83.3329, + "step": 2740 + }, + { + "epoch": 0.44407662340283804, + "grad_norm": 3.1645138263702393, + "learning_rate": 0.0001, + "loss": 83.7232, + "step": 2750 + }, + { + "epoch": 0.44569144748793926, + "grad_norm": 0.934644877910614, + "learning_rate": 0.0001, + "loss": 85.0688, + "step": 2760 + }, + { + "epoch": 0.44730627157304054, + "grad_norm": 0.8120711445808411, + "learning_rate": 0.0001, + "loss": 82.7706, + "step": 2770 + }, + { + "epoch": 0.44892109565814176, + "grad_norm": 0.8161435127258301, + "learning_rate": 0.0001, + "loss": 84.4826, + "step": 2780 + }, + { + "epoch": 0.450535919743243, + "grad_norm": 0.8659493327140808, + "learning_rate": 0.0001, + "loss": 82.6222, + "step": 2790 + }, + { + "epoch": 0.4521507438283442, + "grad_norm": 0.9306702613830566, + "learning_rate": 0.0001, + "loss": 83.5897, + "step": 2800 + }, + { + "epoch": 0.4537655679134454, + "grad_norm": 2.2544431686401367, + "learning_rate": 0.0001, + "loss": 82.9603, + "step": 2810 + }, + { + "epoch": 0.4553803919985467, + "grad_norm": 0.8868395686149597, + "learning_rate": 0.0001, + "loss": 82.6674, + "step": 2820 + }, + { + "epoch": 0.4569952160836479, + "grad_norm": 0.9336455464363098, + "learning_rate": 0.0001, + "loss": 84.6974, + "step": 2830 + }, + { + "epoch": 0.4586100401687491, + "grad_norm": 0.8814165592193604, + "learning_rate": 0.0001, + "loss": 83.9329, + "step": 2840 + }, + { + "epoch": 0.46022486425385034, + "grad_norm": 0.8125660419464111, + "learning_rate": 0.0001, + "loss": 84.6168, + "step": 2850 + }, + { + "epoch": 0.46183968833895156, + "grad_norm": 0.9447190165519714, + "learning_rate": 0.0001, + "loss": 82.4098, + "step": 2860 + }, + { + "epoch": 0.4634545124240528, + "grad_norm": 0.8080969452857971, + "learning_rate": 0.0001, + "loss": 83.4076, + "step": 2870 + }, + { + "epoch": 0.46506933650915405, + "grad_norm": 1.0586694478988647, + "learning_rate": 0.0001, + "loss": 83.3295, + "step": 2880 + }, + { + "epoch": 0.46668416059425527, + "grad_norm": 0.8207664489746094, + "learning_rate": 0.0001, + "loss": 84.5056, + "step": 2890 + }, + { + "epoch": 0.4682989846793565, + "grad_norm": 1.0647571086883545, + "learning_rate": 0.0001, + "loss": 83.0766, + "step": 2900 + }, + { + "epoch": 0.4699138087644577, + "grad_norm": 0.9416915774345398, + "learning_rate": 0.0001, + "loss": 83.8166, + "step": 2910 + }, + { + "epoch": 0.4715286328495589, + "grad_norm": 0.7479362487792969, + "learning_rate": 0.0001, + "loss": 82.7507, + "step": 2920 + }, + { + "epoch": 0.4731434569346602, + "grad_norm": 1.0775514841079712, + "learning_rate": 0.0001, + "loss": 84.4641, + "step": 2930 + }, + { + "epoch": 0.4747582810197614, + "grad_norm": 0.8736841082572937, + "learning_rate": 0.0001, + "loss": 82.0451, + "step": 2940 + }, + { + "epoch": 0.47637310510486264, + "grad_norm": 0.9500495791435242, + "learning_rate": 0.0001, + "loss": 82.6425, + "step": 2950 + }, + { + "epoch": 0.47798792918996386, + "grad_norm": 0.8551210165023804, + "learning_rate": 0.0001, + "loss": 81.1971, + "step": 2960 + }, + { + "epoch": 0.4796027532750651, + "grad_norm": 0.8149814605712891, + "learning_rate": 0.0001, + "loss": 82.7919, + "step": 2970 + }, + { + "epoch": 0.48121757736016635, + "grad_norm": 0.9012249112129211, + "learning_rate": 0.0001, + "loss": 81.975, + "step": 2980 + }, + { + "epoch": 0.48283240144526757, + "grad_norm": 1.3784620761871338, + "learning_rate": 0.0001, + "loss": 81.541, + "step": 2990 + }, + { + "epoch": 0.4844472255303688, + "grad_norm": 0.7921801209449768, + "learning_rate": 0.0001, + "loss": 83.6215, + "step": 3000 + }, + { + "epoch": 0.48606204961547, + "grad_norm": 0.9051580429077148, + "learning_rate": 0.0001, + "loss": 82.4048, + "step": 3010 + }, + { + "epoch": 0.4876768737005712, + "grad_norm": 1.0717192888259888, + "learning_rate": 0.0001, + "loss": 82.4079, + "step": 3020 + }, + { + "epoch": 0.4892916977856725, + "grad_norm": 0.8971781730651855, + "learning_rate": 0.0001, + "loss": 83.3668, + "step": 3030 + }, + { + "epoch": 0.4909065218707737, + "grad_norm": 0.9635320901870728, + "learning_rate": 0.0001, + "loss": 82.8244, + "step": 3040 + }, + { + "epoch": 0.49252134595587493, + "grad_norm": 1.1009949445724487, + "learning_rate": 0.0001, + "loss": 82.2174, + "step": 3050 + }, + { + "epoch": 0.49413617004097615, + "grad_norm": 0.9805473685264587, + "learning_rate": 0.0001, + "loss": 83.7897, + "step": 3060 + }, + { + "epoch": 0.49575099412607737, + "grad_norm": 1.2107361555099487, + "learning_rate": 0.0001, + "loss": 82.5108, + "step": 3070 + }, + { + "epoch": 0.49736581821117865, + "grad_norm": 0.9171829223632812, + "learning_rate": 0.0001, + "loss": 80.9126, + "step": 3080 + }, + { + "epoch": 0.49898064229627986, + "grad_norm": 0.8093295097351074, + "learning_rate": 0.0001, + "loss": 83.8147, + "step": 3090 + }, + { + "epoch": 0.5005954663813811, + "grad_norm": 1.2166653871536255, + "learning_rate": 0.0001, + "loss": 82.0288, + "step": 3100 + }, + { + "epoch": 0.5022102904664824, + "grad_norm": 1.0471502542495728, + "learning_rate": 0.0001, + "loss": 82.0706, + "step": 3110 + }, + { + "epoch": 0.5038251145515835, + "grad_norm": 1.007645845413208, + "learning_rate": 0.0001, + "loss": 81.7387, + "step": 3120 + }, + { + "epoch": 0.5054399386366848, + "grad_norm": 0.7252638339996338, + "learning_rate": 0.0001, + "loss": 82.9107, + "step": 3130 + }, + { + "epoch": 0.507054762721786, + "grad_norm": 0.8565683364868164, + "learning_rate": 0.0001, + "loss": 80.2935, + "step": 3140 + }, + { + "epoch": 0.5086695868068872, + "grad_norm": 0.9905404448509216, + "learning_rate": 0.0001, + "loss": 82.5201, + "step": 3150 + }, + { + "epoch": 0.5102844108919885, + "grad_norm": 0.887428343296051, + "learning_rate": 0.0001, + "loss": 80.8528, + "step": 3160 + }, + { + "epoch": 0.5118992349770897, + "grad_norm": 1.062942624092102, + "learning_rate": 0.0001, + "loss": 80.7035, + "step": 3170 + }, + { + "epoch": 0.5135140590621909, + "grad_norm": 1.203599214553833, + "learning_rate": 0.0001, + "loss": 81.2521, + "step": 3180 + }, + { + "epoch": 0.5151288831472921, + "grad_norm": 0.8452383875846863, + "learning_rate": 0.0001, + "loss": 81.6079, + "step": 3190 + }, + { + "epoch": 0.5167437072323934, + "grad_norm": 1.2530587911605835, + "learning_rate": 0.0001, + "loss": 82.2148, + "step": 3200 + }, + { + "epoch": 0.5183585313174947, + "grad_norm": 0.9351627826690674, + "learning_rate": 0.0001, + "loss": 82.7857, + "step": 3210 + }, + { + "epoch": 0.5199733554025958, + "grad_norm": 1.0128077268600464, + "learning_rate": 0.0001, + "loss": 81.0532, + "step": 3220 + }, + { + "epoch": 0.5215881794876971, + "grad_norm": 0.9316917061805725, + "learning_rate": 0.0001, + "loss": 80.7443, + "step": 3230 + }, + { + "epoch": 0.5232030035727983, + "grad_norm": 1.502705693244934, + "learning_rate": 0.0001, + "loss": 80.0608, + "step": 3240 + }, + { + "epoch": 0.5248178276578995, + "grad_norm": 0.8166837692260742, + "learning_rate": 0.0001, + "loss": 81.2013, + "step": 3250 + }, + { + "epoch": 0.5264326517430008, + "grad_norm": 0.8210042119026184, + "learning_rate": 0.0001, + "loss": 80.2491, + "step": 3260 + }, + { + "epoch": 0.528047475828102, + "grad_norm": 0.7854771018028259, + "learning_rate": 0.0001, + "loss": 81.1017, + "step": 3270 + }, + { + "epoch": 0.5296622999132032, + "grad_norm": 0.8530565500259399, + "learning_rate": 0.0001, + "loss": 80.8123, + "step": 3280 + }, + { + "epoch": 0.5312771239983044, + "grad_norm": 0.9446498155593872, + "learning_rate": 0.0001, + "loss": 80.0029, + "step": 3290 + }, + { + "epoch": 0.5328919480834057, + "grad_norm": 0.9646537899971008, + "learning_rate": 0.0001, + "loss": 81.5413, + "step": 3300 + }, + { + "epoch": 0.5345067721685068, + "grad_norm": 0.9262019395828247, + "learning_rate": 0.0001, + "loss": 80.9399, + "step": 3310 + }, + { + "epoch": 0.5361215962536081, + "grad_norm": 0.8515486121177673, + "learning_rate": 0.0001, + "loss": 80.9421, + "step": 3320 + }, + { + "epoch": 0.5377364203387094, + "grad_norm": 1.679321050643921, + "learning_rate": 0.0001, + "loss": 82.4851, + "step": 3330 + }, + { + "epoch": 0.5393512444238105, + "grad_norm": 0.8289812207221985, + "learning_rate": 0.0001, + "loss": 80.5664, + "step": 3340 + }, + { + "epoch": 0.5409660685089118, + "grad_norm": 0.7463789582252502, + "learning_rate": 0.0001, + "loss": 81.5193, + "step": 3350 + }, + { + "epoch": 0.542580892594013, + "grad_norm": 0.8332446217536926, + "learning_rate": 0.0001, + "loss": 80.8091, + "step": 3360 + }, + { + "epoch": 0.5441957166791143, + "grad_norm": 0.7612847089767456, + "learning_rate": 0.0001, + "loss": 79.8893, + "step": 3370 + }, + { + "epoch": 0.5458105407642155, + "grad_norm": 1.0557948350906372, + "learning_rate": 0.0001, + "loss": 81.4305, + "step": 3380 + }, + { + "epoch": 0.5474253648493167, + "grad_norm": 0.7975788116455078, + "learning_rate": 0.0001, + "loss": 80.0105, + "step": 3390 + }, + { + "epoch": 0.549040188934418, + "grad_norm": 0.9049227237701416, + "learning_rate": 0.0001, + "loss": 81.2689, + "step": 3400 + }, + { + "epoch": 0.5506550130195191, + "grad_norm": 1.1195895671844482, + "learning_rate": 0.0001, + "loss": 80.7163, + "step": 3410 + }, + { + "epoch": 0.5522698371046204, + "grad_norm": 0.7506540417671204, + "learning_rate": 0.0001, + "loss": 79.0855, + "step": 3420 + }, + { + "epoch": 0.5538846611897217, + "grad_norm": 0.73989337682724, + "learning_rate": 0.0001, + "loss": 81.1865, + "step": 3430 + }, + { + "epoch": 0.5554994852748228, + "grad_norm": 0.8418401479721069, + "learning_rate": 0.0001, + "loss": 79.5714, + "step": 3440 + }, + { + "epoch": 0.5571143093599241, + "grad_norm": 0.9807226061820984, + "learning_rate": 0.0001, + "loss": 78.5699, + "step": 3450 + }, + { + "epoch": 0.5587291334450253, + "grad_norm": 1.2890545129776, + "learning_rate": 0.0001, + "loss": 81.0128, + "step": 3460 + }, + { + "epoch": 0.5603439575301266, + "grad_norm": 0.8593361973762512, + "learning_rate": 0.0001, + "loss": 80.1638, + "step": 3470 + }, + { + "epoch": 0.5619587816152278, + "grad_norm": 0.7766354084014893, + "learning_rate": 0.0001, + "loss": 81.5597, + "step": 3480 + }, + { + "epoch": 0.563573605700329, + "grad_norm": 0.7785626649856567, + "learning_rate": 0.0001, + "loss": 79.3862, + "step": 3490 + }, + { + "epoch": 0.5651884297854303, + "grad_norm": 0.8429279327392578, + "learning_rate": 0.0001, + "loss": 76.6294, + "step": 3500 + }, + { + "epoch": 0.5668032538705314, + "grad_norm": 0.8141765594482422, + "learning_rate": 0.0001, + "loss": 80.2584, + "step": 3510 + }, + { + "epoch": 0.5684180779556327, + "grad_norm": 0.7065490484237671, + "learning_rate": 0.0001, + "loss": 79.3626, + "step": 3520 + }, + { + "epoch": 0.570032902040734, + "grad_norm": 0.7002031803131104, + "learning_rate": 0.0001, + "loss": 79.283, + "step": 3530 + }, + { + "epoch": 0.5716477261258351, + "grad_norm": 0.8715317845344543, + "learning_rate": 0.0001, + "loss": 80.3145, + "step": 3540 + }, + { + "epoch": 0.5732625502109364, + "grad_norm": 0.9584882855415344, + "learning_rate": 0.0001, + "loss": 78.7443, + "step": 3550 + }, + { + "epoch": 0.5748773742960376, + "grad_norm": 0.7908946871757507, + "learning_rate": 0.0001, + "loss": 79.1906, + "step": 3560 + }, + { + "epoch": 0.5764921983811389, + "grad_norm": 0.8416746258735657, + "learning_rate": 0.0001, + "loss": 77.8172, + "step": 3570 + }, + { + "epoch": 0.5781070224662401, + "grad_norm": 0.823667049407959, + "learning_rate": 0.0001, + "loss": 81.864, + "step": 3580 + }, + { + "epoch": 0.5797218465513413, + "grad_norm": 1.4282617568969727, + "learning_rate": 0.0001, + "loss": 79.9052, + "step": 3590 + }, + { + "epoch": 0.5813366706364426, + "grad_norm": 0.8747813701629639, + "learning_rate": 0.0001, + "loss": 78.8234, + "step": 3600 + }, + { + "epoch": 0.5829514947215437, + "grad_norm": 0.8415511846542358, + "learning_rate": 0.0001, + "loss": 78.8171, + "step": 3610 + }, + { + "epoch": 0.584566318806645, + "grad_norm": 0.9014492034912109, + "learning_rate": 0.0001, + "loss": 80.5011, + "step": 3620 + }, + { + "epoch": 0.5861811428917463, + "grad_norm": 0.7535204887390137, + "learning_rate": 0.0001, + "loss": 79.4895, + "step": 3630 + }, + { + "epoch": 0.5877959669768474, + "grad_norm": 0.8196508884429932, + "learning_rate": 0.0001, + "loss": 79.7101, + "step": 3640 + }, + { + "epoch": 0.5894107910619487, + "grad_norm": 1.5975492000579834, + "learning_rate": 0.0001, + "loss": 79.0687, + "step": 3650 + }, + { + "epoch": 0.5910256151470499, + "grad_norm": 0.9251692295074463, + "learning_rate": 0.0001, + "loss": 80.2377, + "step": 3660 + }, + { + "epoch": 0.5926404392321512, + "grad_norm": 0.7143557071685791, + "learning_rate": 0.0001, + "loss": 78.0671, + "step": 3670 + }, + { + "epoch": 0.5942552633172524, + "grad_norm": 0.7876290678977966, + "learning_rate": 0.0001, + "loss": 79.1455, + "step": 3680 + }, + { + "epoch": 0.5958700874023536, + "grad_norm": 1.5361895561218262, + "learning_rate": 0.0001, + "loss": 78.1253, + "step": 3690 + }, + { + "epoch": 0.5974849114874549, + "grad_norm": 0.7637040019035339, + "learning_rate": 0.0001, + "loss": 77.4658, + "step": 3700 + }, + { + "epoch": 0.599099735572556, + "grad_norm": 0.8166250586509705, + "learning_rate": 0.0001, + "loss": 79.5806, + "step": 3710 + }, + { + "epoch": 0.6007145596576573, + "grad_norm": 1.2767380475997925, + "learning_rate": 0.0001, + "loss": 79.1894, + "step": 3720 + }, + { + "epoch": 0.6023293837427586, + "grad_norm": 1.3048640489578247, + "learning_rate": 0.0001, + "loss": 78.0593, + "step": 3730 + }, + { + "epoch": 0.6039442078278597, + "grad_norm": 0.7945445775985718, + "learning_rate": 0.0001, + "loss": 79.7623, + "step": 3740 + }, + { + "epoch": 0.605559031912961, + "grad_norm": 0.7366836667060852, + "learning_rate": 0.0001, + "loss": 77.9571, + "step": 3750 + }, + { + "epoch": 0.6071738559980622, + "grad_norm": 0.9126115441322327, + "learning_rate": 0.0001, + "loss": 79.9502, + "step": 3760 + }, + { + "epoch": 0.6087886800831634, + "grad_norm": 0.800355851650238, + "learning_rate": 0.0001, + "loss": 78.7244, + "step": 3770 + }, + { + "epoch": 0.6104035041682647, + "grad_norm": 0.8290980458259583, + "learning_rate": 0.0001, + "loss": 78.7724, + "step": 3780 + }, + { + "epoch": 0.6120183282533659, + "grad_norm": 0.8512316942214966, + "learning_rate": 0.0001, + "loss": 78.6277, + "step": 3790 + }, + { + "epoch": 0.6136331523384672, + "grad_norm": 0.8909353613853455, + "learning_rate": 0.0001, + "loss": 78.5956, + "step": 3800 + }, + { + "epoch": 0.6152479764235683, + "grad_norm": 0.8328800797462463, + "learning_rate": 0.0001, + "loss": 78.1323, + "step": 3810 + }, + { + "epoch": 0.6168628005086696, + "grad_norm": 0.7950423359870911, + "learning_rate": 0.0001, + "loss": 78.7529, + "step": 3820 + }, + { + "epoch": 0.6184776245937709, + "grad_norm": 0.9988545179367065, + "learning_rate": 0.0001, + "loss": 80.7727, + "step": 3830 + }, + { + "epoch": 0.620092448678872, + "grad_norm": 0.7243085503578186, + "learning_rate": 0.0001, + "loss": 76.7616, + "step": 3840 + }, + { + "epoch": 0.6217072727639733, + "grad_norm": 0.7674855589866638, + "learning_rate": 0.0001, + "loss": 79.1938, + "step": 3850 + }, + { + "epoch": 0.6233220968490745, + "grad_norm": 0.8979483842849731, + "learning_rate": 0.0001, + "loss": 79.1522, + "step": 3860 + }, + { + "epoch": 0.6249369209341757, + "grad_norm": 0.7759745717048645, + "learning_rate": 0.0001, + "loss": 77.7001, + "step": 3870 + }, + { + "epoch": 0.626551745019277, + "grad_norm": 0.9940371513366699, + "learning_rate": 0.0001, + "loss": 79.8712, + "step": 3880 + }, + { + "epoch": 0.6281665691043782, + "grad_norm": 0.8103354573249817, + "learning_rate": 0.0001, + "loss": 77.5229, + "step": 3890 + }, + { + "epoch": 0.6297813931894795, + "grad_norm": 0.9188411831855774, + "learning_rate": 0.0001, + "loss": 78.2933, + "step": 3900 + }, + { + "epoch": 0.6313962172745806, + "grad_norm": 0.823017954826355, + "learning_rate": 0.0001, + "loss": 77.8692, + "step": 3910 + }, + { + "epoch": 0.6330110413596819, + "grad_norm": 0.8833358287811279, + "learning_rate": 0.0001, + "loss": 78.4465, + "step": 3920 + }, + { + "epoch": 0.634625865444783, + "grad_norm": 0.6880685687065125, + "learning_rate": 0.0001, + "loss": 78.8881, + "step": 3930 + }, + { + "epoch": 0.6362406895298843, + "grad_norm": 0.778455913066864, + "learning_rate": 0.0001, + "loss": 76.0277, + "step": 3940 + }, + { + "epoch": 0.6378555136149856, + "grad_norm": 0.8132772445678711, + "learning_rate": 0.0001, + "loss": 79.1284, + "step": 3950 + }, + { + "epoch": 0.6394703377000868, + "grad_norm": 0.7703788876533508, + "learning_rate": 0.0001, + "loss": 76.9013, + "step": 3960 + }, + { + "epoch": 0.641085161785188, + "grad_norm": 0.7876151204109192, + "learning_rate": 0.0001, + "loss": 77.1354, + "step": 3970 + }, + { + "epoch": 0.6426999858702892, + "grad_norm": 0.8027064800262451, + "learning_rate": 0.0001, + "loss": 77.4936, + "step": 3980 + }, + { + "epoch": 0.6443148099553905, + "grad_norm": 0.8261818885803223, + "learning_rate": 0.0001, + "loss": 76.7634, + "step": 3990 + }, + { + "epoch": 0.6459296340404918, + "grad_norm": 0.7024852633476257, + "learning_rate": 0.0001, + "loss": 79.951, + "step": 4000 + }, + { + "epoch": 0.6475444581255929, + "grad_norm": 1.015307068824768, + "learning_rate": 0.0001, + "loss": 76.0354, + "step": 4010 + }, + { + "epoch": 0.6491592822106942, + "grad_norm": 0.715346097946167, + "learning_rate": 0.0001, + "loss": 78.9021, + "step": 4020 + }, + { + "epoch": 0.6507741062957954, + "grad_norm": 0.8994929790496826, + "learning_rate": 0.0001, + "loss": 77.0587, + "step": 4030 + }, + { + "epoch": 0.6523889303808966, + "grad_norm": 0.7683436274528503, + "learning_rate": 0.0001, + "loss": 78.6943, + "step": 4040 + }, + { + "epoch": 0.6540037544659979, + "grad_norm": 0.7273952960968018, + "learning_rate": 0.0001, + "loss": 77.8396, + "step": 4050 + }, + { + "epoch": 0.6556185785510991, + "grad_norm": 1.0676878690719604, + "learning_rate": 0.0001, + "loss": 79.3449, + "step": 4060 + }, + { + "epoch": 0.6572334026362003, + "grad_norm": 0.6916168928146362, + "learning_rate": 0.0001, + "loss": 76.5462, + "step": 4070 + }, + { + "epoch": 0.6588482267213015, + "grad_norm": 0.6466719508171082, + "learning_rate": 0.0001, + "loss": 77.9354, + "step": 4080 + }, + { + "epoch": 0.6604630508064028, + "grad_norm": 0.7406665086746216, + "learning_rate": 0.0001, + "loss": 76.0135, + "step": 4090 + }, + { + "epoch": 0.662077874891504, + "grad_norm": 0.7104353904724121, + "learning_rate": 0.0001, + "loss": 78.3727, + "step": 4100 + }, + { + "epoch": 0.6636926989766052, + "grad_norm": 0.8328452110290527, + "learning_rate": 0.0001, + "loss": 77.5354, + "step": 4110 + }, + { + "epoch": 0.6653075230617065, + "grad_norm": 0.7012260556221008, + "learning_rate": 0.0001, + "loss": 78.7406, + "step": 4120 + }, + { + "epoch": 0.6669223471468076, + "grad_norm": 1.0436632633209229, + "learning_rate": 0.0001, + "loss": 78.0063, + "step": 4130 + }, + { + "epoch": 0.6685371712319089, + "grad_norm": 0.8471246361732483, + "learning_rate": 0.0001, + "loss": 77.8871, + "step": 4140 + }, + { + "epoch": 0.6701519953170102, + "grad_norm": 0.7378994822502136, + "learning_rate": 0.0001, + "loss": 76.9109, + "step": 4150 + }, + { + "epoch": 0.6717668194021114, + "grad_norm": 0.7778745889663696, + "learning_rate": 0.0001, + "loss": 78.3374, + "step": 4160 + }, + { + "epoch": 0.6733816434872126, + "grad_norm": 0.7841216325759888, + "learning_rate": 0.0001, + "loss": 78.1657, + "step": 4170 + }, + { + "epoch": 0.6749964675723138, + "grad_norm": 1.0027995109558105, + "learning_rate": 0.0001, + "loss": 76.6232, + "step": 4180 + }, + { + "epoch": 0.6766112916574151, + "grad_norm": 1.0034840106964111, + "learning_rate": 0.0001, + "loss": 79.5785, + "step": 4190 + }, + { + "epoch": 0.6782261157425163, + "grad_norm": 0.8399350643157959, + "learning_rate": 0.0001, + "loss": 76.8101, + "step": 4200 + }, + { + "epoch": 0.6798409398276175, + "grad_norm": 0.8864424824714661, + "learning_rate": 0.0001, + "loss": 77.8671, + "step": 4210 + }, + { + "epoch": 0.6814557639127188, + "grad_norm": 0.8322188258171082, + "learning_rate": 0.0001, + "loss": 77.6597, + "step": 4220 + }, + { + "epoch": 0.68307058799782, + "grad_norm": 0.8962367177009583, + "learning_rate": 0.0001, + "loss": 77.7324, + "step": 4230 + }, + { + "epoch": 0.6846854120829212, + "grad_norm": 0.9727205038070679, + "learning_rate": 0.0001, + "loss": 77.8531, + "step": 4240 + }, + { + "epoch": 0.6863002361680225, + "grad_norm": 0.956584095954895, + "learning_rate": 0.0001, + "loss": 78.709, + "step": 4250 + }, + { + "epoch": 0.6879150602531237, + "grad_norm": 0.8531600832939148, + "learning_rate": 0.0001, + "loss": 78.7755, + "step": 4260 + }, + { + "epoch": 0.6895298843382249, + "grad_norm": 0.8059844374656677, + "learning_rate": 0.0001, + "loss": 77.1636, + "step": 4270 + }, + { + "epoch": 0.6911447084233261, + "grad_norm": 0.8373456597328186, + "learning_rate": 0.0001, + "loss": 78.6561, + "step": 4280 + }, + { + "epoch": 0.6927595325084274, + "grad_norm": 0.7819933891296387, + "learning_rate": 0.0001, + "loss": 77.6061, + "step": 4290 + }, + { + "epoch": 0.6943743565935286, + "grad_norm": 0.8829817771911621, + "learning_rate": 0.0001, + "loss": 77.3387, + "step": 4300 + }, + { + "epoch": 0.6959891806786298, + "grad_norm": 0.791772723197937, + "learning_rate": 0.0001, + "loss": 77.4368, + "step": 4310 + }, + { + "epoch": 0.6976040047637311, + "grad_norm": 0.9949231147766113, + "learning_rate": 0.0001, + "loss": 73.4359, + "step": 4320 + }, + { + "epoch": 0.6992188288488322, + "grad_norm": 0.8676188588142395, + "learning_rate": 0.0001, + "loss": 78.4836, + "step": 4330 + }, + { + "epoch": 0.7008336529339335, + "grad_norm": 1.1704007387161255, + "learning_rate": 0.0001, + "loss": 76.5091, + "step": 4340 + }, + { + "epoch": 0.7024484770190348, + "grad_norm": 0.8877903819084167, + "learning_rate": 0.0001, + "loss": 78.021, + "step": 4350 + }, + { + "epoch": 0.704063301104136, + "grad_norm": 1.381914496421814, + "learning_rate": 0.0001, + "loss": 77.8544, + "step": 4360 + }, + { + "epoch": 0.7056781251892372, + "grad_norm": 0.835023045539856, + "learning_rate": 0.0001, + "loss": 76.1552, + "step": 4370 + }, + { + "epoch": 0.7072929492743384, + "grad_norm": 0.9814149141311646, + "learning_rate": 0.0001, + "loss": 76.7646, + "step": 4380 + }, + { + "epoch": 0.7089077733594397, + "grad_norm": 0.8012816309928894, + "learning_rate": 0.0001, + "loss": 75.794, + "step": 4390 + }, + { + "epoch": 0.7105225974445409, + "grad_norm": 0.8340234160423279, + "learning_rate": 0.0001, + "loss": 77.4417, + "step": 4400 + }, + { + "epoch": 0.7121374215296421, + "grad_norm": 0.7257950305938721, + "learning_rate": 0.0001, + "loss": 77.1807, + "step": 4410 + }, + { + "epoch": 0.7137522456147434, + "grad_norm": 0.7342708706855774, + "learning_rate": 0.0001, + "loss": 75.5091, + "step": 4420 + }, + { + "epoch": 0.7153670696998445, + "grad_norm": 0.8298904895782471, + "learning_rate": 0.0001, + "loss": 76.6651, + "step": 4430 + }, + { + "epoch": 0.7169818937849458, + "grad_norm": 0.8564396500587463, + "learning_rate": 0.0001, + "loss": 76.0055, + "step": 4440 + }, + { + "epoch": 0.7185967178700471, + "grad_norm": 0.9145296812057495, + "learning_rate": 0.0001, + "loss": 76.0369, + "step": 4450 + }, + { + "epoch": 0.7202115419551482, + "grad_norm": 0.8074191212654114, + "learning_rate": 0.0001, + "loss": 77.4436, + "step": 4460 + }, + { + "epoch": 0.7218263660402495, + "grad_norm": 0.8212682008743286, + "learning_rate": 0.0001, + "loss": 76.3129, + "step": 4470 + }, + { + "epoch": 0.7234411901253507, + "grad_norm": 0.751323938369751, + "learning_rate": 0.0001, + "loss": 77.603, + "step": 4480 + }, + { + "epoch": 0.725056014210452, + "grad_norm": 0.9664170145988464, + "learning_rate": 0.0001, + "loss": 77.481, + "step": 4490 + }, + { + "epoch": 0.7266708382955531, + "grad_norm": 0.7369455695152283, + "learning_rate": 0.0001, + "loss": 76.3238, + "step": 4500 + }, + { + "epoch": 0.7282856623806544, + "grad_norm": 0.8448513746261597, + "learning_rate": 0.0001, + "loss": 76.5679, + "step": 4510 + }, + { + "epoch": 0.7299004864657557, + "grad_norm": 0.8605676293373108, + "learning_rate": 0.0001, + "loss": 75.7667, + "step": 4520 + }, + { + "epoch": 0.7315153105508568, + "grad_norm": 0.8034664392471313, + "learning_rate": 0.0001, + "loss": 76.2163, + "step": 4530 + }, + { + "epoch": 0.7331301346359581, + "grad_norm": 0.7846826910972595, + "learning_rate": 0.0001, + "loss": 73.5979, + "step": 4540 + }, + { + "epoch": 0.7347449587210593, + "grad_norm": 0.7422606348991394, + "learning_rate": 0.0001, + "loss": 76.2334, + "step": 4550 + }, + { + "epoch": 0.7363597828061605, + "grad_norm": 1.4478483200073242, + "learning_rate": 0.0001, + "loss": 77.6168, + "step": 4560 + }, + { + "epoch": 0.7379746068912618, + "grad_norm": 2.0957767963409424, + "learning_rate": 0.0001, + "loss": 76.3199, + "step": 4570 + }, + { + "epoch": 0.739589430976363, + "grad_norm": 0.6757177114486694, + "learning_rate": 0.0001, + "loss": 77.2923, + "step": 4580 + }, + { + "epoch": 0.7412042550614643, + "grad_norm": 0.9281920790672302, + "learning_rate": 0.0001, + "loss": 76.9963, + "step": 4590 + }, + { + "epoch": 0.7428190791465654, + "grad_norm": 0.8702669143676758, + "learning_rate": 0.0001, + "loss": 76.2553, + "step": 4600 + }, + { + "epoch": 0.7444339032316667, + "grad_norm": 0.7976512312889099, + "learning_rate": 0.0001, + "loss": 74.7622, + "step": 4610 + }, + { + "epoch": 0.746048727316768, + "grad_norm": 0.9116189479827881, + "learning_rate": 0.0001, + "loss": 76.3198, + "step": 4620 + }, + { + "epoch": 0.7476635514018691, + "grad_norm": 0.6239303946495056, + "learning_rate": 0.0001, + "loss": 75.7678, + "step": 4630 + }, + { + "epoch": 0.7492783754869704, + "grad_norm": 0.8380976915359497, + "learning_rate": 0.0001, + "loss": 76.8305, + "step": 4640 + }, + { + "epoch": 0.7508931995720716, + "grad_norm": 0.7394141554832458, + "learning_rate": 0.0001, + "loss": 76.5033, + "step": 4650 + }, + { + "epoch": 0.7525080236571728, + "grad_norm": 1.0694036483764648, + "learning_rate": 0.0001, + "loss": 75.0084, + "step": 4660 + }, + { + "epoch": 0.7541228477422741, + "grad_norm": 0.7530115246772766, + "learning_rate": 0.0001, + "loss": 74.4977, + "step": 4670 + }, + { + "epoch": 0.7557376718273753, + "grad_norm": 1.9561909437179565, + "learning_rate": 0.0001, + "loss": 74.7077, + "step": 4680 + }, + { + "epoch": 0.7573524959124766, + "grad_norm": 0.625341534614563, + "learning_rate": 0.0001, + "loss": 76.1238, + "step": 4690 + }, + { + "epoch": 0.7589673199975777, + "grad_norm": 0.9790500998497009, + "learning_rate": 0.0001, + "loss": 76.3997, + "step": 4700 + }, + { + "epoch": 0.760582144082679, + "grad_norm": 0.7087915539741516, + "learning_rate": 0.0001, + "loss": 75.6893, + "step": 4710 + }, + { + "epoch": 0.7621969681677803, + "grad_norm": 1.0995875597000122, + "learning_rate": 0.0001, + "loss": 74.8588, + "step": 4720 + }, + { + "epoch": 0.7638117922528814, + "grad_norm": 1.0309056043624878, + "learning_rate": 0.0001, + "loss": 73.5934, + "step": 4730 + }, + { + "epoch": 0.7654266163379827, + "grad_norm": 1.0132921934127808, + "learning_rate": 0.0001, + "loss": 73.9868, + "step": 4740 + }, + { + "epoch": 0.7670414404230839, + "grad_norm": 0.7138039469718933, + "learning_rate": 0.0001, + "loss": 76.4459, + "step": 4750 + }, + { + "epoch": 0.7686562645081851, + "grad_norm": 0.7055657505989075, + "learning_rate": 0.0001, + "loss": 75.1068, + "step": 4760 + }, + { + "epoch": 0.7702710885932864, + "grad_norm": 0.7398490905761719, + "learning_rate": 0.0001, + "loss": 75.3697, + "step": 4770 + }, + { + "epoch": 0.7718859126783876, + "grad_norm": 0.6836295127868652, + "learning_rate": 0.0001, + "loss": 77.4988, + "step": 4780 + }, + { + "epoch": 0.7735007367634888, + "grad_norm": 0.7552574276924133, + "learning_rate": 0.0001, + "loss": 75.149, + "step": 4790 + }, + { + "epoch": 0.77511556084859, + "grad_norm": 0.7930851578712463, + "learning_rate": 0.0001, + "loss": 73.8661, + "step": 4800 + }, + { + "epoch": 0.7767303849336913, + "grad_norm": 0.7549886107444763, + "learning_rate": 0.0001, + "loss": 75.042, + "step": 4810 + }, + { + "epoch": 0.7783452090187926, + "grad_norm": 0.9932214617729187, + "learning_rate": 0.0001, + "loss": 75.6619, + "step": 4820 + }, + { + "epoch": 0.7799600331038937, + "grad_norm": 0.7685471773147583, + "learning_rate": 0.0001, + "loss": 75.8404, + "step": 4830 + }, + { + "epoch": 0.781574857188995, + "grad_norm": 0.828737199306488, + "learning_rate": 0.0001, + "loss": 75.5839, + "step": 4840 + }, + { + "epoch": 0.7831896812740962, + "grad_norm": 0.8575420379638672, + "learning_rate": 0.0001, + "loss": 75.7889, + "step": 4850 + }, + { + "epoch": 0.7848045053591974, + "grad_norm": 0.7053834199905396, + "learning_rate": 0.0001, + "loss": 75.9372, + "step": 4860 + }, + { + "epoch": 0.7864193294442987, + "grad_norm": 0.7861505150794983, + "learning_rate": 0.0001, + "loss": 76.1532, + "step": 4870 + }, + { + "epoch": 0.7880341535293999, + "grad_norm": 0.8357386589050293, + "learning_rate": 0.0001, + "loss": 75.8536, + "step": 4880 + }, + { + "epoch": 0.7896489776145011, + "grad_norm": 3.448834180831909, + "learning_rate": 0.0001, + "loss": 74.1794, + "step": 4890 + }, + { + "epoch": 0.7912638016996023, + "grad_norm": 0.813522458076477, + "learning_rate": 0.0001, + "loss": 74.5202, + "step": 4900 + }, + { + "epoch": 0.7928786257847036, + "grad_norm": 0.840230405330658, + "learning_rate": 0.0001, + "loss": 74.6634, + "step": 4910 + }, + { + "epoch": 0.7944934498698049, + "grad_norm": 0.7855745553970337, + "learning_rate": 0.0001, + "loss": 73.5369, + "step": 4920 + }, + { + "epoch": 0.796108273954906, + "grad_norm": 0.7708212733268738, + "learning_rate": 0.0001, + "loss": 74.6225, + "step": 4930 + }, + { + "epoch": 0.7977230980400073, + "grad_norm": 0.7140830159187317, + "learning_rate": 0.0001, + "loss": 75.0585, + "step": 4940 + }, + { + "epoch": 0.7993379221251085, + "grad_norm": 0.7960936427116394, + "learning_rate": 0.0001, + "loss": 74.0574, + "step": 4950 + }, + { + "epoch": 0.8009527462102097, + "grad_norm": 0.7269165515899658, + "learning_rate": 0.0001, + "loss": 74.0379, + "step": 4960 + }, + { + "epoch": 0.802567570295311, + "grad_norm": 0.8505013585090637, + "learning_rate": 0.0001, + "loss": 74.7448, + "step": 4970 + }, + { + "epoch": 0.8041823943804122, + "grad_norm": 0.8935654759407043, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 4980 + }, + { + "epoch": 0.8057972184655134, + "grad_norm": 0.9447595477104187, + "learning_rate": 0.0001, + "loss": 75.9674, + "step": 4990 + }, + { + "epoch": 0.8074120425506146, + "grad_norm": 1.6739715337753296, + "learning_rate": 0.0001, + "loss": 73.9035, + "step": 5000 + }, + { + "epoch": 0.8090268666357159, + "grad_norm": 0.861902117729187, + "learning_rate": 0.0001, + "loss": 74.906, + "step": 5010 + }, + { + "epoch": 0.8106416907208172, + "grad_norm": 0.8328655362129211, + "learning_rate": 0.0001, + "loss": 74.1493, + "step": 5020 + }, + { + "epoch": 0.8122565148059183, + "grad_norm": 0.8193135261535645, + "learning_rate": 0.0001, + "loss": 74.6614, + "step": 5030 + }, + { + "epoch": 0.8138713388910196, + "grad_norm": 0.6954565048217773, + "learning_rate": 0.0001, + "loss": 73.7366, + "step": 5040 + }, + { + "epoch": 0.8154861629761208, + "grad_norm": 0.7381289005279541, + "learning_rate": 0.0001, + "loss": 73.5086, + "step": 5050 + }, + { + "epoch": 0.817100987061222, + "grad_norm": 1.090443730354309, + "learning_rate": 0.0001, + "loss": 75.0695, + "step": 5060 + }, + { + "epoch": 0.8187158111463233, + "grad_norm": 1.2544814348220825, + "learning_rate": 0.0001, + "loss": 72.89, + "step": 5070 + }, + { + "epoch": 0.8203306352314245, + "grad_norm": 0.632146418094635, + "learning_rate": 0.0001, + "loss": 74.8604, + "step": 5080 + }, + { + "epoch": 0.8219454593165257, + "grad_norm": 1.1038259267807007, + "learning_rate": 0.0001, + "loss": 74.7354, + "step": 5090 + }, + { + "epoch": 0.8235602834016269, + "grad_norm": 0.6886508464813232, + "learning_rate": 0.0001, + "loss": 75.4243, + "step": 5100 + }, + { + "epoch": 0.8251751074867282, + "grad_norm": 0.8367568850517273, + "learning_rate": 0.0001, + "loss": 75.2732, + "step": 5110 + }, + { + "epoch": 0.8267899315718293, + "grad_norm": 0.9302715063095093, + "learning_rate": 0.0001, + "loss": 75.7924, + "step": 5120 + }, + { + "epoch": 0.8284047556569306, + "grad_norm": 0.6871219873428345, + "learning_rate": 0.0001, + "loss": 74.2905, + "step": 5130 + }, + { + "epoch": 0.8300195797420319, + "grad_norm": 0.7319445013999939, + "learning_rate": 0.0001, + "loss": 73.7774, + "step": 5140 + }, + { + "epoch": 0.831634403827133, + "grad_norm": 0.7184522151947021, + "learning_rate": 0.0001, + "loss": 73.0554, + "step": 5150 + }, + { + "epoch": 0.8332492279122343, + "grad_norm": 0.8462343811988831, + "learning_rate": 0.0001, + "loss": 73.7615, + "step": 5160 + }, + { + "epoch": 0.8348640519973355, + "grad_norm": 0.9436241984367371, + "learning_rate": 0.0001, + "loss": 73.9464, + "step": 5170 + }, + { + "epoch": 0.8364788760824368, + "grad_norm": 0.9079630374908447, + "learning_rate": 0.0001, + "loss": 73.2727, + "step": 5180 + }, + { + "epoch": 0.838093700167538, + "grad_norm": 0.8280741572380066, + "learning_rate": 0.0001, + "loss": 73.9751, + "step": 5190 + }, + { + "epoch": 0.8397085242526392, + "grad_norm": 0.6475157141685486, + "learning_rate": 0.0001, + "loss": 74.7105, + "step": 5200 + }, + { + "epoch": 0.8413233483377405, + "grad_norm": 0.7090312838554382, + "learning_rate": 0.0001, + "loss": 73.6489, + "step": 5210 + }, + { + "epoch": 0.8429381724228416, + "grad_norm": 1.20967698097229, + "learning_rate": 0.0001, + "loss": 75.1335, + "step": 5220 + }, + { + "epoch": 0.8445529965079429, + "grad_norm": 2.053532600402832, + "learning_rate": 0.0001, + "loss": 75.3111, + "step": 5230 + }, + { + "epoch": 0.8461678205930442, + "grad_norm": 0.7563302516937256, + "learning_rate": 0.0001, + "loss": 72.9935, + "step": 5240 + }, + { + "epoch": 0.8477826446781453, + "grad_norm": 0.7801489233970642, + "learning_rate": 0.0001, + "loss": 74.7856, + "step": 5250 + }, + { + "epoch": 0.8493974687632466, + "grad_norm": 0.737870454788208, + "learning_rate": 0.0001, + "loss": 75.6281, + "step": 5260 + }, + { + "epoch": 0.8510122928483478, + "grad_norm": 0.704352617263794, + "learning_rate": 0.0001, + "loss": 75.4598, + "step": 5270 + }, + { + "epoch": 0.8526271169334491, + "grad_norm": 0.6452057361602783, + "learning_rate": 0.0001, + "loss": 74.0341, + "step": 5280 + }, + { + "epoch": 0.8542419410185503, + "grad_norm": 0.6863471865653992, + "learning_rate": 0.0001, + "loss": 74.3727, + "step": 5290 + }, + { + "epoch": 0.8558567651036515, + "grad_norm": 9.756170272827148, + "learning_rate": 0.0001, + "loss": 75.5231, + "step": 5300 + }, + { + "epoch": 0.8574715891887528, + "grad_norm": 0.8624544739723206, + "learning_rate": 0.0001, + "loss": 74.7821, + "step": 5310 + }, + { + "epoch": 0.8590864132738539, + "grad_norm": 0.6637155413627625, + "learning_rate": 0.0001, + "loss": 74.1153, + "step": 5320 + }, + { + "epoch": 0.8607012373589552, + "grad_norm": 1.0764445066452026, + "learning_rate": 0.0001, + "loss": 73.7208, + "step": 5330 + }, + { + "epoch": 0.8623160614440565, + "grad_norm": 0.8567569255828857, + "learning_rate": 0.0001, + "loss": 73.1399, + "step": 5340 + }, + { + "epoch": 0.8639308855291576, + "grad_norm": 0.7681576013565063, + "learning_rate": 0.0001, + "loss": 73.7577, + "step": 5350 + }, + { + "epoch": 0.8655457096142589, + "grad_norm": 0.9336384534835815, + "learning_rate": 0.0001, + "loss": 74.1022, + "step": 5360 + }, + { + "epoch": 0.8671605336993601, + "grad_norm": 0.8603724837303162, + "learning_rate": 0.0001, + "loss": 75.9579, + "step": 5370 + }, + { + "epoch": 0.8687753577844614, + "grad_norm": 0.77194744348526, + "learning_rate": 0.0001, + "loss": 72.4799, + "step": 5380 + }, + { + "epoch": 0.8703901818695626, + "grad_norm": 0.7480376958847046, + "learning_rate": 0.0001, + "loss": 76.1257, + "step": 5390 + }, + { + "epoch": 0.8720050059546638, + "grad_norm": 0.7841023206710815, + "learning_rate": 0.0001, + "loss": 73.1512, + "step": 5400 + }, + { + "epoch": 0.8736198300397651, + "grad_norm": 0.8741798996925354, + "learning_rate": 0.0001, + "loss": 74.2151, + "step": 5410 + }, + { + "epoch": 0.8752346541248662, + "grad_norm": 0.9762082099914551, + "learning_rate": 0.0001, + "loss": 76.7138, + "step": 5420 + }, + { + "epoch": 0.8768494782099675, + "grad_norm": 0.7670348286628723, + "learning_rate": 0.0001, + "loss": 72.9567, + "step": 5430 + }, + { + "epoch": 0.8784643022950688, + "grad_norm": 0.7449469566345215, + "learning_rate": 0.0001, + "loss": 74.8841, + "step": 5440 + }, + { + "epoch": 0.8800791263801699, + "grad_norm": 0.6984641551971436, + "learning_rate": 0.0001, + "loss": 73.8906, + "step": 5450 + }, + { + "epoch": 0.8816939504652712, + "grad_norm": 0.7854361534118652, + "learning_rate": 0.0001, + "loss": 74.626, + "step": 5460 + }, + { + "epoch": 0.8833087745503724, + "grad_norm": 0.9584240317344666, + "learning_rate": 0.0001, + "loss": 74.7048, + "step": 5470 + }, + { + "epoch": 0.8849235986354737, + "grad_norm": 0.6040284037590027, + "learning_rate": 0.0001, + "loss": 74.7194, + "step": 5480 + }, + { + "epoch": 0.8865384227205749, + "grad_norm": 0.7756215929985046, + "learning_rate": 0.0001, + "loss": 73.8178, + "step": 5490 + }, + { + "epoch": 0.8881532468056761, + "grad_norm": 0.9070586562156677, + "learning_rate": 0.0001, + "loss": 72.8309, + "step": 5500 + }, + { + "epoch": 0.8897680708907774, + "grad_norm": 0.8464986681938171, + "learning_rate": 0.0001, + "loss": 73.8847, + "step": 5510 + }, + { + "epoch": 0.8913828949758785, + "grad_norm": 0.7968605160713196, + "learning_rate": 0.0001, + "loss": 73.1691, + "step": 5520 + }, + { + "epoch": 0.8929977190609798, + "grad_norm": 0.9934583306312561, + "learning_rate": 0.0001, + "loss": 76.2662, + "step": 5530 + }, + { + "epoch": 0.8946125431460811, + "grad_norm": 0.7272152900695801, + "learning_rate": 0.0001, + "loss": 75.5295, + "step": 5540 + }, + { + "epoch": 0.8962273672311822, + "grad_norm": 0.857345461845398, + "learning_rate": 0.0001, + "loss": 75.3703, + "step": 5550 + }, + { + "epoch": 0.8978421913162835, + "grad_norm": 0.713918924331665, + "learning_rate": 0.0001, + "loss": 73.4563, + "step": 5560 + }, + { + "epoch": 0.8994570154013847, + "grad_norm": 0.759233832359314, + "learning_rate": 0.0001, + "loss": 73.2744, + "step": 5570 + }, + { + "epoch": 0.901071839486486, + "grad_norm": 0.7085754871368408, + "learning_rate": 0.0001, + "loss": 74.7792, + "step": 5580 + }, + { + "epoch": 0.9026866635715872, + "grad_norm": 0.7370456457138062, + "learning_rate": 0.0001, + "loss": 74.4216, + "step": 5590 + }, + { + "epoch": 0.9043014876566884, + "grad_norm": 0.779746949672699, + "learning_rate": 0.0001, + "loss": 75.9822, + "step": 5600 + }, + { + "epoch": 0.9059163117417897, + "grad_norm": 0.7915763854980469, + "learning_rate": 0.0001, + "loss": 74.3706, + "step": 5610 + }, + { + "epoch": 0.9075311358268908, + "grad_norm": 0.7886125445365906, + "learning_rate": 0.0001, + "loss": 72.9453, + "step": 5620 + }, + { + "epoch": 0.9091459599119921, + "grad_norm": 0.7804751992225647, + "learning_rate": 0.0001, + "loss": 73.514, + "step": 5630 + }, + { + "epoch": 0.9107607839970934, + "grad_norm": 0.7168806195259094, + "learning_rate": 0.0001, + "loss": 71.4159, + "step": 5640 + }, + { + "epoch": 0.9123756080821945, + "grad_norm": 0.7553626894950867, + "learning_rate": 0.0001, + "loss": 74.4353, + "step": 5650 + }, + { + "epoch": 0.9139904321672958, + "grad_norm": 0.6688916683197021, + "learning_rate": 0.0001, + "loss": 74.3786, + "step": 5660 + }, + { + "epoch": 0.915605256252397, + "grad_norm": 0.7425532341003418, + "learning_rate": 0.0001, + "loss": 73.3355, + "step": 5670 + }, + { + "epoch": 0.9172200803374982, + "grad_norm": 0.6473882794380188, + "learning_rate": 0.0001, + "loss": 74.2315, + "step": 5680 + }, + { + "epoch": 0.9188349044225994, + "grad_norm": 0.7222439050674438, + "learning_rate": 0.0001, + "loss": 74.2885, + "step": 5690 + }, + { + "epoch": 0.9204497285077007, + "grad_norm": 1.5716948509216309, + "learning_rate": 0.0001, + "loss": 74.6739, + "step": 5700 + }, + { + "epoch": 0.922064552592802, + "grad_norm": 0.8077759742736816, + "learning_rate": 0.0001, + "loss": 72.6617, + "step": 5710 + }, + { + "epoch": 0.9236793766779031, + "grad_norm": 0.6948631405830383, + "learning_rate": 0.0001, + "loss": 73.6247, + "step": 5720 + }, + { + "epoch": 0.9252942007630044, + "grad_norm": 0.8823757767677307, + "learning_rate": 0.0001, + "loss": 73.2536, + "step": 5730 + }, + { + "epoch": 0.9269090248481056, + "grad_norm": 0.7066764235496521, + "learning_rate": 0.0001, + "loss": 74.1748, + "step": 5740 + }, + { + "epoch": 0.9285238489332068, + "grad_norm": 0.6979793906211853, + "learning_rate": 0.0001, + "loss": 72.3189, + "step": 5750 + }, + { + "epoch": 0.9301386730183081, + "grad_norm": 0.6961462497711182, + "learning_rate": 0.0001, + "loss": 74.6196, + "step": 5760 + }, + { + "epoch": 0.9317534971034093, + "grad_norm": 0.7797656655311584, + "learning_rate": 0.0001, + "loss": 74.5472, + "step": 5770 + }, + { + "epoch": 0.9333683211885105, + "grad_norm": 1.0671838521957397, + "learning_rate": 0.0001, + "loss": 73.241, + "step": 5780 + }, + { + "epoch": 0.9349831452736117, + "grad_norm": 0.7566089034080505, + "learning_rate": 0.0001, + "loss": 73.6784, + "step": 5790 + }, + { + "epoch": 0.936597969358713, + "grad_norm": 0.8563273549079895, + "learning_rate": 0.0001, + "loss": 73.6871, + "step": 5800 + }, + { + "epoch": 0.9382127934438143, + "grad_norm": 0.7885987162590027, + "learning_rate": 0.0001, + "loss": 73.4447, + "step": 5810 + }, + { + "epoch": 0.9398276175289154, + "grad_norm": 0.8751112818717957, + "learning_rate": 0.0001, + "loss": 74.0531, + "step": 5820 + }, + { + "epoch": 0.9414424416140167, + "grad_norm": 0.6663336157798767, + "learning_rate": 0.0001, + "loss": 73.0557, + "step": 5830 + }, + { + "epoch": 0.9430572656991179, + "grad_norm": 0.7993099689483643, + "learning_rate": 0.0001, + "loss": 74.4467, + "step": 5840 + }, + { + "epoch": 0.9446720897842191, + "grad_norm": 0.6987188458442688, + "learning_rate": 0.0001, + "loss": 74.0327, + "step": 5850 + }, + { + "epoch": 0.9462869138693204, + "grad_norm": 0.8458583354949951, + "learning_rate": 0.0001, + "loss": 73.1757, + "step": 5860 + }, + { + "epoch": 0.9479017379544216, + "grad_norm": 0.6372436881065369, + "learning_rate": 0.0001, + "loss": 73.1067, + "step": 5870 + }, + { + "epoch": 0.9495165620395228, + "grad_norm": 0.7447753548622131, + "learning_rate": 0.0001, + "loss": 75.5461, + "step": 5880 + }, + { + "epoch": 0.951131386124624, + "grad_norm": 0.6633971929550171, + "learning_rate": 0.0001, + "loss": 72.9966, + "step": 5890 + }, + { + "epoch": 0.9527462102097253, + "grad_norm": 0.8576784729957581, + "learning_rate": 0.0001, + "loss": 71.5712, + "step": 5900 + }, + { + "epoch": 0.9543610342948265, + "grad_norm": 1.4696853160858154, + "learning_rate": 0.0001, + "loss": 74.7893, + "step": 5910 + }, + { + "epoch": 0.9559758583799277, + "grad_norm": 0.611210286617279, + "learning_rate": 0.0001, + "loss": 74.3426, + "step": 5920 + }, + { + "epoch": 0.957590682465029, + "grad_norm": 0.5947410464286804, + "learning_rate": 0.0001, + "loss": 71.2563, + "step": 5930 + }, + { + "epoch": 0.9592055065501301, + "grad_norm": 0.7169992923736572, + "learning_rate": 0.0001, + "loss": 71.6049, + "step": 5940 + }, + { + "epoch": 0.9608203306352314, + "grad_norm": 0.6773818731307983, + "learning_rate": 0.0001, + "loss": 75.0328, + "step": 5950 + }, + { + "epoch": 0.9624351547203327, + "grad_norm": 0.6714807748794556, + "learning_rate": 0.0001, + "loss": 71.5105, + "step": 5960 + }, + { + "epoch": 0.9640499788054339, + "grad_norm": 1.0064337253570557, + "learning_rate": 0.0001, + "loss": 75.8887, + "step": 5970 + }, + { + "epoch": 0.9656648028905351, + "grad_norm": 0.9802227020263672, + "learning_rate": 0.0001, + "loss": 73.5655, + "step": 5980 + }, + { + "epoch": 0.9672796269756363, + "grad_norm": 0.7097306251525879, + "learning_rate": 0.0001, + "loss": 73.298, + "step": 5990 + }, + { + "epoch": 0.9688944510607376, + "grad_norm": 0.6680361032485962, + "learning_rate": 0.0001, + "loss": 73.8873, + "step": 6000 + }, + { + "epoch": 0.9705092751458388, + "grad_norm": 1.1302506923675537, + "learning_rate": 0.0001, + "loss": 73.3328, + "step": 6010 + }, + { + "epoch": 0.97212409923094, + "grad_norm": 0.7163954377174377, + "learning_rate": 0.0001, + "loss": 70.5955, + "step": 6020 + }, + { + "epoch": 0.9737389233160413, + "grad_norm": 0.8182240128517151, + "learning_rate": 0.0001, + "loss": 72.6397, + "step": 6030 + }, + { + "epoch": 0.9753537474011424, + "grad_norm": 0.6752052903175354, + "learning_rate": 0.0001, + "loss": 73.1188, + "step": 6040 + }, + { + "epoch": 0.9769685714862437, + "grad_norm": 0.8829680681228638, + "learning_rate": 0.0001, + "loss": 75.7346, + "step": 6050 + }, + { + "epoch": 0.978583395571345, + "grad_norm": 0.9254767894744873, + "learning_rate": 0.0001, + "loss": 73.6221, + "step": 6060 + }, + { + "epoch": 0.9801982196564462, + "grad_norm": 0.7527495622634888, + "learning_rate": 0.0001, + "loss": 73.948, + "step": 6070 + }, + { + "epoch": 0.9818130437415474, + "grad_norm": 1.0135066509246826, + "learning_rate": 0.0001, + "loss": 72.8929, + "step": 6080 + }, + { + "epoch": 0.9834278678266486, + "grad_norm": 0.7093394994735718, + "learning_rate": 0.0001, + "loss": 73.5826, + "step": 6090 + }, + { + "epoch": 0.9850426919117499, + "grad_norm": 0.6651231646537781, + "learning_rate": 0.0001, + "loss": 73.167, + "step": 6100 + }, + { + "epoch": 0.9866575159968511, + "grad_norm": 0.720552921295166, + "learning_rate": 0.0001, + "loss": 74.8985, + "step": 6110 + }, + { + "epoch": 0.9882723400819523, + "grad_norm": 1.7821632623672485, + "learning_rate": 0.0001, + "loss": 74.1414, + "step": 6120 + }, + { + "epoch": 0.9898871641670536, + "grad_norm": 0.6405150890350342, + "learning_rate": 0.0001, + "loss": 73.5989, + "step": 6130 + }, + { + "epoch": 0.9915019882521547, + "grad_norm": 0.8207482695579529, + "learning_rate": 0.0001, + "loss": 73.0977, + "step": 6140 + }, + { + "epoch": 0.993116812337256, + "grad_norm": 0.7320757508277893, + "learning_rate": 0.0001, + "loss": 72.0241, + "step": 6150 + }, + { + "epoch": 0.9947316364223573, + "grad_norm": 0.9522950649261475, + "learning_rate": 0.0001, + "loss": 73.9831, + "step": 6160 + }, + { + "epoch": 0.9963464605074585, + "grad_norm": 0.7437159419059753, + "learning_rate": 0.0001, + "loss": 72.1054, + "step": 6170 + }, + { + "epoch": 0.9979612845925597, + "grad_norm": 0.7035623788833618, + "learning_rate": 0.0001, + "loss": 73.959, + "step": 6180 + }, + { + "epoch": 0.9995761086776609, + "grad_norm": 0.7559277415275574, + "learning_rate": 0.0001, + "loss": 71.4651, + "step": 6190 + }, + { + "epoch": 1.001130376859571, + "grad_norm": 0.852342426776886, + "learning_rate": 0.0001, + "loss": 71.8844, + "step": 6200 + }, + { + "epoch": 1.002745200944672, + "grad_norm": 0.8018911480903625, + "learning_rate": 0.0001, + "loss": 74.5093, + "step": 6210 + }, + { + "epoch": 1.0043600250297733, + "grad_norm": 0.6613730192184448, + "learning_rate": 0.0001, + "loss": 71.9953, + "step": 6220 + }, + { + "epoch": 1.0059748491148746, + "grad_norm": 0.9815977215766907, + "learning_rate": 0.0001, + "loss": 70.8632, + "step": 6230 + }, + { + "epoch": 1.0075896731999758, + "grad_norm": 0.781581699848175, + "learning_rate": 0.0001, + "loss": 72.3603, + "step": 6240 + }, + { + "epoch": 1.009204497285077, + "grad_norm": 0.8079054355621338, + "learning_rate": 0.0001, + "loss": 73.6153, + "step": 6250 + }, + { + "epoch": 1.0108193213701782, + "grad_norm": 0.8356053233146667, + "learning_rate": 0.0001, + "loss": 73.1562, + "step": 6260 + }, + { + "epoch": 1.0124341454552794, + "grad_norm": 0.7107778787612915, + "learning_rate": 0.0001, + "loss": 74.5276, + "step": 6270 + }, + { + "epoch": 1.0140489695403807, + "grad_norm": 0.8601720929145813, + "learning_rate": 0.0001, + "loss": 70.103, + "step": 6280 + }, + { + "epoch": 1.015663793625482, + "grad_norm": 0.675837516784668, + "learning_rate": 0.0001, + "loss": 72.6102, + "step": 6290 + }, + { + "epoch": 1.0172786177105833, + "grad_norm": 0.7160016298294067, + "learning_rate": 0.0001, + "loss": 72.2404, + "step": 6300 + }, + { + "epoch": 1.0188934417956843, + "grad_norm": 0.7286407351493835, + "learning_rate": 0.0001, + "loss": 72.7149, + "step": 6310 + }, + { + "epoch": 1.0205082658807856, + "grad_norm": 0.8183980584144592, + "learning_rate": 0.0001, + "loss": 73.5827, + "step": 6320 + }, + { + "epoch": 1.0221230899658869, + "grad_norm": 0.7300626039505005, + "learning_rate": 0.0001, + "loss": 72.3459, + "step": 6330 + }, + { + "epoch": 1.0237379140509881, + "grad_norm": 0.7222613096237183, + "learning_rate": 0.0001, + "loss": 71.881, + "step": 6340 + }, + { + "epoch": 1.0253527381360894, + "grad_norm": 0.7324674725532532, + "learning_rate": 0.0001, + "loss": 74.4781, + "step": 6350 + }, + { + "epoch": 1.0269675622211905, + "grad_norm": 0.9173555374145508, + "learning_rate": 0.0001, + "loss": 74.3222, + "step": 6360 + }, + { + "epoch": 1.0285823863062917, + "grad_norm": 0.8404964208602905, + "learning_rate": 0.0001, + "loss": 71.8959, + "step": 6370 + }, + { + "epoch": 1.030197210391393, + "grad_norm": 0.8735223412513733, + "learning_rate": 0.0001, + "loss": 72.7251, + "step": 6380 + }, + { + "epoch": 1.0318120344764943, + "grad_norm": 0.8467233777046204, + "learning_rate": 0.0001, + "loss": 72.5218, + "step": 6390 + }, + { + "epoch": 1.0334268585615956, + "grad_norm": 0.673565149307251, + "learning_rate": 0.0001, + "loss": 70.6392, + "step": 6400 + }, + { + "epoch": 1.0350416826466966, + "grad_norm": 0.6558571457862854, + "learning_rate": 0.0001, + "loss": 73.5305, + "step": 6410 + }, + { + "epoch": 1.0366565067317979, + "grad_norm": 0.7251600623130798, + "learning_rate": 0.0001, + "loss": 72.76, + "step": 6420 + }, + { + "epoch": 1.0382713308168992, + "grad_norm": 0.7458305358886719, + "learning_rate": 0.0001, + "loss": 73.6812, + "step": 6430 + }, + { + "epoch": 1.0398861549020004, + "grad_norm": 1.493802547454834, + "learning_rate": 0.0001, + "loss": 74.2455, + "step": 6440 + }, + { + "epoch": 1.0415009789871017, + "grad_norm": 0.894632875919342, + "learning_rate": 0.0001, + "loss": 74.2072, + "step": 6450 + }, + { + "epoch": 1.0431158030722028, + "grad_norm": 0.7142218947410583, + "learning_rate": 0.0001, + "loss": 71.4293, + "step": 6460 + }, + { + "epoch": 1.044730627157304, + "grad_norm": 0.702639639377594, + "learning_rate": 0.0001, + "loss": 71.5383, + "step": 6470 + }, + { + "epoch": 1.0463454512424053, + "grad_norm": 0.9378212094306946, + "learning_rate": 0.0001, + "loss": 72.8916, + "step": 6480 + }, + { + "epoch": 1.0479602753275066, + "grad_norm": 0.8298363089561462, + "learning_rate": 0.0001, + "loss": 72.3344, + "step": 6490 + }, + { + "epoch": 1.0495750994126078, + "grad_norm": 0.6617016792297363, + "learning_rate": 0.0001, + "loss": 71.0487, + "step": 6500 + }, + { + "epoch": 1.051189923497709, + "grad_norm": 0.638520359992981, + "learning_rate": 0.0001, + "loss": 72.2583, + "step": 6510 + }, + { + "epoch": 1.0528047475828102, + "grad_norm": 0.7101534008979797, + "learning_rate": 0.0001, + "loss": 70.8697, + "step": 6520 + }, + { + "epoch": 1.0544195716679114, + "grad_norm": 0.9425339698791504, + "learning_rate": 0.0001, + "loss": 71.0641, + "step": 6530 + }, + { + "epoch": 1.0560343957530127, + "grad_norm": 0.6572297811508179, + "learning_rate": 0.0001, + "loss": 72.6265, + "step": 6540 + }, + { + "epoch": 1.057649219838114, + "grad_norm": 0.745622456073761, + "learning_rate": 0.0001, + "loss": 73.0587, + "step": 6550 + }, + { + "epoch": 1.059264043923215, + "grad_norm": 0.7667096257209778, + "learning_rate": 0.0001, + "loss": 72.1779, + "step": 6560 + }, + { + "epoch": 1.0608788680083163, + "grad_norm": 0.7322410941123962, + "learning_rate": 0.0001, + "loss": 71.5503, + "step": 6570 + }, + { + "epoch": 1.0624936920934176, + "grad_norm": 0.5739616751670837, + "learning_rate": 0.0001, + "loss": 72.3846, + "step": 6580 + }, + { + "epoch": 1.0641085161785189, + "grad_norm": 0.7149553894996643, + "learning_rate": 0.0001, + "loss": 71.5459, + "step": 6590 + }, + { + "epoch": 1.0657233402636201, + "grad_norm": 0.6634597778320312, + "learning_rate": 0.0001, + "loss": 72.2528, + "step": 6600 + }, + { + "epoch": 1.0673381643487212, + "grad_norm": 0.7208803296089172, + "learning_rate": 0.0001, + "loss": 71.1115, + "step": 6610 + }, + { + "epoch": 1.0689529884338225, + "grad_norm": 0.9845868945121765, + "learning_rate": 0.0001, + "loss": 72.6418, + "step": 6620 + }, + { + "epoch": 1.0705678125189237, + "grad_norm": 0.9322167634963989, + "learning_rate": 0.0001, + "loss": 71.937, + "step": 6630 + }, + { + "epoch": 1.072182636604025, + "grad_norm": 0.6323093771934509, + "learning_rate": 0.0001, + "loss": 71.1537, + "step": 6640 + }, + { + "epoch": 1.0737974606891263, + "grad_norm": 0.6476802229881287, + "learning_rate": 0.0001, + "loss": 73.7811, + "step": 6650 + }, + { + "epoch": 1.0754122847742273, + "grad_norm": 0.7882822751998901, + "learning_rate": 0.0001, + "loss": 71.9548, + "step": 6660 + }, + { + "epoch": 1.0770271088593286, + "grad_norm": 0.7651168704032898, + "learning_rate": 0.0001, + "loss": 73.4066, + "step": 6670 + }, + { + "epoch": 1.07864193294443, + "grad_norm": 0.7536904215812683, + "learning_rate": 0.0001, + "loss": 72.1063, + "step": 6680 + }, + { + "epoch": 1.0802567570295312, + "grad_norm": 0.7908427119255066, + "learning_rate": 0.0001, + "loss": 72.6374, + "step": 6690 + }, + { + "epoch": 1.0818715811146324, + "grad_norm": 0.7796691060066223, + "learning_rate": 0.0001, + "loss": 72.2938, + "step": 6700 + }, + { + "epoch": 1.0834864051997335, + "grad_norm": 0.8512274622917175, + "learning_rate": 0.0001, + "loss": 71.556, + "step": 6710 + }, + { + "epoch": 1.0851012292848348, + "grad_norm": 0.7285851836204529, + "learning_rate": 0.0001, + "loss": 72.406, + "step": 6720 + }, + { + "epoch": 1.086716053369936, + "grad_norm": 0.6613008975982666, + "learning_rate": 0.0001, + "loss": 70.6509, + "step": 6730 + }, + { + "epoch": 1.0883308774550373, + "grad_norm": 0.7158271074295044, + "learning_rate": 0.0001, + "loss": 71.8434, + "step": 6740 + }, + { + "epoch": 1.0899457015401384, + "grad_norm": 0.8635967373847961, + "learning_rate": 0.0001, + "loss": 71.8279, + "step": 6750 + }, + { + "epoch": 1.0915605256252396, + "grad_norm": 0.7020159959793091, + "learning_rate": 0.0001, + "loss": 72.2417, + "step": 6760 + }, + { + "epoch": 1.093175349710341, + "grad_norm": 0.6389853954315186, + "learning_rate": 0.0001, + "loss": 72.1879, + "step": 6770 + }, + { + "epoch": 1.0947901737954422, + "grad_norm": 0.6460697650909424, + "learning_rate": 0.0001, + "loss": 72.3356, + "step": 6780 + }, + { + "epoch": 1.0964049978805435, + "grad_norm": 0.7152006030082703, + "learning_rate": 0.0001, + "loss": 71.3725, + "step": 6790 + }, + { + "epoch": 1.0980198219656445, + "grad_norm": 1.0050997734069824, + "learning_rate": 0.0001, + "loss": 72.5452, + "step": 6800 + }, + { + "epoch": 1.0996346460507458, + "grad_norm": 0.6967180371284485, + "learning_rate": 0.0001, + "loss": 72.0832, + "step": 6810 + }, + { + "epoch": 1.101249470135847, + "grad_norm": 0.7472837567329407, + "learning_rate": 0.0001, + "loss": 72.8122, + "step": 6820 + }, + { + "epoch": 1.1028642942209483, + "grad_norm": 0.8198382258415222, + "learning_rate": 0.0001, + "loss": 72.5116, + "step": 6830 + }, + { + "epoch": 1.1044791183060496, + "grad_norm": 0.8112207651138306, + "learning_rate": 0.0001, + "loss": 72.4218, + "step": 6840 + }, + { + "epoch": 1.1060939423911507, + "grad_norm": 0.8412114977836609, + "learning_rate": 0.0001, + "loss": 71.0198, + "step": 6850 + }, + { + "epoch": 1.107708766476252, + "grad_norm": 0.6730719804763794, + "learning_rate": 0.0001, + "loss": 71.2045, + "step": 6860 + }, + { + "epoch": 1.1093235905613532, + "grad_norm": 0.6895949244499207, + "learning_rate": 0.0001, + "loss": 72.3056, + "step": 6870 + }, + { + "epoch": 1.1109384146464545, + "grad_norm": 2.476609945297241, + "learning_rate": 0.0001, + "loss": 70.3865, + "step": 6880 + }, + { + "epoch": 1.1125532387315558, + "grad_norm": 0.7285001873970032, + "learning_rate": 0.0001, + "loss": 71.5761, + "step": 6890 + }, + { + "epoch": 1.1141680628166568, + "grad_norm": 0.7546679973602295, + "learning_rate": 0.0001, + "loss": 72.5979, + "step": 6900 + }, + { + "epoch": 1.115782886901758, + "grad_norm": 0.8974841237068176, + "learning_rate": 0.0001, + "loss": 74.5888, + "step": 6910 + }, + { + "epoch": 1.1173977109868594, + "grad_norm": 0.8785758018493652, + "learning_rate": 0.0001, + "loss": 71.662, + "step": 6920 + }, + { + "epoch": 1.1190125350719606, + "grad_norm": 0.6805328130722046, + "learning_rate": 0.0001, + "loss": 72.201, + "step": 6930 + }, + { + "epoch": 1.120627359157062, + "grad_norm": 0.8132315278053284, + "learning_rate": 0.0001, + "loss": 72.7473, + "step": 6940 + }, + { + "epoch": 1.122242183242163, + "grad_norm": 1.1625338792800903, + "learning_rate": 0.0001, + "loss": 70.5987, + "step": 6950 + }, + { + "epoch": 1.1238570073272642, + "grad_norm": 1.0984737873077393, + "learning_rate": 0.0001, + "loss": 72.7335, + "step": 6960 + }, + { + "epoch": 1.1254718314123655, + "grad_norm": 0.6798779368400574, + "learning_rate": 0.0001, + "loss": 72.1942, + "step": 6970 + }, + { + "epoch": 1.1270866554974668, + "grad_norm": 0.7546010613441467, + "learning_rate": 0.0001, + "loss": 70.8014, + "step": 6980 + }, + { + "epoch": 1.128701479582568, + "grad_norm": 1.0238053798675537, + "learning_rate": 0.0001, + "loss": 72.5617, + "step": 6990 + }, + { + "epoch": 1.130316303667669, + "grad_norm": 0.6535419821739197, + "learning_rate": 0.0001, + "loss": 70.2193, + "step": 7000 + }, + { + "epoch": 1.1319311277527704, + "grad_norm": 0.7354230284690857, + "learning_rate": 0.0001, + "loss": 72.0685, + "step": 7010 + }, + { + "epoch": 1.1335459518378717, + "grad_norm": 0.7848799824714661, + "learning_rate": 0.0001, + "loss": 72.2869, + "step": 7020 + }, + { + "epoch": 1.135160775922973, + "grad_norm": 0.6519262790679932, + "learning_rate": 0.0001, + "loss": 72.2023, + "step": 7030 + }, + { + "epoch": 1.1367756000080742, + "grad_norm": 0.8428511023521423, + "learning_rate": 0.0001, + "loss": 71.3396, + "step": 7040 + }, + { + "epoch": 1.1383904240931753, + "grad_norm": 0.8873165249824524, + "learning_rate": 0.0001, + "loss": 69.8284, + "step": 7050 + }, + { + "epoch": 1.1400052481782765, + "grad_norm": 0.9141374230384827, + "learning_rate": 0.0001, + "loss": 70.3666, + "step": 7060 + }, + { + "epoch": 1.1416200722633778, + "grad_norm": 0.6556552648544312, + "learning_rate": 0.0001, + "loss": 71.1316, + "step": 7070 + }, + { + "epoch": 1.143234896348479, + "grad_norm": 0.6736201047897339, + "learning_rate": 0.0001, + "loss": 70.2601, + "step": 7080 + }, + { + "epoch": 1.1448497204335804, + "grad_norm": 0.6965599656105042, + "learning_rate": 0.0001, + "loss": 71.9409, + "step": 7090 + }, + { + "epoch": 1.1464645445186814, + "grad_norm": 1.0600581169128418, + "learning_rate": 0.0001, + "loss": 72.7344, + "step": 7100 + }, + { + "epoch": 1.1480793686037827, + "grad_norm": 0.7501764297485352, + "learning_rate": 0.0001, + "loss": 72.3129, + "step": 7110 + }, + { + "epoch": 1.149694192688884, + "grad_norm": 0.7105675935745239, + "learning_rate": 0.0001, + "loss": 71.1356, + "step": 7120 + }, + { + "epoch": 1.1513090167739852, + "grad_norm": 0.7446026802062988, + "learning_rate": 0.0001, + "loss": 69.8208, + "step": 7130 + }, + { + "epoch": 1.1529238408590865, + "grad_norm": 0.6664865016937256, + "learning_rate": 0.0001, + "loss": 72.2466, + "step": 7140 + }, + { + "epoch": 1.1545386649441876, + "grad_norm": 1.7590374946594238, + "learning_rate": 0.0001, + "loss": 71.4987, + "step": 7150 + }, + { + "epoch": 1.1561534890292888, + "grad_norm": 0.7941271066665649, + "learning_rate": 0.0001, + "loss": 70.787, + "step": 7160 + }, + { + "epoch": 1.15776831311439, + "grad_norm": 0.7473573088645935, + "learning_rate": 0.0001, + "loss": 71.2516, + "step": 7170 + }, + { + "epoch": 1.1593831371994914, + "grad_norm": 0.7589637041091919, + "learning_rate": 0.0001, + "loss": 71.4781, + "step": 7180 + }, + { + "epoch": 1.1609979612845926, + "grad_norm": 0.8148510456085205, + "learning_rate": 0.0001, + "loss": 71.1491, + "step": 7190 + }, + { + "epoch": 1.1626127853696937, + "grad_norm": 0.6803365349769592, + "learning_rate": 0.0001, + "loss": 71.7937, + "step": 7200 + }, + { + "epoch": 1.164227609454795, + "grad_norm": 0.7314755320549011, + "learning_rate": 0.0001, + "loss": 70.6566, + "step": 7210 + }, + { + "epoch": 1.1658424335398963, + "grad_norm": 0.6494631171226501, + "learning_rate": 0.0001, + "loss": 70.5057, + "step": 7220 + }, + { + "epoch": 1.1674572576249975, + "grad_norm": 0.6076239943504333, + "learning_rate": 0.0001, + "loss": 72.2985, + "step": 7230 + }, + { + "epoch": 1.1690720817100988, + "grad_norm": 0.6650206446647644, + "learning_rate": 0.0001, + "loss": 71.0719, + "step": 7240 + }, + { + "epoch": 1.1706869057951999, + "grad_norm": 0.6926266551017761, + "learning_rate": 0.0001, + "loss": 70.3714, + "step": 7250 + }, + { + "epoch": 1.1723017298803011, + "grad_norm": 0.6534716486930847, + "learning_rate": 0.0001, + "loss": 71.3194, + "step": 7260 + }, + { + "epoch": 1.1739165539654024, + "grad_norm": 0.7194528579711914, + "learning_rate": 0.0001, + "loss": 72.3277, + "step": 7270 + }, + { + "epoch": 1.1755313780505037, + "grad_norm": 0.7496437430381775, + "learning_rate": 0.0001, + "loss": 71.1136, + "step": 7280 + }, + { + "epoch": 1.177146202135605, + "grad_norm": 0.690522313117981, + "learning_rate": 0.0001, + "loss": 70.065, + "step": 7290 + }, + { + "epoch": 1.178761026220706, + "grad_norm": 0.6966914534568787, + "learning_rate": 0.0001, + "loss": 73.1137, + "step": 7300 + }, + { + "epoch": 1.1803758503058073, + "grad_norm": 0.7304641604423523, + "learning_rate": 0.0001, + "loss": 71.2466, + "step": 7310 + }, + { + "epoch": 1.1819906743909085, + "grad_norm": 0.7564008235931396, + "learning_rate": 0.0001, + "loss": 71.4147, + "step": 7320 + }, + { + "epoch": 1.1836054984760098, + "grad_norm": 0.61886066198349, + "learning_rate": 0.0001, + "loss": 70.3297, + "step": 7330 + }, + { + "epoch": 1.185220322561111, + "grad_norm": 0.6281574964523315, + "learning_rate": 0.0001, + "loss": 70.9607, + "step": 7340 + }, + { + "epoch": 1.1868351466462121, + "grad_norm": 0.6583316326141357, + "learning_rate": 0.0001, + "loss": 70.3443, + "step": 7350 + }, + { + "epoch": 1.1884499707313134, + "grad_norm": 1.8159942626953125, + "learning_rate": 0.0001, + "loss": 71.4792, + "step": 7360 + }, + { + "epoch": 1.1900647948164147, + "grad_norm": 0.7253140807151794, + "learning_rate": 0.0001, + "loss": 72.2182, + "step": 7370 + }, + { + "epoch": 1.191679618901516, + "grad_norm": 0.9141016602516174, + "learning_rate": 0.0001, + "loss": 68.7443, + "step": 7380 + }, + { + "epoch": 1.193294442986617, + "grad_norm": 0.7392430901527405, + "learning_rate": 0.0001, + "loss": 69.1888, + "step": 7390 + }, + { + "epoch": 1.1949092670717183, + "grad_norm": 0.7713924646377563, + "learning_rate": 0.0001, + "loss": 69.1301, + "step": 7400 + }, + { + "epoch": 1.1965240911568196, + "grad_norm": 0.6000798344612122, + "learning_rate": 0.0001, + "loss": 71.2576, + "step": 7410 + }, + { + "epoch": 1.1981389152419208, + "grad_norm": 0.687111496925354, + "learning_rate": 0.0001, + "loss": 70.2919, + "step": 7420 + }, + { + "epoch": 1.1997537393270221, + "grad_norm": 1.1718226671218872, + "learning_rate": 0.0001, + "loss": 71.9498, + "step": 7430 + }, + { + "epoch": 1.2013685634121232, + "grad_norm": 0.7176038026809692, + "learning_rate": 0.0001, + "loss": 71.91, + "step": 7440 + }, + { + "epoch": 1.2029833874972244, + "grad_norm": 0.7705256938934326, + "learning_rate": 0.0001, + "loss": 70.1241, + "step": 7450 + }, + { + "epoch": 1.2045982115823257, + "grad_norm": 0.8004609942436218, + "learning_rate": 0.0001, + "loss": 69.5121, + "step": 7460 + }, + { + "epoch": 1.206213035667427, + "grad_norm": 0.6708329319953918, + "learning_rate": 0.0001, + "loss": 73.0173, + "step": 7470 + }, + { + "epoch": 1.2078278597525283, + "grad_norm": 0.8777983784675598, + "learning_rate": 0.0001, + "loss": 71.6843, + "step": 7480 + }, + { + "epoch": 1.2094426838376293, + "grad_norm": 0.7683864831924438, + "learning_rate": 0.0001, + "loss": 71.6914, + "step": 7490 + }, + { + "epoch": 1.2110575079227306, + "grad_norm": 0.8022210597991943, + "learning_rate": 0.0001, + "loss": 69.1724, + "step": 7500 + }, + { + "epoch": 1.2126723320078319, + "grad_norm": 0.7250640392303467, + "learning_rate": 0.0001, + "loss": 70.6568, + "step": 7510 + }, + { + "epoch": 1.2142871560929331, + "grad_norm": 0.5625214576721191, + "learning_rate": 0.0001, + "loss": 69.4849, + "step": 7520 + }, + { + "epoch": 1.2159019801780344, + "grad_norm": 0.7980468273162842, + "learning_rate": 0.0001, + "loss": 70.8164, + "step": 7530 + }, + { + "epoch": 1.2175168042631355, + "grad_norm": 0.6900951862335205, + "learning_rate": 0.0001, + "loss": 71.1719, + "step": 7540 + }, + { + "epoch": 1.2191316283482367, + "grad_norm": 0.6284971833229065, + "learning_rate": 0.0001, + "loss": 70.1751, + "step": 7550 + }, + { + "epoch": 1.220746452433338, + "grad_norm": 0.6641798615455627, + "learning_rate": 0.0001, + "loss": 71.9043, + "step": 7560 + }, + { + "epoch": 1.2223612765184393, + "grad_norm": 0.7439492344856262, + "learning_rate": 0.0001, + "loss": 70.7286, + "step": 7570 + }, + { + "epoch": 1.2239761006035406, + "grad_norm": 0.8018355369567871, + "learning_rate": 0.0001, + "loss": 71.638, + "step": 7580 + }, + { + "epoch": 1.2255909246886416, + "grad_norm": 0.7318522930145264, + "learning_rate": 0.0001, + "loss": 70.0867, + "step": 7590 + }, + { + "epoch": 1.2272057487737429, + "grad_norm": 0.6799285411834717, + "learning_rate": 0.0001, + "loss": 71.3427, + "step": 7600 + }, + { + "epoch": 1.2288205728588442, + "grad_norm": 0.6796864867210388, + "learning_rate": 0.0001, + "loss": 71.1698, + "step": 7610 + }, + { + "epoch": 1.2304353969439454, + "grad_norm": 0.7256141304969788, + "learning_rate": 0.0001, + "loss": 71.1051, + "step": 7620 + }, + { + "epoch": 1.2320502210290467, + "grad_norm": 0.8318199515342712, + "learning_rate": 0.0001, + "loss": 71.3179, + "step": 7630 + }, + { + "epoch": 1.2336650451141478, + "grad_norm": 0.6594699025154114, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 7640 + }, + { + "epoch": 1.235279869199249, + "grad_norm": 0.7900195717811584, + "learning_rate": 0.0001, + "loss": 72.0139, + "step": 7650 + }, + { + "epoch": 1.2368946932843503, + "grad_norm": 0.7845672965049744, + "learning_rate": 0.0001, + "loss": 70.808, + "step": 7660 + }, + { + "epoch": 1.2385095173694516, + "grad_norm": 0.6598045229911804, + "learning_rate": 0.0001, + "loss": 69.8953, + "step": 7670 + }, + { + "epoch": 1.2401243414545529, + "grad_norm": 1.3222978115081787, + "learning_rate": 0.0001, + "loss": 68.787, + "step": 7680 + }, + { + "epoch": 1.241739165539654, + "grad_norm": 0.6660508513450623, + "learning_rate": 0.0001, + "loss": 71.5831, + "step": 7690 + }, + { + "epoch": 1.2433539896247552, + "grad_norm": 0.6305698156356812, + "learning_rate": 0.0001, + "loss": 70.3232, + "step": 7700 + }, + { + "epoch": 1.2449688137098565, + "grad_norm": 1.2648323774337769, + "learning_rate": 0.0001, + "loss": 67.8901, + "step": 7710 + }, + { + "epoch": 1.2465836377949577, + "grad_norm": 0.7177619934082031, + "learning_rate": 0.0001, + "loss": 69.6582, + "step": 7720 + }, + { + "epoch": 1.248198461880059, + "grad_norm": 0.6716091632843018, + "learning_rate": 0.0001, + "loss": 70.2034, + "step": 7730 + }, + { + "epoch": 1.24981328596516, + "grad_norm": 0.6050158143043518, + "learning_rate": 0.0001, + "loss": 69.3942, + "step": 7740 + }, + { + "epoch": 1.2514281100502613, + "grad_norm": 0.7001014947891235, + "learning_rate": 0.0001, + "loss": 69.5971, + "step": 7750 + }, + { + "epoch": 1.2530429341353626, + "grad_norm": 0.7221280932426453, + "learning_rate": 0.0001, + "loss": 71.3651, + "step": 7760 + }, + { + "epoch": 1.2546577582204639, + "grad_norm": 0.9130340218544006, + "learning_rate": 0.0001, + "loss": 68.3038, + "step": 7770 + }, + { + "epoch": 1.2562725823055652, + "grad_norm": 0.7703199982643127, + "learning_rate": 0.0001, + "loss": 71.6127, + "step": 7780 + }, + { + "epoch": 1.2578874063906662, + "grad_norm": 0.6334695816040039, + "learning_rate": 0.0001, + "loss": 68.9074, + "step": 7790 + }, + { + "epoch": 1.2595022304757675, + "grad_norm": 0.7074242234230042, + "learning_rate": 0.0001, + "loss": 70.0569, + "step": 7800 + }, + { + "epoch": 1.2611170545608688, + "grad_norm": 0.6956025958061218, + "learning_rate": 0.0001, + "loss": 69.892, + "step": 7810 + }, + { + "epoch": 1.26273187864597, + "grad_norm": 0.7235708236694336, + "learning_rate": 0.0001, + "loss": 71.1777, + "step": 7820 + }, + { + "epoch": 1.2643467027310713, + "grad_norm": 0.6157007813453674, + "learning_rate": 0.0001, + "loss": 71.5091, + "step": 7830 + }, + { + "epoch": 1.2659615268161724, + "grad_norm": 0.5352533459663391, + "learning_rate": 0.0001, + "loss": 70.083, + "step": 7840 + }, + { + "epoch": 1.2675763509012736, + "grad_norm": 0.7171182632446289, + "learning_rate": 0.0001, + "loss": 68.4585, + "step": 7850 + }, + { + "epoch": 1.269191174986375, + "grad_norm": 0.7503113746643066, + "learning_rate": 0.0001, + "loss": 70.4206, + "step": 7860 + }, + { + "epoch": 1.2708059990714762, + "grad_norm": 0.6552693843841553, + "learning_rate": 0.0001, + "loss": 70.8455, + "step": 7870 + }, + { + "epoch": 1.2724208231565775, + "grad_norm": 0.7173179984092712, + "learning_rate": 0.0001, + "loss": 68.82, + "step": 7880 + }, + { + "epoch": 1.2740356472416785, + "grad_norm": 0.7352942824363708, + "learning_rate": 0.0001, + "loss": 70.4974, + "step": 7890 + }, + { + "epoch": 1.2756504713267798, + "grad_norm": 0.6120197176933289, + "learning_rate": 0.0001, + "loss": 72.1638, + "step": 7900 + }, + { + "epoch": 1.277265295411881, + "grad_norm": 0.6271925568580627, + "learning_rate": 0.0001, + "loss": 69.3267, + "step": 7910 + }, + { + "epoch": 1.2788801194969823, + "grad_norm": 0.752636194229126, + "learning_rate": 0.0001, + "loss": 70.6761, + "step": 7920 + }, + { + "epoch": 1.2804949435820836, + "grad_norm": 0.7307374477386475, + "learning_rate": 0.0001, + "loss": 70.2909, + "step": 7930 + }, + { + "epoch": 1.2821097676671847, + "grad_norm": 0.5977495908737183, + "learning_rate": 0.0001, + "loss": 72.2969, + "step": 7940 + }, + { + "epoch": 1.283724591752286, + "grad_norm": 0.7927457094192505, + "learning_rate": 0.0001, + "loss": 70.3784, + "step": 7950 + }, + { + "epoch": 1.2853394158373872, + "grad_norm": 0.7520087957382202, + "learning_rate": 0.0001, + "loss": 70.7122, + "step": 7960 + }, + { + "epoch": 1.2869542399224885, + "grad_norm": 0.6036237478256226, + "learning_rate": 0.0001, + "loss": 69.8718, + "step": 7970 + }, + { + "epoch": 1.2885690640075897, + "grad_norm": 0.6805875897407532, + "learning_rate": 0.0001, + "loss": 70.2391, + "step": 7980 + }, + { + "epoch": 1.2901838880926908, + "grad_norm": 0.774559497833252, + "learning_rate": 0.0001, + "loss": 71.1605, + "step": 7990 + }, + { + "epoch": 1.291798712177792, + "grad_norm": 0.6991063356399536, + "learning_rate": 0.0001, + "loss": 69.9779, + "step": 8000 + }, + { + "epoch": 1.2934135362628933, + "grad_norm": 0.5877949595451355, + "learning_rate": 0.0001, + "loss": 71.16, + "step": 8010 + }, + { + "epoch": 1.2950283603479946, + "grad_norm": 0.8840436339378357, + "learning_rate": 0.0001, + "loss": 70.9454, + "step": 8020 + }, + { + "epoch": 1.296643184433096, + "grad_norm": 0.7532126307487488, + "learning_rate": 0.0001, + "loss": 70.141, + "step": 8030 + }, + { + "epoch": 1.298258008518197, + "grad_norm": 0.6880044341087341, + "learning_rate": 0.0001, + "loss": 70.0746, + "step": 8040 + }, + { + "epoch": 1.2998728326032982, + "grad_norm": 0.6319366097450256, + "learning_rate": 0.0001, + "loss": 69.9396, + "step": 8050 + }, + { + "epoch": 1.3014876566883995, + "grad_norm": 0.6329445838928223, + "learning_rate": 0.0001, + "loss": 69.9043, + "step": 8060 + }, + { + "epoch": 1.3031024807735008, + "grad_norm": 0.8892332315444946, + "learning_rate": 0.0001, + "loss": 69.4187, + "step": 8070 + }, + { + "epoch": 1.304717304858602, + "grad_norm": 0.696500301361084, + "learning_rate": 0.0001, + "loss": 68.1885, + "step": 8080 + }, + { + "epoch": 1.306332128943703, + "grad_norm": 0.5933213233947754, + "learning_rate": 0.0001, + "loss": 68.8017, + "step": 8090 + }, + { + "epoch": 1.3079469530288044, + "grad_norm": 0.9274749159812927, + "learning_rate": 0.0001, + "loss": 71.5875, + "step": 8100 + }, + { + "epoch": 1.3095617771139056, + "grad_norm": 0.7540446519851685, + "learning_rate": 0.0001, + "loss": 71.1233, + "step": 8110 + }, + { + "epoch": 1.311176601199007, + "grad_norm": 0.6701658964157104, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8120 + }, + { + "epoch": 1.3127914252841082, + "grad_norm": 0.6096424460411072, + "learning_rate": 0.0001, + "loss": 69.9583, + "step": 8130 + }, + { + "epoch": 1.3144062493692092, + "grad_norm": 0.809524416923523, + "learning_rate": 0.0001, + "loss": 71.8149, + "step": 8140 + }, + { + "epoch": 1.3160210734543105, + "grad_norm": 0.7928242683410645, + "learning_rate": 0.0001, + "loss": 70.2084, + "step": 8150 + }, + { + "epoch": 1.3176358975394118, + "grad_norm": 0.7462850213050842, + "learning_rate": 0.0001, + "loss": 71.4789, + "step": 8160 + }, + { + "epoch": 1.319250721624513, + "grad_norm": 0.8166030049324036, + "learning_rate": 0.0001, + "loss": 69.3154, + "step": 8170 + }, + { + "epoch": 1.3208655457096143, + "grad_norm": 0.7002414464950562, + "learning_rate": 0.0001, + "loss": 70.8048, + "step": 8180 + }, + { + "epoch": 1.3224803697947154, + "grad_norm": 0.7653809189796448, + "learning_rate": 0.0001, + "loss": 70.0221, + "step": 8190 + }, + { + "epoch": 1.3240951938798167, + "grad_norm": 0.6777089238166809, + "learning_rate": 0.0001, + "loss": 70.614, + "step": 8200 + }, + { + "epoch": 1.325710017964918, + "grad_norm": 0.692739725112915, + "learning_rate": 0.0001, + "loss": 71.2311, + "step": 8210 + }, + { + "epoch": 1.3273248420500192, + "grad_norm": 0.7985519766807556, + "learning_rate": 0.0001, + "loss": 68.1982, + "step": 8220 + }, + { + "epoch": 1.3289396661351205, + "grad_norm": 0.7002689242362976, + "learning_rate": 0.0001, + "loss": 70.9181, + "step": 8230 + }, + { + "epoch": 1.3305544902202215, + "grad_norm": 0.7029191255569458, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 8240 + }, + { + "epoch": 1.3321693143053228, + "grad_norm": 0.7292912602424622, + "learning_rate": 0.0001, + "loss": 71.5386, + "step": 8250 + }, + { + "epoch": 1.333784138390424, + "grad_norm": 0.6141605973243713, + "learning_rate": 0.0001, + "loss": 72.3396, + "step": 8260 + }, + { + "epoch": 1.3353989624755254, + "grad_norm": 0.6486957669258118, + "learning_rate": 0.0001, + "loss": 69.7607, + "step": 8270 + }, + { + "epoch": 1.3370137865606266, + "grad_norm": 0.7813800573348999, + "learning_rate": 0.0001, + "loss": 68.8979, + "step": 8280 + }, + { + "epoch": 1.3386286106457277, + "grad_norm": 0.6805126070976257, + "learning_rate": 0.0001, + "loss": 70.2179, + "step": 8290 + }, + { + "epoch": 1.340243434730829, + "grad_norm": 0.7202772498130798, + "learning_rate": 0.0001, + "loss": 72.5008, + "step": 8300 + }, + { + "epoch": 1.3418582588159302, + "grad_norm": 0.6881810426712036, + "learning_rate": 0.0001, + "loss": 70.1987, + "step": 8310 + }, + { + "epoch": 1.3434730829010315, + "grad_norm": 0.6337536573410034, + "learning_rate": 0.0001, + "loss": 70.4073, + "step": 8320 + }, + { + "epoch": 1.3450879069861328, + "grad_norm": 0.7313241958618164, + "learning_rate": 0.0001, + "loss": 69.3598, + "step": 8330 + }, + { + "epoch": 1.3467027310712338, + "grad_norm": 0.635947585105896, + "learning_rate": 0.0001, + "loss": 69.5143, + "step": 8340 + }, + { + "epoch": 1.3483175551563351, + "grad_norm": 0.6219074726104736, + "learning_rate": 0.0001, + "loss": 69.9903, + "step": 8350 + }, + { + "epoch": 1.3499323792414364, + "grad_norm": 0.5928048491477966, + "learning_rate": 0.0001, + "loss": 70.4222, + "step": 8360 + }, + { + "epoch": 1.3515472033265377, + "grad_norm": 0.5927143096923828, + "learning_rate": 0.0001, + "loss": 69.8642, + "step": 8370 + }, + { + "epoch": 1.353162027411639, + "grad_norm": 0.7614993453025818, + "learning_rate": 0.0001, + "loss": 69.5787, + "step": 8380 + }, + { + "epoch": 1.35477685149674, + "grad_norm": 0.5488380193710327, + "learning_rate": 0.0001, + "loss": 69.0028, + "step": 8390 + }, + { + "epoch": 1.3563916755818413, + "grad_norm": 0.6219164133071899, + "learning_rate": 0.0001, + "loss": 69.4591, + "step": 8400 + }, + { + "epoch": 1.3580064996669425, + "grad_norm": 0.6940358877182007, + "learning_rate": 0.0001, + "loss": 71.3795, + "step": 8410 + }, + { + "epoch": 1.3596213237520438, + "grad_norm": 0.5919192433357239, + "learning_rate": 0.0001, + "loss": 71.2761, + "step": 8420 + }, + { + "epoch": 1.361236147837145, + "grad_norm": 1.0803378820419312, + "learning_rate": 0.0001, + "loss": 69.0046, + "step": 8430 + }, + { + "epoch": 1.3628509719222461, + "grad_norm": 0.7028629183769226, + "learning_rate": 0.0001, + "loss": 71.5996, + "step": 8440 + }, + { + "epoch": 1.3644657960073474, + "grad_norm": 0.6829854249954224, + "learning_rate": 0.0001, + "loss": 67.8831, + "step": 8450 + }, + { + "epoch": 1.3660806200924487, + "grad_norm": 0.6937337517738342, + "learning_rate": 0.0001, + "loss": 69.8321, + "step": 8460 + }, + { + "epoch": 1.36769544417755, + "grad_norm": 0.7725892066955566, + "learning_rate": 0.0001, + "loss": 71.0784, + "step": 8470 + }, + { + "epoch": 1.3693102682626512, + "grad_norm": 0.7171871066093445, + "learning_rate": 0.0001, + "loss": 69.0828, + "step": 8480 + }, + { + "epoch": 1.3709250923477523, + "grad_norm": 0.6605113744735718, + "learning_rate": 0.0001, + "loss": 70.9461, + "step": 8490 + }, + { + "epoch": 1.3725399164328536, + "grad_norm": 0.9005604982376099, + "learning_rate": 0.0001, + "loss": 71.474, + "step": 8500 + }, + { + "epoch": 1.3741547405179548, + "grad_norm": 0.7454419732093811, + "learning_rate": 0.0001, + "loss": 70.3113, + "step": 8510 + }, + { + "epoch": 1.375769564603056, + "grad_norm": 0.7574180960655212, + "learning_rate": 0.0001, + "loss": 70.9015, + "step": 8520 + }, + { + "epoch": 1.3773843886881574, + "grad_norm": 0.6525025367736816, + "learning_rate": 0.0001, + "loss": 72.3132, + "step": 8530 + }, + { + "epoch": 1.3789992127732584, + "grad_norm": 0.6548861861228943, + "learning_rate": 0.0001, + "loss": 68.6495, + "step": 8540 + }, + { + "epoch": 1.3806140368583597, + "grad_norm": 0.7881495952606201, + "learning_rate": 0.0001, + "loss": 70.1335, + "step": 8550 + }, + { + "epoch": 1.382228860943461, + "grad_norm": 0.619380533695221, + "learning_rate": 0.0001, + "loss": 71.7469, + "step": 8560 + }, + { + "epoch": 1.3838436850285623, + "grad_norm": 0.7637182474136353, + "learning_rate": 0.0001, + "loss": 69.4691, + "step": 8570 + }, + { + "epoch": 1.3854585091136635, + "grad_norm": 0.8711991906166077, + "learning_rate": 0.0001, + "loss": 70.8131, + "step": 8580 + }, + { + "epoch": 1.3870733331987646, + "grad_norm": 0.6550050973892212, + "learning_rate": 0.0001, + "loss": 70.0096, + "step": 8590 + }, + { + "epoch": 1.3886881572838659, + "grad_norm": 0.6310496926307678, + "learning_rate": 0.0001, + "loss": 70.4305, + "step": 8600 + }, + { + "epoch": 1.3903029813689671, + "grad_norm": 0.5780142545700073, + "learning_rate": 0.0001, + "loss": 68.9639, + "step": 8610 + }, + { + "epoch": 1.3919178054540684, + "grad_norm": 0.6526662111282349, + "learning_rate": 0.0001, + "loss": 69.1345, + "step": 8620 + }, + { + "epoch": 1.3935326295391697, + "grad_norm": 1.2579848766326904, + "learning_rate": 0.0001, + "loss": 71.6908, + "step": 8630 + }, + { + "epoch": 1.3951474536242707, + "grad_norm": 0.7702625393867493, + "learning_rate": 0.0001, + "loss": 69.1064, + "step": 8640 + }, + { + "epoch": 1.396762277709372, + "grad_norm": 0.7414085865020752, + "learning_rate": 0.0001, + "loss": 69.5106, + "step": 8650 + }, + { + "epoch": 1.3983771017944733, + "grad_norm": 0.666235089302063, + "learning_rate": 0.0001, + "loss": 67.938, + "step": 8660 + }, + { + "epoch": 1.3999919258795746, + "grad_norm": 0.7879519462585449, + "learning_rate": 0.0001, + "loss": 69.5422, + "step": 8670 + }, + { + "epoch": 1.4016067499646758, + "grad_norm": 0.6552250981330872, + "learning_rate": 0.0001, + "loss": 69.4161, + "step": 8680 + }, + { + "epoch": 1.4032215740497769, + "grad_norm": 0.7558363676071167, + "learning_rate": 0.0001, + "loss": 71.3052, + "step": 8690 + }, + { + "epoch": 1.4048363981348782, + "grad_norm": 0.677007794380188, + "learning_rate": 0.0001, + "loss": 70.0824, + "step": 8700 + }, + { + "epoch": 1.4064512222199794, + "grad_norm": 0.6416738629341125, + "learning_rate": 0.0001, + "loss": 68.3357, + "step": 8710 + }, + { + "epoch": 1.4080660463050807, + "grad_norm": 0.8067912459373474, + "learning_rate": 0.0001, + "loss": 69.8551, + "step": 8720 + }, + { + "epoch": 1.409680870390182, + "grad_norm": 0.6017255783081055, + "learning_rate": 0.0001, + "loss": 70.3936, + "step": 8730 + }, + { + "epoch": 1.411295694475283, + "grad_norm": 0.5473611354827881, + "learning_rate": 0.0001, + "loss": 69.1556, + "step": 8740 + }, + { + "epoch": 1.4129105185603843, + "grad_norm": 0.641984760761261, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 8750 + }, + { + "epoch": 1.4145253426454856, + "grad_norm": 0.7509932518005371, + "learning_rate": 0.0001, + "loss": 68.793, + "step": 8760 + }, + { + "epoch": 1.4161401667305868, + "grad_norm": 0.615459144115448, + "learning_rate": 0.0001, + "loss": 69.0787, + "step": 8770 + }, + { + "epoch": 1.4177549908156881, + "grad_norm": 0.625276505947113, + "learning_rate": 0.0001, + "loss": 68.5643, + "step": 8780 + }, + { + "epoch": 1.4193698149007892, + "grad_norm": 0.7294536828994751, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 8790 + }, + { + "epoch": 1.4209846389858904, + "grad_norm": 0.8188890814781189, + "learning_rate": 0.0001, + "loss": 70.7486, + "step": 8800 + }, + { + "epoch": 1.4225994630709917, + "grad_norm": 0.8662137389183044, + "learning_rate": 0.0001, + "loss": 69.0625, + "step": 8810 + }, + { + "epoch": 1.424214287156093, + "grad_norm": 0.7548071146011353, + "learning_rate": 0.0001, + "loss": 68.9828, + "step": 8820 + }, + { + "epoch": 1.4258291112411943, + "grad_norm": 0.7295185327529907, + "learning_rate": 0.0001, + "loss": 69.0606, + "step": 8830 + }, + { + "epoch": 1.4274439353262953, + "grad_norm": 0.7937467098236084, + "learning_rate": 0.0001, + "loss": 69.0135, + "step": 8840 + }, + { + "epoch": 1.4290587594113966, + "grad_norm": 0.5957842469215393, + "learning_rate": 0.0001, + "loss": 68.3992, + "step": 8850 + }, + { + "epoch": 1.4306735834964979, + "grad_norm": 0.786580502986908, + "learning_rate": 0.0001, + "loss": 69.4485, + "step": 8860 + }, + { + "epoch": 1.4322884075815991, + "grad_norm": 0.8539725542068481, + "learning_rate": 0.0001, + "loss": 67.1037, + "step": 8870 + }, + { + "epoch": 1.4339032316667004, + "grad_norm": 0.684476375579834, + "learning_rate": 0.0001, + "loss": 68.9814, + "step": 8880 + }, + { + "epoch": 1.4355180557518015, + "grad_norm": 0.6335768103599548, + "learning_rate": 0.0001, + "loss": 69.9716, + "step": 8890 + }, + { + "epoch": 1.4371328798369027, + "grad_norm": 0.9881808757781982, + "learning_rate": 0.0001, + "loss": 68.0338, + "step": 8900 + }, + { + "epoch": 1.438747703922004, + "grad_norm": 0.783983588218689, + "learning_rate": 0.0001, + "loss": 70.124, + "step": 8910 + }, + { + "epoch": 1.4403625280071053, + "grad_norm": 0.817820131778717, + "learning_rate": 0.0001, + "loss": 67.933, + "step": 8920 + }, + { + "epoch": 1.4419773520922066, + "grad_norm": 0.6900199055671692, + "learning_rate": 0.0001, + "loss": 69.8871, + "step": 8930 + }, + { + "epoch": 1.4435921761773076, + "grad_norm": 0.6702716946601868, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8940 + }, + { + "epoch": 1.445207000262409, + "grad_norm": 0.6699396967887878, + "learning_rate": 0.0001, + "loss": 67.8654, + "step": 8950 + }, + { + "epoch": 1.4468218243475102, + "grad_norm": 0.6388489603996277, + "learning_rate": 0.0001, + "loss": 69.5605, + "step": 8960 + }, + { + "epoch": 1.4484366484326114, + "grad_norm": 0.735910952091217, + "learning_rate": 0.0001, + "loss": 66.7865, + "step": 8970 + }, + { + "epoch": 1.4500514725177127, + "grad_norm": 0.6826426386833191, + "learning_rate": 0.0001, + "loss": 68.2896, + "step": 8980 + }, + { + "epoch": 1.4516662966028138, + "grad_norm": 0.8190213441848755, + "learning_rate": 0.0001, + "loss": 70.3524, + "step": 8990 + }, + { + "epoch": 1.453281120687915, + "grad_norm": 0.6774603724479675, + "learning_rate": 0.0001, + "loss": 68.9084, + "step": 9000 + }, + { + "epoch": 1.4548959447730163, + "grad_norm": 0.8953077793121338, + "learning_rate": 0.0001, + "loss": 69.228, + "step": 9010 + }, + { + "epoch": 1.4565107688581176, + "grad_norm": 1.8509501218795776, + "learning_rate": 0.0001, + "loss": 68.0847, + "step": 9020 + }, + { + "epoch": 1.4581255929432189, + "grad_norm": 0.671280026435852, + "learning_rate": 0.0001, + "loss": 70.0691, + "step": 9030 + }, + { + "epoch": 1.45974041702832, + "grad_norm": 0.7242400646209717, + "learning_rate": 0.0001, + "loss": 68.8001, + "step": 9040 + }, + { + "epoch": 1.4613552411134212, + "grad_norm": 0.6609033942222595, + "learning_rate": 0.0001, + "loss": 69.0398, + "step": 9050 + }, + { + "epoch": 1.4629700651985225, + "grad_norm": 0.6850711703300476, + "learning_rate": 0.0001, + "loss": 70.8849, + "step": 9060 + }, + { + "epoch": 1.4645848892836237, + "grad_norm": 0.7409449219703674, + "learning_rate": 0.0001, + "loss": 68.0411, + "step": 9070 + }, + { + "epoch": 1.466199713368725, + "grad_norm": 0.6832754611968994, + "learning_rate": 0.0001, + "loss": 69.1219, + "step": 9080 + }, + { + "epoch": 1.467814537453826, + "grad_norm": 0.6671826243400574, + "learning_rate": 0.0001, + "loss": 68.1225, + "step": 9090 + }, + { + "epoch": 1.4694293615389273, + "grad_norm": 0.6758706569671631, + "learning_rate": 0.0001, + "loss": 68.6981, + "step": 9100 + }, + { + "epoch": 1.4710441856240286, + "grad_norm": 0.7885150909423828, + "learning_rate": 0.0001, + "loss": 69.29, + "step": 9110 + }, + { + "epoch": 1.4726590097091299, + "grad_norm": 0.7636108994483948, + "learning_rate": 0.0001, + "loss": 67.3151, + "step": 9120 + }, + { + "epoch": 1.4742738337942312, + "grad_norm": 0.6518180966377258, + "learning_rate": 0.0001, + "loss": 68.6208, + "step": 9130 + }, + { + "epoch": 1.4758886578793322, + "grad_norm": 0.6715161204338074, + "learning_rate": 0.0001, + "loss": 70.5106, + "step": 9140 + }, + { + "epoch": 1.4775034819644335, + "grad_norm": 0.6342995762825012, + "learning_rate": 0.0001, + "loss": 70.3067, + "step": 9150 + }, + { + "epoch": 1.4791183060495348, + "grad_norm": 0.6773675084114075, + "learning_rate": 0.0001, + "loss": 69.2888, + "step": 9160 + }, + { + "epoch": 1.480733130134636, + "grad_norm": 0.6887578368186951, + "learning_rate": 0.0001, + "loss": 69.2788, + "step": 9170 + }, + { + "epoch": 1.4823479542197373, + "grad_norm": 0.6793885231018066, + "learning_rate": 0.0001, + "loss": 68.3523, + "step": 9180 + }, + { + "epoch": 1.4839627783048384, + "grad_norm": 0.7483563423156738, + "learning_rate": 0.0001, + "loss": 67.9991, + "step": 9190 + }, + { + "epoch": 1.4855776023899396, + "grad_norm": 0.7007381916046143, + "learning_rate": 0.0001, + "loss": 67.5335, + "step": 9200 + }, + { + "epoch": 1.487192426475041, + "grad_norm": 0.7565139532089233, + "learning_rate": 0.0001, + "loss": 69.1093, + "step": 9210 + }, + { + "epoch": 1.4888072505601422, + "grad_norm": 1.0212360620498657, + "learning_rate": 0.0001, + "loss": 68.5061, + "step": 9220 + }, + { + "epoch": 1.4904220746452435, + "grad_norm": 0.9198092818260193, + "learning_rate": 0.0001, + "loss": 67.9735, + "step": 9230 + }, + { + "epoch": 1.4920368987303445, + "grad_norm": 0.6228001713752747, + "learning_rate": 0.0001, + "loss": 69.2231, + "step": 9240 + }, + { + "epoch": 1.4936517228154458, + "grad_norm": 0.6618618965148926, + "learning_rate": 0.0001, + "loss": 68.0434, + "step": 9250 + }, + { + "epoch": 1.495266546900547, + "grad_norm": 0.6241064071655273, + "learning_rate": 0.0001, + "loss": 68.4004, + "step": 9260 + }, + { + "epoch": 1.4968813709856483, + "grad_norm": 0.7336916327476501, + "learning_rate": 0.0001, + "loss": 69.0389, + "step": 9270 + }, + { + "epoch": 1.4984961950707496, + "grad_norm": 0.718125581741333, + "learning_rate": 0.0001, + "loss": 68.4923, + "step": 9280 + }, + { + "epoch": 1.5001110191558507, + "grad_norm": 0.6694948673248291, + "learning_rate": 0.0001, + "loss": 68.1006, + "step": 9290 + }, + { + "epoch": 1.501725843240952, + "grad_norm": 0.5815121531486511, + "learning_rate": 0.0001, + "loss": 68.5943, + "step": 9300 + }, + { + "epoch": 1.5033406673260532, + "grad_norm": 1.4076049327850342, + "learning_rate": 0.0001, + "loss": 68.1251, + "step": 9310 + }, + { + "epoch": 1.5049554914111543, + "grad_norm": 0.7294307947158813, + "learning_rate": 0.0001, + "loss": 68.0539, + "step": 9320 + }, + { + "epoch": 1.5065703154962558, + "grad_norm": 0.627821683883667, + "learning_rate": 0.0001, + "loss": 68.6859, + "step": 9330 + }, + { + "epoch": 1.5081851395813568, + "grad_norm": 0.6683267951011658, + "learning_rate": 0.0001, + "loss": 68.2727, + "step": 9340 + }, + { + "epoch": 1.509799963666458, + "grad_norm": 0.7070158123970032, + "learning_rate": 0.0001, + "loss": 69.1171, + "step": 9350 + }, + { + "epoch": 1.5114147877515594, + "grad_norm": 0.6324070692062378, + "learning_rate": 0.0001, + "loss": 69.7498, + "step": 9360 + }, + { + "epoch": 1.5130296118366604, + "grad_norm": 0.9814477562904358, + "learning_rate": 0.0001, + "loss": 69.1426, + "step": 9370 + }, + { + "epoch": 1.514644435921762, + "grad_norm": 0.5883511304855347, + "learning_rate": 0.0001, + "loss": 68.5363, + "step": 9380 + }, + { + "epoch": 1.516259260006863, + "grad_norm": 0.8325281739234924, + "learning_rate": 0.0001, + "loss": 67.5951, + "step": 9390 + }, + { + "epoch": 1.5178740840919642, + "grad_norm": 0.7456872463226318, + "learning_rate": 0.0001, + "loss": 68.1876, + "step": 9400 + }, + { + "epoch": 1.5194889081770655, + "grad_norm": 0.7692755460739136, + "learning_rate": 0.0001, + "loss": 69.1677, + "step": 9410 + }, + { + "epoch": 1.5211037322621666, + "grad_norm": 0.7346193194389343, + "learning_rate": 0.0001, + "loss": 69.8608, + "step": 9420 + }, + { + "epoch": 1.522718556347268, + "grad_norm": 0.7658166289329529, + "learning_rate": 0.0001, + "loss": 68.8584, + "step": 9430 + }, + { + "epoch": 1.524333380432369, + "grad_norm": 0.7892371416091919, + "learning_rate": 0.0001, + "loss": 68.3882, + "step": 9440 + }, + { + "epoch": 1.5259482045174704, + "grad_norm": 0.5943968296051025, + "learning_rate": 0.0001, + "loss": 68.7935, + "step": 9450 + }, + { + "epoch": 1.5275630286025716, + "grad_norm": 0.6611922383308411, + "learning_rate": 0.0001, + "loss": 69.0865, + "step": 9460 + }, + { + "epoch": 1.5291778526876727, + "grad_norm": 0.6697778105735779, + "learning_rate": 0.0001, + "loss": 68.6131, + "step": 9470 + }, + { + "epoch": 1.5307926767727742, + "grad_norm": 0.862293541431427, + "learning_rate": 0.0001, + "loss": 71.0167, + "step": 9480 + }, + { + "epoch": 1.5324075008578752, + "grad_norm": 0.7098677754402161, + "learning_rate": 0.0001, + "loss": 67.7216, + "step": 9490 + }, + { + "epoch": 1.5340223249429765, + "grad_norm": 0.6578338742256165, + "learning_rate": 0.0001, + "loss": 69.872, + "step": 9500 + }, + { + "epoch": 1.5356371490280778, + "grad_norm": 0.7272375822067261, + "learning_rate": 0.0001, + "loss": 70.0971, + "step": 9510 + }, + { + "epoch": 1.5372519731131788, + "grad_norm": 0.6497064828872681, + "learning_rate": 0.0001, + "loss": 70.5921, + "step": 9520 + }, + { + "epoch": 1.5388667971982803, + "grad_norm": 0.6691288352012634, + "learning_rate": 0.0001, + "loss": 68.5213, + "step": 9530 + }, + { + "epoch": 1.5404816212833814, + "grad_norm": 0.9180259108543396, + "learning_rate": 0.0001, + "loss": 67.0699, + "step": 9540 + }, + { + "epoch": 1.5420964453684827, + "grad_norm": 0.7062082886695862, + "learning_rate": 0.0001, + "loss": 69.3965, + "step": 9550 + }, + { + "epoch": 1.543711269453584, + "grad_norm": 0.711613655090332, + "learning_rate": 0.0001, + "loss": 67.8276, + "step": 9560 + }, + { + "epoch": 1.545326093538685, + "grad_norm": 0.670032799243927, + "learning_rate": 0.0001, + "loss": 68.1687, + "step": 9570 + }, + { + "epoch": 1.5469409176237865, + "grad_norm": 0.7531955242156982, + "learning_rate": 0.0001, + "loss": 69.2144, + "step": 9580 + }, + { + "epoch": 1.5485557417088875, + "grad_norm": 0.6759967803955078, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 9590 + }, + { + "epoch": 1.5501705657939888, + "grad_norm": 0.6134868860244751, + "learning_rate": 0.0001, + "loss": 67.4348, + "step": 9600 + }, + { + "epoch": 1.55178538987909, + "grad_norm": 0.638785719871521, + "learning_rate": 0.0001, + "loss": 68.4635, + "step": 9610 + }, + { + "epoch": 1.5534002139641911, + "grad_norm": 0.7901169657707214, + "learning_rate": 0.0001, + "loss": 68.1079, + "step": 9620 + }, + { + "epoch": 1.5550150380492926, + "grad_norm": 0.6970444917678833, + "learning_rate": 0.0001, + "loss": 69.3883, + "step": 9630 + }, + { + "epoch": 1.5566298621343937, + "grad_norm": 0.6954355239868164, + "learning_rate": 0.0001, + "loss": 69.3841, + "step": 9640 + }, + { + "epoch": 1.558244686219495, + "grad_norm": 0.7097641825675964, + "learning_rate": 0.0001, + "loss": 68.3417, + "step": 9650 + }, + { + "epoch": 1.5598595103045962, + "grad_norm": 0.7082354426383972, + "learning_rate": 0.0001, + "loss": 69.9314, + "step": 9660 + }, + { + "epoch": 1.5614743343896973, + "grad_norm": 0.6570103764533997, + "learning_rate": 0.0001, + "loss": 69.7219, + "step": 9670 + }, + { + "epoch": 1.5630891584747988, + "grad_norm": 0.5705522894859314, + "learning_rate": 0.0001, + "loss": 67.5177, + "step": 9680 + }, + { + "epoch": 1.5647039825598998, + "grad_norm": 0.6992695331573486, + "learning_rate": 0.0001, + "loss": 67.6538, + "step": 9690 + }, + { + "epoch": 1.5663188066450011, + "grad_norm": 0.7523599863052368, + "learning_rate": 0.0001, + "loss": 66.869, + "step": 9700 + }, + { + "epoch": 1.5679336307301024, + "grad_norm": 0.6267555356025696, + "learning_rate": 0.0001, + "loss": 66.6805, + "step": 9710 + }, + { + "epoch": 1.5695484548152034, + "grad_norm": 0.6138116717338562, + "learning_rate": 0.0001, + "loss": 71.3217, + "step": 9720 + }, + { + "epoch": 1.571163278900305, + "grad_norm": 0.7408527731895447, + "learning_rate": 0.0001, + "loss": 69.2759, + "step": 9730 + }, + { + "epoch": 1.572778102985406, + "grad_norm": 0.813761830329895, + "learning_rate": 0.0001, + "loss": 70.4959, + "step": 9740 + }, + { + "epoch": 1.5743929270705073, + "grad_norm": 1.0363599061965942, + "learning_rate": 0.0001, + "loss": 68.1996, + "step": 9750 + }, + { + "epoch": 1.5760077511556085, + "grad_norm": 0.7773903012275696, + "learning_rate": 0.0001, + "loss": 67.9354, + "step": 9760 + }, + { + "epoch": 1.5776225752407096, + "grad_norm": 0.6465738415718079, + "learning_rate": 0.0001, + "loss": 69.5247, + "step": 9770 + }, + { + "epoch": 1.579237399325811, + "grad_norm": 0.8433385491371155, + "learning_rate": 0.0001, + "loss": 66.9448, + "step": 9780 + }, + { + "epoch": 1.5808522234109121, + "grad_norm": 0.8651545643806458, + "learning_rate": 0.0001, + "loss": 68.6994, + "step": 9790 + }, + { + "epoch": 1.5824670474960134, + "grad_norm": 0.6940448880195618, + "learning_rate": 0.0001, + "loss": 67.622, + "step": 9800 + }, + { + "epoch": 1.5840818715811147, + "grad_norm": 0.622756838798523, + "learning_rate": 0.0001, + "loss": 69.0994, + "step": 9810 + }, + { + "epoch": 1.5856966956662157, + "grad_norm": 0.6119604110717773, + "learning_rate": 0.0001, + "loss": 67.7682, + "step": 9820 + }, + { + "epoch": 1.5873115197513172, + "grad_norm": 0.5729681849479675, + "learning_rate": 0.0001, + "loss": 67.3113, + "step": 9830 + }, + { + "epoch": 1.5889263438364183, + "grad_norm": 0.7143844366073608, + "learning_rate": 0.0001, + "loss": 68.8128, + "step": 9840 + }, + { + "epoch": 1.5905411679215196, + "grad_norm": 0.9814873337745667, + "learning_rate": 0.0001, + "loss": 69.3099, + "step": 9850 + }, + { + "epoch": 1.5921559920066208, + "grad_norm": 0.8044945001602173, + "learning_rate": 0.0001, + "loss": 69.3939, + "step": 9860 + }, + { + "epoch": 1.5937708160917219, + "grad_norm": 0.8494839668273926, + "learning_rate": 0.0001, + "loss": 69.3889, + "step": 9870 + }, + { + "epoch": 1.5953856401768234, + "grad_norm": 0.6468870639801025, + "learning_rate": 0.0001, + "loss": 67.2047, + "step": 9880 + }, + { + "epoch": 1.5970004642619244, + "grad_norm": 0.6270474791526794, + "learning_rate": 0.0001, + "loss": 69.6813, + "step": 9890 + }, + { + "epoch": 1.5986152883470257, + "grad_norm": 0.7102645039558411, + "learning_rate": 0.0001, + "loss": 69.1546, + "step": 9900 + }, + { + "epoch": 1.600230112432127, + "grad_norm": 0.5929123759269714, + "learning_rate": 0.0001, + "loss": 67.2187, + "step": 9910 + }, + { + "epoch": 1.601844936517228, + "grad_norm": 0.6466200947761536, + "learning_rate": 0.0001, + "loss": 68.6876, + "step": 9920 + }, + { + "epoch": 1.6034597606023295, + "grad_norm": 0.6441522836685181, + "learning_rate": 0.0001, + "loss": 68.7955, + "step": 9930 + }, + { + "epoch": 1.6050745846874306, + "grad_norm": 0.7264062166213989, + "learning_rate": 0.0001, + "loss": 67.8876, + "step": 9940 + }, + { + "epoch": 1.6066894087725319, + "grad_norm": 0.6973199248313904, + "learning_rate": 0.0001, + "loss": 69.0904, + "step": 9950 + }, + { + "epoch": 1.6083042328576331, + "grad_norm": 0.582053005695343, + "learning_rate": 0.0001, + "loss": 69.3876, + "step": 9960 + }, + { + "epoch": 1.6099190569427342, + "grad_norm": 0.6431038975715637, + "learning_rate": 0.0001, + "loss": 67.4232, + "step": 9970 + }, + { + "epoch": 1.6115338810278357, + "grad_norm": 0.6313247084617615, + "learning_rate": 0.0001, + "loss": 68.3759, + "step": 9980 + }, + { + "epoch": 1.6131487051129367, + "grad_norm": 1.2003700733184814, + "learning_rate": 0.0001, + "loss": 68.3413, + "step": 9990 + }, + { + "epoch": 1.614763529198038, + "grad_norm": 1.0262068510055542, + "learning_rate": 0.0001, + "loss": 66.7308, + "step": 10000 + }, + { + "epoch": 1.6163783532831393, + "grad_norm": 0.5975973010063171, + "learning_rate": 0.0001, + "loss": 68.6893, + "step": 10010 + }, + { + "epoch": 1.6179931773682403, + "grad_norm": 0.6045264601707458, + "learning_rate": 0.0001, + "loss": 69.5241, + "step": 10020 + }, + { + "epoch": 1.6196080014533418, + "grad_norm": 0.654019832611084, + "learning_rate": 0.0001, + "loss": 66.1208, + "step": 10030 + }, + { + "epoch": 1.6212228255384429, + "grad_norm": 0.533981204032898, + "learning_rate": 0.0001, + "loss": 68.2129, + "step": 10040 + }, + { + "epoch": 1.6228376496235442, + "grad_norm": 1.184734582901001, + "learning_rate": 0.0001, + "loss": 70.0547, + "step": 10050 + }, + { + "epoch": 1.6244524737086454, + "grad_norm": 0.5836975574493408, + "learning_rate": 0.0001, + "loss": 68.1098, + "step": 10060 + }, + { + "epoch": 1.6260672977937465, + "grad_norm": 0.5856032967567444, + "learning_rate": 0.0001, + "loss": 67.483, + "step": 10070 + }, + { + "epoch": 1.627682121878848, + "grad_norm": 0.6735382080078125, + "learning_rate": 0.0001, + "loss": 68.8628, + "step": 10080 + }, + { + "epoch": 1.629296945963949, + "grad_norm": 0.5753204226493835, + "learning_rate": 0.0001, + "loss": 66.2777, + "step": 10090 + }, + { + "epoch": 1.6309117700490503, + "grad_norm": 0.8327603340148926, + "learning_rate": 0.0001, + "loss": 67.9309, + "step": 10100 + }, + { + "epoch": 1.6325265941341516, + "grad_norm": 0.6873740553855896, + "learning_rate": 0.0001, + "loss": 68.7627, + "step": 10110 + }, + { + "epoch": 1.6341414182192526, + "grad_norm": 0.7218474745750427, + "learning_rate": 0.0001, + "loss": 69.3637, + "step": 10120 + }, + { + "epoch": 1.6357562423043541, + "grad_norm": 0.8393228650093079, + "learning_rate": 0.0001, + "loss": 66.7686, + "step": 10130 + }, + { + "epoch": 1.6373710663894552, + "grad_norm": 0.6433869004249573, + "learning_rate": 0.0001, + "loss": 68.0896, + "step": 10140 + }, + { + "epoch": 1.6389858904745565, + "grad_norm": 0.7419982552528381, + "learning_rate": 0.0001, + "loss": 67.9707, + "step": 10150 + }, + { + "epoch": 1.6406007145596577, + "grad_norm": 0.5884562730789185, + "learning_rate": 0.0001, + "loss": 68.0287, + "step": 10160 + }, + { + "epoch": 1.6422155386447588, + "grad_norm": 0.729557454586029, + "learning_rate": 0.0001, + "loss": 69.942, + "step": 10170 + }, + { + "epoch": 1.6438303627298603, + "grad_norm": 0.6026669144630432, + "learning_rate": 0.0001, + "loss": 66.8192, + "step": 10180 + }, + { + "epoch": 1.6454451868149613, + "grad_norm": 0.6921732425689697, + "learning_rate": 0.0001, + "loss": 66.7087, + "step": 10190 + }, + { + "epoch": 1.6470600109000626, + "grad_norm": 0.755172848701477, + "learning_rate": 0.0001, + "loss": 70.799, + "step": 10200 + }, + { + "epoch": 1.6486748349851639, + "grad_norm": 0.7318310737609863, + "learning_rate": 0.0001, + "loss": 67.6957, + "step": 10210 + }, + { + "epoch": 1.650289659070265, + "grad_norm": 0.6915041208267212, + "learning_rate": 0.0001, + "loss": 69.3388, + "step": 10220 + }, + { + "epoch": 1.6519044831553664, + "grad_norm": 0.7173444628715515, + "learning_rate": 0.0001, + "loss": 68.3612, + "step": 10230 + }, + { + "epoch": 1.6535193072404675, + "grad_norm": 0.7485487461090088, + "learning_rate": 0.0001, + "loss": 68.5366, + "step": 10240 + }, + { + "epoch": 1.6551341313255687, + "grad_norm": 0.5838190913200378, + "learning_rate": 0.0001, + "loss": 68.3841, + "step": 10250 + }, + { + "epoch": 1.65674895541067, + "grad_norm": 0.5908819437026978, + "learning_rate": 0.0001, + "loss": 67.8741, + "step": 10260 + }, + { + "epoch": 1.658363779495771, + "grad_norm": 0.7475612163543701, + "learning_rate": 0.0001, + "loss": 69.0636, + "step": 10270 + }, + { + "epoch": 1.6599786035808726, + "grad_norm": 0.5451382994651794, + "learning_rate": 0.0001, + "loss": 68.6695, + "step": 10280 + }, + { + "epoch": 1.6615934276659736, + "grad_norm": 0.6277157068252563, + "learning_rate": 0.0001, + "loss": 67.5432, + "step": 10290 + }, + { + "epoch": 1.663208251751075, + "grad_norm": 0.8144198060035706, + "learning_rate": 0.0001, + "loss": 66.7892, + "step": 10300 + }, + { + "epoch": 1.6648230758361762, + "grad_norm": 0.7400529980659485, + "learning_rate": 0.0001, + "loss": 66.2653, + "step": 10310 + }, + { + "epoch": 1.6664378999212772, + "grad_norm": 0.74524986743927, + "learning_rate": 0.0001, + "loss": 67.8252, + "step": 10320 + }, + { + "epoch": 1.6680527240063787, + "grad_norm": 0.5554162263870239, + "learning_rate": 0.0001, + "loss": 69.7704, + "step": 10330 + }, + { + "epoch": 1.6696675480914798, + "grad_norm": 0.8194338083267212, + "learning_rate": 0.0001, + "loss": 69.4668, + "step": 10340 + }, + { + "epoch": 1.671282372176581, + "grad_norm": 0.7947783470153809, + "learning_rate": 0.0001, + "loss": 64.9276, + "step": 10350 + }, + { + "epoch": 1.6728971962616823, + "grad_norm": 0.6145230531692505, + "learning_rate": 0.0001, + "loss": 68.1815, + "step": 10360 + }, + { + "epoch": 1.6745120203467834, + "grad_norm": 0.6293516755104065, + "learning_rate": 0.0001, + "loss": 67.8994, + "step": 10370 + }, + { + "epoch": 1.6761268444318849, + "grad_norm": 0.6160871386528015, + "learning_rate": 0.0001, + "loss": 66.7463, + "step": 10380 + }, + { + "epoch": 1.677741668516986, + "grad_norm": 0.6925204992294312, + "learning_rate": 0.0001, + "loss": 68.5261, + "step": 10390 + }, + { + "epoch": 1.6793564926020872, + "grad_norm": 0.6892788410186768, + "learning_rate": 0.0001, + "loss": 66.8928, + "step": 10400 + }, + { + "epoch": 1.6809713166871885, + "grad_norm": 0.9066945910453796, + "learning_rate": 0.0001, + "loss": 66.2556, + "step": 10410 + }, + { + "epoch": 1.6825861407722895, + "grad_norm": 0.5857477784156799, + "learning_rate": 0.0001, + "loss": 67.1866, + "step": 10420 + }, + { + "epoch": 1.684200964857391, + "grad_norm": 0.7989841103553772, + "learning_rate": 0.0001, + "loss": 67.7762, + "step": 10430 + }, + { + "epoch": 1.685815788942492, + "grad_norm": 0.559428870677948, + "learning_rate": 0.0001, + "loss": 67.5628, + "step": 10440 + }, + { + "epoch": 1.6874306130275933, + "grad_norm": 0.6718077659606934, + "learning_rate": 0.0001, + "loss": 69.2877, + "step": 10450 + }, + { + "epoch": 1.6890454371126946, + "grad_norm": 0.6952136754989624, + "learning_rate": 0.0001, + "loss": 67.2734, + "step": 10460 + }, + { + "epoch": 1.6906602611977957, + "grad_norm": 0.6079897880554199, + "learning_rate": 0.0001, + "loss": 67.4677, + "step": 10470 + }, + { + "epoch": 1.692275085282897, + "grad_norm": 0.6139684915542603, + "learning_rate": 0.0001, + "loss": 69.2464, + "step": 10480 + }, + { + "epoch": 1.6938899093679982, + "grad_norm": 0.7009351253509521, + "learning_rate": 0.0001, + "loss": 67.5424, + "step": 10490 + }, + { + "epoch": 1.6955047334530995, + "grad_norm": 0.8567383885383606, + "learning_rate": 0.0001, + "loss": 65.6573, + "step": 10500 + }, + { + "epoch": 1.6971195575382008, + "grad_norm": 0.6903256177902222, + "learning_rate": 0.0001, + "loss": 69.2218, + "step": 10510 + }, + { + "epoch": 1.6987343816233018, + "grad_norm": 0.5958806872367859, + "learning_rate": 0.0001, + "loss": 67.3718, + "step": 10520 + }, + { + "epoch": 1.700349205708403, + "grad_norm": 0.5983722805976868, + "learning_rate": 0.0001, + "loss": 68.7944, + "step": 10530 + }, + { + "epoch": 1.7019640297935044, + "grad_norm": 0.8119397759437561, + "learning_rate": 0.0001, + "loss": 66.7936, + "step": 10540 + }, + { + "epoch": 1.7035788538786056, + "grad_norm": 0.6383484601974487, + "learning_rate": 0.0001, + "loss": 67.8636, + "step": 10550 + }, + { + "epoch": 1.705193677963707, + "grad_norm": 0.6574084162712097, + "learning_rate": 0.0001, + "loss": 67.26, + "step": 10560 + }, + { + "epoch": 1.706808502048808, + "grad_norm": 0.6716108918190002, + "learning_rate": 0.0001, + "loss": 66.485, + "step": 10570 + }, + { + "epoch": 1.7084233261339092, + "grad_norm": 0.6998292803764343, + "learning_rate": 0.0001, + "loss": 66.4201, + "step": 10580 + }, + { + "epoch": 1.7100381502190105, + "grad_norm": 0.7300788760185242, + "learning_rate": 0.0001, + "loss": 67.1446, + "step": 10590 + }, + { + "epoch": 1.7116529743041118, + "grad_norm": 0.7066853642463684, + "learning_rate": 0.0001, + "loss": 70.8954, + "step": 10600 + }, + { + "epoch": 1.713267798389213, + "grad_norm": 0.6933701038360596, + "learning_rate": 0.0001, + "loss": 67.8473, + "step": 10610 + }, + { + "epoch": 1.7148826224743141, + "grad_norm": 0.6885779500007629, + "learning_rate": 0.0001, + "loss": 65.3276, + "step": 10620 + }, + { + "epoch": 1.7164974465594154, + "grad_norm": 0.6542202830314636, + "learning_rate": 0.0001, + "loss": 69.2596, + "step": 10630 + }, + { + "epoch": 1.7181122706445167, + "grad_norm": 0.5719691514968872, + "learning_rate": 0.0001, + "loss": 69.1907, + "step": 10640 + }, + { + "epoch": 1.719727094729618, + "grad_norm": 0.6596891283988953, + "learning_rate": 0.0001, + "loss": 69.0778, + "step": 10650 + }, + { + "epoch": 1.7213419188147192, + "grad_norm": 0.6860799193382263, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 10660 + }, + { + "epoch": 1.7229567428998203, + "grad_norm": 0.7912564277648926, + "learning_rate": 0.0001, + "loss": 68.2091, + "step": 10670 + }, + { + "epoch": 1.7245715669849215, + "grad_norm": 0.6673903465270996, + "learning_rate": 0.0001, + "loss": 68.24, + "step": 10680 + }, + { + "epoch": 1.7261863910700228, + "grad_norm": 0.6243769526481628, + "learning_rate": 0.0001, + "loss": 68.6396, + "step": 10690 + }, + { + "epoch": 1.727801215155124, + "grad_norm": 0.6622587442398071, + "learning_rate": 0.0001, + "loss": 67.9406, + "step": 10700 + }, + { + "epoch": 1.7294160392402254, + "grad_norm": 0.6808138489723206, + "learning_rate": 0.0001, + "loss": 69.1403, + "step": 10710 + }, + { + "epoch": 1.7310308633253264, + "grad_norm": 0.8498415350914001, + "learning_rate": 0.0001, + "loss": 70.1438, + "step": 10720 + }, + { + "epoch": 1.7326456874104277, + "grad_norm": 1.0091742277145386, + "learning_rate": 0.0001, + "loss": 67.0196, + "step": 10730 + }, + { + "epoch": 1.734260511495529, + "grad_norm": 0.6997767686843872, + "learning_rate": 0.0001, + "loss": 67.0307, + "step": 10740 + }, + { + "epoch": 1.7358753355806302, + "grad_norm": 0.6618793606758118, + "learning_rate": 0.0001, + "loss": 67.906, + "step": 10750 + }, + { + "epoch": 1.7374901596657315, + "grad_norm": 0.7082014679908752, + "learning_rate": 0.0001, + "loss": 68.0665, + "step": 10760 + }, + { + "epoch": 1.7391049837508326, + "grad_norm": 0.8494391441345215, + "learning_rate": 0.0001, + "loss": 68.4202, + "step": 10770 + }, + { + "epoch": 1.7407198078359338, + "grad_norm": 0.6247998476028442, + "learning_rate": 0.0001, + "loss": 66.4399, + "step": 10780 + }, + { + "epoch": 1.742334631921035, + "grad_norm": 0.8239936828613281, + "learning_rate": 0.0001, + "loss": 68.4374, + "step": 10790 + }, + { + "epoch": 1.7439494560061364, + "grad_norm": 0.6734293103218079, + "learning_rate": 0.0001, + "loss": 66.3411, + "step": 10800 + }, + { + "epoch": 1.7455642800912377, + "grad_norm": 0.7066450119018555, + "learning_rate": 0.0001, + "loss": 67.7102, + "step": 10810 + }, + { + "epoch": 1.7471791041763387, + "grad_norm": 0.7440647482872009, + "learning_rate": 0.0001, + "loss": 67.7221, + "step": 10820 + }, + { + "epoch": 1.74879392826144, + "grad_norm": 0.7771329283714294, + "learning_rate": 0.0001, + "loss": 67.5907, + "step": 10830 + }, + { + "epoch": 1.7504087523465413, + "grad_norm": 1.2484230995178223, + "learning_rate": 0.0001, + "loss": 66.3242, + "step": 10840 + }, + { + "epoch": 1.7520235764316425, + "grad_norm": 0.6946772933006287, + "learning_rate": 0.0001, + "loss": 67.8741, + "step": 10850 + }, + { + "epoch": 1.7536384005167438, + "grad_norm": 0.7676035761833191, + "learning_rate": 0.0001, + "loss": 67.3776, + "step": 10860 + }, + { + "epoch": 1.7552532246018449, + "grad_norm": 0.5534589290618896, + "learning_rate": 0.0001, + "loss": 67.6656, + "step": 10870 + }, + { + "epoch": 1.7568680486869461, + "grad_norm": 1.2745810747146606, + "learning_rate": 0.0001, + "loss": 68.5959, + "step": 10880 + }, + { + "epoch": 1.7584828727720474, + "grad_norm": 0.6241582036018372, + "learning_rate": 0.0001, + "loss": 67.7142, + "step": 10890 + }, + { + "epoch": 1.7600976968571487, + "grad_norm": 0.9308313131332397, + "learning_rate": 0.0001, + "loss": 67.0322, + "step": 10900 + }, + { + "epoch": 1.76171252094225, + "grad_norm": 0.5639570951461792, + "learning_rate": 0.0001, + "loss": 67.928, + "step": 10910 + }, + { + "epoch": 1.763327345027351, + "grad_norm": 0.6692782044410706, + "learning_rate": 0.0001, + "loss": 67.625, + "step": 10920 + }, + { + "epoch": 1.7649421691124523, + "grad_norm": 0.7322304248809814, + "learning_rate": 0.0001, + "loss": 66.2372, + "step": 10930 + }, + { + "epoch": 1.7665569931975535, + "grad_norm": 0.6275238394737244, + "learning_rate": 0.0001, + "loss": 67.7628, + "step": 10940 + }, + { + "epoch": 1.7681718172826548, + "grad_norm": 0.6832380890846252, + "learning_rate": 0.0001, + "loss": 67.6587, + "step": 10950 + }, + { + "epoch": 1.769786641367756, + "grad_norm": 0.6934792399406433, + "learning_rate": 0.0001, + "loss": 67.1411, + "step": 10960 + }, + { + "epoch": 1.7714014654528571, + "grad_norm": 0.7031943202018738, + "learning_rate": 0.0001, + "loss": 69.2734, + "step": 10970 + }, + { + "epoch": 1.7730162895379584, + "grad_norm": 0.91269451379776, + "learning_rate": 0.0001, + "loss": 68.1353, + "step": 10980 + }, + { + "epoch": 1.7746311136230597, + "grad_norm": 0.8927266001701355, + "learning_rate": 0.0001, + "loss": 68.003, + "step": 10990 + }, + { + "epoch": 1.776245937708161, + "grad_norm": 0.9115116596221924, + "learning_rate": 0.0001, + "loss": 69.1506, + "step": 11000 + }, + { + "epoch": 1.7778607617932622, + "grad_norm": 0.6055391430854797, + "learning_rate": 0.0001, + "loss": 68.9785, + "step": 11010 + }, + { + "epoch": 1.7794755858783633, + "grad_norm": 0.872809112071991, + "learning_rate": 0.0001, + "loss": 66.373, + "step": 11020 + }, + { + "epoch": 1.7810904099634646, + "grad_norm": 0.5838716626167297, + "learning_rate": 0.0001, + "loss": 69.507, + "step": 11030 + }, + { + "epoch": 1.7827052340485658, + "grad_norm": 0.6108883619308472, + "learning_rate": 0.0001, + "loss": 67.3743, + "step": 11040 + }, + { + "epoch": 1.784320058133667, + "grad_norm": 0.5457114577293396, + "learning_rate": 0.0001, + "loss": 66.1342, + "step": 11050 + }, + { + "epoch": 1.7859348822187684, + "grad_norm": 0.7696211338043213, + "learning_rate": 0.0001, + "loss": 67.9779, + "step": 11060 + }, + { + "epoch": 1.7875497063038694, + "grad_norm": 0.5795412659645081, + "learning_rate": 0.0001, + "loss": 67.1763, + "step": 11070 + }, + { + "epoch": 1.7891645303889707, + "grad_norm": 0.7401878833770752, + "learning_rate": 0.0001, + "loss": 67.4972, + "step": 11080 + }, + { + "epoch": 1.790779354474072, + "grad_norm": 0.666051983833313, + "learning_rate": 0.0001, + "loss": 68.7621, + "step": 11090 + }, + { + "epoch": 1.792394178559173, + "grad_norm": 0.9650185108184814, + "learning_rate": 0.0001, + "loss": 66.7721, + "step": 11100 + }, + { + "epoch": 1.7940090026442745, + "grad_norm": 0.6026908159255981, + "learning_rate": 0.0001, + "loss": 66.2993, + "step": 11110 + }, + { + "epoch": 1.7956238267293756, + "grad_norm": 0.6560576558113098, + "learning_rate": 0.0001, + "loss": 67.8543, + "step": 11120 + }, + { + "epoch": 1.7972386508144769, + "grad_norm": 0.9523794054985046, + "learning_rate": 0.0001, + "loss": 66.7881, + "step": 11130 + }, + { + "epoch": 1.7988534748995781, + "grad_norm": 0.7241806983947754, + "learning_rate": 0.0001, + "loss": 68.1071, + "step": 11140 + }, + { + "epoch": 1.8004682989846792, + "grad_norm": 0.639866292476654, + "learning_rate": 0.0001, + "loss": 68.992, + "step": 11150 + }, + { + "epoch": 1.8020831230697807, + "grad_norm": 0.6050712466239929, + "learning_rate": 0.0001, + "loss": 67.7993, + "step": 11160 + }, + { + "epoch": 1.8036979471548817, + "grad_norm": 0.5777723789215088, + "learning_rate": 0.0001, + "loss": 66.9515, + "step": 11170 + }, + { + "epoch": 1.805312771239983, + "grad_norm": 0.6153426766395569, + "learning_rate": 0.0001, + "loss": 68.3413, + "step": 11180 + }, + { + "epoch": 1.8069275953250843, + "grad_norm": 0.6118552088737488, + "learning_rate": 0.0001, + "loss": 67.7853, + "step": 11190 + }, + { + "epoch": 1.8085424194101853, + "grad_norm": 0.6520341038703918, + "learning_rate": 0.0001, + "loss": 67.3511, + "step": 11200 + }, + { + "epoch": 1.8101572434952868, + "grad_norm": 0.6189568638801575, + "learning_rate": 0.0001, + "loss": 69.28, + "step": 11210 + }, + { + "epoch": 1.811772067580388, + "grad_norm": 0.602411150932312, + "learning_rate": 0.0001, + "loss": 67.5133, + "step": 11220 + }, + { + "epoch": 1.8133868916654892, + "grad_norm": 0.7359414100646973, + "learning_rate": 0.0001, + "loss": 68.631, + "step": 11230 + }, + { + "epoch": 1.8150017157505904, + "grad_norm": 0.6216995120048523, + "learning_rate": 0.0001, + "loss": 68.3588, + "step": 11240 + }, + { + "epoch": 1.8166165398356915, + "grad_norm": 0.7683486938476562, + "learning_rate": 0.0001, + "loss": 67.1434, + "step": 11250 + }, + { + "epoch": 1.818231363920793, + "grad_norm": 0.5565283894538879, + "learning_rate": 0.0001, + "loss": 66.9995, + "step": 11260 + }, + { + "epoch": 1.819846188005894, + "grad_norm": 0.5623223781585693, + "learning_rate": 0.0001, + "loss": 68.1316, + "step": 11270 + }, + { + "epoch": 1.8214610120909953, + "grad_norm": 0.532821774482727, + "learning_rate": 0.0001, + "loss": 69.1903, + "step": 11280 + }, + { + "epoch": 1.8230758361760966, + "grad_norm": 0.6297196745872498, + "learning_rate": 0.0001, + "loss": 68.4965, + "step": 11290 + }, + { + "epoch": 1.8246906602611976, + "grad_norm": 0.839862048625946, + "learning_rate": 0.0001, + "loss": 67.1782, + "step": 11300 + }, + { + "epoch": 1.8263054843462991, + "grad_norm": 0.7021651864051819, + "learning_rate": 0.0001, + "loss": 67.4355, + "step": 11310 + }, + { + "epoch": 1.8279203084314002, + "grad_norm": 0.676034152507782, + "learning_rate": 0.0001, + "loss": 68.3721, + "step": 11320 + }, + { + "epoch": 1.8295351325165015, + "grad_norm": 0.6621078252792358, + "learning_rate": 0.0001, + "loss": 67.3255, + "step": 11330 + }, + { + "epoch": 1.8311499566016027, + "grad_norm": 0.5902897119522095, + "learning_rate": 0.0001, + "loss": 68.0641, + "step": 11340 + }, + { + "epoch": 1.8327647806867038, + "grad_norm": 0.6334371566772461, + "learning_rate": 0.0001, + "loss": 68.2682, + "step": 11350 + }, + { + "epoch": 1.8343796047718053, + "grad_norm": 0.6753348708152771, + "learning_rate": 0.0001, + "loss": 66.6749, + "step": 11360 + }, + { + "epoch": 1.8359944288569063, + "grad_norm": 0.7221750617027283, + "learning_rate": 0.0001, + "loss": 68.4142, + "step": 11370 + }, + { + "epoch": 1.8376092529420076, + "grad_norm": 0.7476344108581543, + "learning_rate": 0.0001, + "loss": 67.1416, + "step": 11380 + }, + { + "epoch": 1.8392240770271089, + "grad_norm": 0.5927479863166809, + "learning_rate": 0.0001, + "loss": 67.6368, + "step": 11390 + }, + { + "epoch": 1.84083890111221, + "grad_norm": 0.6753717660903931, + "learning_rate": 0.0001, + "loss": 67.1026, + "step": 11400 + }, + { + "epoch": 1.8424537251973114, + "grad_norm": 0.6500268578529358, + "learning_rate": 0.0001, + "loss": 68.7537, + "step": 11410 + }, + { + "epoch": 1.8440685492824125, + "grad_norm": 0.5930709838867188, + "learning_rate": 0.0001, + "loss": 65.7429, + "step": 11420 + }, + { + "epoch": 1.8456833733675138, + "grad_norm": 0.6467777490615845, + "learning_rate": 0.0001, + "loss": 68.1333, + "step": 11430 + }, + { + "epoch": 1.847298197452615, + "grad_norm": 0.6935669183731079, + "learning_rate": 0.0001, + "loss": 66.7728, + "step": 11440 + }, + { + "epoch": 1.848913021537716, + "grad_norm": 0.5838116407394409, + "learning_rate": 0.0001, + "loss": 66.6005, + "step": 11450 + }, + { + "epoch": 1.8505278456228176, + "grad_norm": 0.8621205687522888, + "learning_rate": 0.0001, + "loss": 66.0859, + "step": 11460 + }, + { + "epoch": 1.8521426697079186, + "grad_norm": 0.7119213342666626, + "learning_rate": 0.0001, + "loss": 66.6198, + "step": 11470 + }, + { + "epoch": 1.85375749379302, + "grad_norm": 0.6861911416053772, + "learning_rate": 0.0001, + "loss": 67.2275, + "step": 11480 + }, + { + "epoch": 1.8553723178781212, + "grad_norm": 0.5653937458992004, + "learning_rate": 0.0001, + "loss": 67.827, + "step": 11490 + }, + { + "epoch": 1.8569871419632222, + "grad_norm": 0.6458482146263123, + "learning_rate": 0.0001, + "loss": 66.3906, + "step": 11500 + }, + { + "epoch": 1.8586019660483237, + "grad_norm": 0.6701959371566772, + "learning_rate": 0.0001, + "loss": 67.88, + "step": 11510 + }, + { + "epoch": 1.8602167901334248, + "grad_norm": 0.6598711609840393, + "learning_rate": 0.0001, + "loss": 66.8373, + "step": 11520 + }, + { + "epoch": 1.861831614218526, + "grad_norm": 0.6095236539840698, + "learning_rate": 0.0001, + "loss": 67.6478, + "step": 11530 + }, + { + "epoch": 1.8634464383036273, + "grad_norm": 0.8192022442817688, + "learning_rate": 0.0001, + "loss": 67.7343, + "step": 11540 + }, + { + "epoch": 1.8650612623887284, + "grad_norm": 0.5788753628730774, + "learning_rate": 0.0001, + "loss": 66.7909, + "step": 11550 + }, + { + "epoch": 1.8666760864738299, + "grad_norm": 0.5383989214897156, + "learning_rate": 0.0001, + "loss": 68.5349, + "step": 11560 + }, + { + "epoch": 1.868290910558931, + "grad_norm": 0.6224480271339417, + "learning_rate": 0.0001, + "loss": 68.1938, + "step": 11570 + }, + { + "epoch": 1.8699057346440322, + "grad_norm": 0.5852788686752319, + "learning_rate": 0.0001, + "loss": 66.4678, + "step": 11580 + }, + { + "epoch": 1.8715205587291335, + "grad_norm": 0.730378270149231, + "learning_rate": 0.0001, + "loss": 67.57, + "step": 11590 + }, + { + "epoch": 1.8731353828142345, + "grad_norm": 0.6197090744972229, + "learning_rate": 0.0001, + "loss": 65.7801, + "step": 11600 + }, + { + "epoch": 1.874750206899336, + "grad_norm": 0.518974781036377, + "learning_rate": 0.0001, + "loss": 68.9308, + "step": 11610 + }, + { + "epoch": 1.876365030984437, + "grad_norm": 0.6601742506027222, + "learning_rate": 0.0001, + "loss": 66.821, + "step": 11620 + }, + { + "epoch": 1.8779798550695384, + "grad_norm": 0.7409974932670593, + "learning_rate": 0.0001, + "loss": 66.6883, + "step": 11630 + }, + { + "epoch": 1.8795946791546396, + "grad_norm": 0.6372189521789551, + "learning_rate": 0.0001, + "loss": 67.2109, + "step": 11640 + }, + { + "epoch": 1.8812095032397407, + "grad_norm": 0.6107574701309204, + "learning_rate": 0.0001, + "loss": 67.029, + "step": 11650 + }, + { + "epoch": 1.8828243273248422, + "grad_norm": 0.6543708443641663, + "learning_rate": 0.0001, + "loss": 64.2133, + "step": 11660 + }, + { + "epoch": 1.8844391514099432, + "grad_norm": 0.5161535739898682, + "learning_rate": 0.0001, + "loss": 66.7091, + "step": 11670 + }, + { + "epoch": 1.8860539754950445, + "grad_norm": 0.6089748740196228, + "learning_rate": 0.0001, + "loss": 69.2073, + "step": 11680 + }, + { + "epoch": 1.8876687995801458, + "grad_norm": 0.6930866837501526, + "learning_rate": 0.0001, + "loss": 68.7227, + "step": 11690 + }, + { + "epoch": 1.8892836236652468, + "grad_norm": 0.6563196182250977, + "learning_rate": 0.0001, + "loss": 68.27, + "step": 11700 + }, + { + "epoch": 1.8908984477503483, + "grad_norm": 0.6173767447471619, + "learning_rate": 0.0001, + "loss": 68.5535, + "step": 11710 + }, + { + "epoch": 1.8925132718354494, + "grad_norm": 0.5498561263084412, + "learning_rate": 0.0001, + "loss": 67.7681, + "step": 11720 + }, + { + "epoch": 1.8941280959205506, + "grad_norm": 0.584891676902771, + "learning_rate": 0.0001, + "loss": 68.1709, + "step": 11730 + }, + { + "epoch": 1.895742920005652, + "grad_norm": 0.7491183280944824, + "learning_rate": 0.0001, + "loss": 65.7435, + "step": 11740 + }, + { + "epoch": 1.897357744090753, + "grad_norm": 0.6519315242767334, + "learning_rate": 0.0001, + "loss": 67.5436, + "step": 11750 + }, + { + "epoch": 1.8989725681758545, + "grad_norm": 0.5740950107574463, + "learning_rate": 0.0001, + "loss": 66.3092, + "step": 11760 + }, + { + "epoch": 1.9005873922609555, + "grad_norm": 0.7978914380073547, + "learning_rate": 0.0001, + "loss": 66.8598, + "step": 11770 + }, + { + "epoch": 1.9022022163460568, + "grad_norm": 0.5536732077598572, + "learning_rate": 0.0001, + "loss": 68.1609, + "step": 11780 + }, + { + "epoch": 1.903817040431158, + "grad_norm": 0.6358754634857178, + "learning_rate": 0.0001, + "loss": 68.2948, + "step": 11790 + }, + { + "epoch": 1.9054318645162591, + "grad_norm": 0.8400535583496094, + "learning_rate": 0.0001, + "loss": 67.7722, + "step": 11800 + }, + { + "epoch": 1.9070466886013606, + "grad_norm": 0.8256928324699402, + "learning_rate": 0.0001, + "loss": 65.8444, + "step": 11810 + }, + { + "epoch": 1.9086615126864617, + "grad_norm": 0.9160122871398926, + "learning_rate": 0.0001, + "loss": 67.4363, + "step": 11820 + }, + { + "epoch": 1.910276336771563, + "grad_norm": 0.6507587432861328, + "learning_rate": 0.0001, + "loss": 65.9515, + "step": 11830 + }, + { + "epoch": 1.9118911608566642, + "grad_norm": 0.5940394401550293, + "learning_rate": 0.0001, + "loss": 65.8083, + "step": 11840 + }, + { + "epoch": 1.9135059849417653, + "grad_norm": 0.7403995990753174, + "learning_rate": 0.0001, + "loss": 66.7136, + "step": 11850 + }, + { + "epoch": 1.9151208090268668, + "grad_norm": 0.6039106845855713, + "learning_rate": 0.0001, + "loss": 67.6913, + "step": 11860 + }, + { + "epoch": 1.9167356331119678, + "grad_norm": 0.9714781641960144, + "learning_rate": 0.0001, + "loss": 67.9658, + "step": 11870 + }, + { + "epoch": 1.918350457197069, + "grad_norm": 0.6880549192428589, + "learning_rate": 0.0001, + "loss": 68.4463, + "step": 11880 + }, + { + "epoch": 1.9199652812821704, + "grad_norm": 0.5718733668327332, + "learning_rate": 0.0001, + "loss": 66.9625, + "step": 11890 + }, + { + "epoch": 1.9215801053672714, + "grad_norm": 0.6208164691925049, + "learning_rate": 0.0001, + "loss": 65.9462, + "step": 11900 + }, + { + "epoch": 1.923194929452373, + "grad_norm": 0.5957098007202148, + "learning_rate": 0.0001, + "loss": 66.9398, + "step": 11910 + }, + { + "epoch": 1.924809753537474, + "grad_norm": 0.6295427680015564, + "learning_rate": 0.0001, + "loss": 66.2246, + "step": 11920 + }, + { + "epoch": 1.9264245776225752, + "grad_norm": 0.8903796672821045, + "learning_rate": 0.0001, + "loss": 66.2999, + "step": 11930 + }, + { + "epoch": 1.9280394017076765, + "grad_norm": 0.6410874128341675, + "learning_rate": 0.0001, + "loss": 67.6857, + "step": 11940 + }, + { + "epoch": 1.9296542257927776, + "grad_norm": 0.5800307393074036, + "learning_rate": 0.0001, + "loss": 66.0943, + "step": 11950 + }, + { + "epoch": 1.931269049877879, + "grad_norm": 0.6834051609039307, + "learning_rate": 0.0001, + "loss": 67.6365, + "step": 11960 + }, + { + "epoch": 1.9328838739629801, + "grad_norm": 0.6112056970596313, + "learning_rate": 0.0001, + "loss": 66.8131, + "step": 11970 + }, + { + "epoch": 1.9344986980480814, + "grad_norm": 0.6657601594924927, + "learning_rate": 0.0001, + "loss": 65.7584, + "step": 11980 + }, + { + "epoch": 1.9361135221331827, + "grad_norm": 0.7570573091506958, + "learning_rate": 0.0001, + "loss": 68.6683, + "step": 11990 + }, + { + "epoch": 1.9377283462182837, + "grad_norm": 0.6231463551521301, + "learning_rate": 0.0001, + "loss": 67.71, + "step": 12000 + }, + { + "epoch": 1.9393431703033852, + "grad_norm": 0.5517798662185669, + "learning_rate": 0.0001, + "loss": 65.5666, + "step": 12010 + }, + { + "epoch": 1.9409579943884863, + "grad_norm": 0.6614659428596497, + "learning_rate": 0.0001, + "loss": 65.4867, + "step": 12020 + }, + { + "epoch": 1.9425728184735875, + "grad_norm": 1.027906060218811, + "learning_rate": 0.0001, + "loss": 65.2863, + "step": 12030 + }, + { + "epoch": 1.9441876425586888, + "grad_norm": 0.5714905261993408, + "learning_rate": 0.0001, + "loss": 67.52, + "step": 12040 + }, + { + "epoch": 1.9458024666437899, + "grad_norm": 0.6580681204795837, + "learning_rate": 0.0001, + "loss": 68.3794, + "step": 12050 + }, + { + "epoch": 1.9474172907288914, + "grad_norm": 0.5728104710578918, + "learning_rate": 0.0001, + "loss": 65.6871, + "step": 12060 + }, + { + "epoch": 1.9490321148139924, + "grad_norm": 0.6828856468200684, + "learning_rate": 0.0001, + "loss": 66.6735, + "step": 12070 + }, + { + "epoch": 1.9506469388990937, + "grad_norm": 0.7448258996009827, + "learning_rate": 0.0001, + "loss": 67.4387, + "step": 12080 + }, + { + "epoch": 1.952261762984195, + "grad_norm": 0.5058867931365967, + "learning_rate": 0.0001, + "loss": 67.3999, + "step": 12090 + }, + { + "epoch": 1.953876587069296, + "grad_norm": 0.5884380340576172, + "learning_rate": 0.0001, + "loss": 66.6189, + "step": 12100 + }, + { + "epoch": 1.9554914111543975, + "grad_norm": 0.7010582685470581, + "learning_rate": 0.0001, + "loss": 66.4048, + "step": 12110 + }, + { + "epoch": 1.9571062352394986, + "grad_norm": 0.556446373462677, + "learning_rate": 0.0001, + "loss": 67.7398, + "step": 12120 + }, + { + "epoch": 1.9587210593245998, + "grad_norm": 0.7856518626213074, + "learning_rate": 0.0001, + "loss": 64.4189, + "step": 12130 + }, + { + "epoch": 1.960335883409701, + "grad_norm": 0.573811948299408, + "learning_rate": 0.0001, + "loss": 66.081, + "step": 12140 + }, + { + "epoch": 1.9619507074948022, + "grad_norm": 0.6113094687461853, + "learning_rate": 0.0001, + "loss": 65.6262, + "step": 12150 + }, + { + "epoch": 1.9635655315799037, + "grad_norm": 0.7158921360969543, + "learning_rate": 0.0001, + "loss": 66.646, + "step": 12160 + }, + { + "epoch": 1.9651803556650047, + "grad_norm": 0.6789208054542542, + "learning_rate": 0.0001, + "loss": 66.9576, + "step": 12170 + }, + { + "epoch": 1.966795179750106, + "grad_norm": 0.6550324559211731, + "learning_rate": 0.0001, + "loss": 67.2544, + "step": 12180 + }, + { + "epoch": 1.9684100038352073, + "grad_norm": 0.6580352187156677, + "learning_rate": 0.0001, + "loss": 67.6755, + "step": 12190 + }, + { + "epoch": 1.9700248279203083, + "grad_norm": 0.7927120327949524, + "learning_rate": 0.0001, + "loss": 66.9819, + "step": 12200 + }, + { + "epoch": 1.9716396520054098, + "grad_norm": 0.5637071132659912, + "learning_rate": 0.0001, + "loss": 67.9308, + "step": 12210 + }, + { + "epoch": 1.9732544760905109, + "grad_norm": 0.5903081297874451, + "learning_rate": 0.0001, + "loss": 67.0279, + "step": 12220 + }, + { + "epoch": 1.9748693001756121, + "grad_norm": 0.7173757553100586, + "learning_rate": 0.0001, + "loss": 66.1328, + "step": 12230 + }, + { + "epoch": 1.9764841242607134, + "grad_norm": 0.7677658796310425, + "learning_rate": 0.0001, + "loss": 66.8729, + "step": 12240 + }, + { + "epoch": 1.9780989483458145, + "grad_norm": 0.5733856558799744, + "learning_rate": 0.0001, + "loss": 65.7204, + "step": 12250 + }, + { + "epoch": 1.979713772430916, + "grad_norm": 0.575779378414154, + "learning_rate": 0.0001, + "loss": 65.082, + "step": 12260 + }, + { + "epoch": 1.981328596516017, + "grad_norm": 0.7983589768409729, + "learning_rate": 0.0001, + "loss": 66.7954, + "step": 12270 + }, + { + "epoch": 1.9829434206011183, + "grad_norm": 0.6657546758651733, + "learning_rate": 0.0001, + "loss": 66.1977, + "step": 12280 + }, + { + "epoch": 1.9845582446862196, + "grad_norm": 0.723057746887207, + "learning_rate": 0.0001, + "loss": 65.2823, + "step": 12290 + }, + { + "epoch": 1.9861730687713206, + "grad_norm": 0.7241561412811279, + "learning_rate": 0.0001, + "loss": 66.5179, + "step": 12300 + }, + { + "epoch": 1.987787892856422, + "grad_norm": 0.7907382249832153, + "learning_rate": 0.0001, + "loss": 67.1835, + "step": 12310 + }, + { + "epoch": 1.9894027169415232, + "grad_norm": 1.0993635654449463, + "learning_rate": 0.0001, + "loss": 66.2844, + "step": 12320 + }, + { + "epoch": 1.9910175410266244, + "grad_norm": 0.6327719688415527, + "learning_rate": 0.0001, + "loss": 64.3111, + "step": 12330 + }, + { + "epoch": 1.9926323651117257, + "grad_norm": 0.5955413579940796, + "learning_rate": 0.0001, + "loss": 67.3991, + "step": 12340 + }, + { + "epoch": 1.9942471891968268, + "grad_norm": 0.7130835056304932, + "learning_rate": 0.0001, + "loss": 67.2216, + "step": 12350 + }, + { + "epoch": 1.9958620132819282, + "grad_norm": 0.5579924583435059, + "learning_rate": 0.0001, + "loss": 65.8894, + "step": 12360 + }, + { + "epoch": 1.9974768373670293, + "grad_norm": 0.7395238876342773, + "learning_rate": 0.0001, + "loss": 66.5182, + "step": 12370 + }, + { + "epoch": 1.9990916614521306, + "grad_norm": 0.6070617437362671, + "learning_rate": 0.0001, + "loss": 67.0429, + "step": 12380 + }, + { + "epoch": 2.0006459296340404, + "grad_norm": 0.6658774018287659, + "learning_rate": 0.0001, + "loss": 65.1403, + "step": 12390 + }, + { + "epoch": 2.002260753719142, + "grad_norm": 0.7703005075454712, + "learning_rate": 0.0001, + "loss": 66.0792, + "step": 12400 + }, + { + "epoch": 2.003875577804243, + "grad_norm": 0.6887365579605103, + "learning_rate": 0.0001, + "loss": 67.5743, + "step": 12410 + }, + { + "epoch": 2.005490401889344, + "grad_norm": 0.6181573271751404, + "learning_rate": 0.0001, + "loss": 65.7725, + "step": 12420 + }, + { + "epoch": 2.0071052259744455, + "grad_norm": 0.6926079988479614, + "learning_rate": 0.0001, + "loss": 66.3818, + "step": 12430 + }, + { + "epoch": 2.0087200500595466, + "grad_norm": 0.9466767311096191, + "learning_rate": 0.0001, + "loss": 66.8674, + "step": 12440 + }, + { + "epoch": 2.010334874144648, + "grad_norm": 0.693327784538269, + "learning_rate": 0.0001, + "loss": 67.1573, + "step": 12450 + }, + { + "epoch": 2.011949698229749, + "grad_norm": 0.6354495286941528, + "learning_rate": 0.0001, + "loss": 67.6646, + "step": 12460 + }, + { + "epoch": 2.01356452231485, + "grad_norm": 0.6671343445777893, + "learning_rate": 0.0001, + "loss": 66.9442, + "step": 12470 + }, + { + "epoch": 2.0151793463999517, + "grad_norm": 0.5987095236778259, + "learning_rate": 0.0001, + "loss": 68.265, + "step": 12480 + }, + { + "epoch": 2.0167941704850527, + "grad_norm": 0.8267726302146912, + "learning_rate": 0.0001, + "loss": 67.7709, + "step": 12490 + }, + { + "epoch": 2.018408994570154, + "grad_norm": 0.5237421989440918, + "learning_rate": 0.0001, + "loss": 64.8426, + "step": 12500 + }, + { + "epoch": 2.0200238186552553, + "grad_norm": 0.5602227449417114, + "learning_rate": 0.0001, + "loss": 65.4287, + "step": 12510 + }, + { + "epoch": 2.0216386427403563, + "grad_norm": 0.6129289269447327, + "learning_rate": 0.0001, + "loss": 65.9583, + "step": 12520 + }, + { + "epoch": 2.023253466825458, + "grad_norm": 0.6047223210334778, + "learning_rate": 0.0001, + "loss": 66.9257, + "step": 12530 + }, + { + "epoch": 2.024868290910559, + "grad_norm": 0.6117433905601501, + "learning_rate": 0.0001, + "loss": 63.0344, + "step": 12540 + }, + { + "epoch": 2.0264831149956604, + "grad_norm": 0.6688846945762634, + "learning_rate": 0.0001, + "loss": 66.2295, + "step": 12550 + }, + { + "epoch": 2.0280979390807614, + "grad_norm": 0.779574453830719, + "learning_rate": 0.0001, + "loss": 67.066, + "step": 12560 + }, + { + "epoch": 2.0297127631658625, + "grad_norm": 0.624839186668396, + "learning_rate": 0.0001, + "loss": 66.7905, + "step": 12570 + }, + { + "epoch": 2.031327587250964, + "grad_norm": 0.5970683693885803, + "learning_rate": 0.0001, + "loss": 67.6997, + "step": 12580 + }, + { + "epoch": 2.032942411336065, + "grad_norm": 0.6005929708480835, + "learning_rate": 0.0001, + "loss": 66.717, + "step": 12590 + }, + { + "epoch": 2.0345572354211665, + "grad_norm": 0.6358321905136108, + "learning_rate": 0.0001, + "loss": 67.8494, + "step": 12600 + }, + { + "epoch": 2.0361720595062676, + "grad_norm": 0.8642273545265198, + "learning_rate": 0.0001, + "loss": 67.1791, + "step": 12610 + }, + { + "epoch": 2.0377868835913686, + "grad_norm": 0.587282121181488, + "learning_rate": 0.0001, + "loss": 66.8169, + "step": 12620 + }, + { + "epoch": 2.03940170767647, + "grad_norm": 0.5977926850318909, + "learning_rate": 0.0001, + "loss": 66.8393, + "step": 12630 + }, + { + "epoch": 2.041016531761571, + "grad_norm": 0.5654817223548889, + "learning_rate": 0.0001, + "loss": 68.8243, + "step": 12640 + }, + { + "epoch": 2.0426313558466727, + "grad_norm": 0.5778902173042297, + "learning_rate": 0.0001, + "loss": 66.8893, + "step": 12650 + }, + { + "epoch": 2.0442461799317737, + "grad_norm": 0.6533665657043457, + "learning_rate": 0.0001, + "loss": 66.6174, + "step": 12660 + }, + { + "epoch": 2.0458610040168748, + "grad_norm": 0.5945654511451721, + "learning_rate": 0.0001, + "loss": 67.8193, + "step": 12670 + }, + { + "epoch": 2.0474758281019763, + "grad_norm": 0.5898523926734924, + "learning_rate": 0.0001, + "loss": 65.6194, + "step": 12680 + }, + { + "epoch": 2.0490906521870773, + "grad_norm": 0.5497958660125732, + "learning_rate": 0.0001, + "loss": 66.4571, + "step": 12690 + }, + { + "epoch": 2.050705476272179, + "grad_norm": 0.6195752024650574, + "learning_rate": 0.0001, + "loss": 66.0555, + "step": 12700 + }, + { + "epoch": 2.05232030035728, + "grad_norm": 0.6400334239006042, + "learning_rate": 0.0001, + "loss": 66.7153, + "step": 12710 + }, + { + "epoch": 2.053935124442381, + "grad_norm": 0.7953216433525085, + "learning_rate": 0.0001, + "loss": 66.5822, + "step": 12720 + }, + { + "epoch": 2.0555499485274824, + "grad_norm": 0.8052420616149902, + "learning_rate": 0.0001, + "loss": 66.4395, + "step": 12730 + }, + { + "epoch": 2.0571647726125835, + "grad_norm": 0.567957878112793, + "learning_rate": 0.0001, + "loss": 65.7762, + "step": 12740 + }, + { + "epoch": 2.058779596697685, + "grad_norm": 0.6517317891120911, + "learning_rate": 0.0001, + "loss": 66.7526, + "step": 12750 + }, + { + "epoch": 2.060394420782786, + "grad_norm": 0.6032442450523376, + "learning_rate": 0.0001, + "loss": 64.2725, + "step": 12760 + }, + { + "epoch": 2.062009244867887, + "grad_norm": 0.6162487268447876, + "learning_rate": 0.0001, + "loss": 65.4797, + "step": 12770 + }, + { + "epoch": 2.0636240689529886, + "grad_norm": 0.7512190937995911, + "learning_rate": 0.0001, + "loss": 65.9602, + "step": 12780 + }, + { + "epoch": 2.0652388930380896, + "grad_norm": 0.7601141929626465, + "learning_rate": 0.0001, + "loss": 66.0806, + "step": 12790 + }, + { + "epoch": 2.066853717123191, + "grad_norm": 0.5737994909286499, + "learning_rate": 0.0001, + "loss": 66.8124, + "step": 12800 + }, + { + "epoch": 2.068468541208292, + "grad_norm": 0.6629387736320496, + "learning_rate": 0.0001, + "loss": 65.8944, + "step": 12810 + }, + { + "epoch": 2.070083365293393, + "grad_norm": 0.6374203562736511, + "learning_rate": 0.0001, + "loss": 67.1589, + "step": 12820 + }, + { + "epoch": 2.0716981893784947, + "grad_norm": 0.6732942461967468, + "learning_rate": 0.0001, + "loss": 67.1901, + "step": 12830 + }, + { + "epoch": 2.0733130134635958, + "grad_norm": 0.6219559907913208, + "learning_rate": 0.0001, + "loss": 66.613, + "step": 12840 + }, + { + "epoch": 2.0749278375486973, + "grad_norm": 0.591238796710968, + "learning_rate": 0.0001, + "loss": 67.6111, + "step": 12850 + }, + { + "epoch": 2.0765426616337983, + "grad_norm": 0.6472163200378418, + "learning_rate": 0.0001, + "loss": 66.305, + "step": 12860 + }, + { + "epoch": 2.0781574857188994, + "grad_norm": 0.6349777579307556, + "learning_rate": 0.0001, + "loss": 65.2865, + "step": 12870 + }, + { + "epoch": 2.079772309804001, + "grad_norm": 0.9257296323776245, + "learning_rate": 0.0001, + "loss": 67.6326, + "step": 12880 + }, + { + "epoch": 2.081387133889102, + "grad_norm": 0.5744728446006775, + "learning_rate": 0.0001, + "loss": 65.7023, + "step": 12890 + }, + { + "epoch": 2.0830019579742034, + "grad_norm": 0.6725465059280396, + "learning_rate": 0.0001, + "loss": 65.1233, + "step": 12900 + }, + { + "epoch": 2.0846167820593045, + "grad_norm": 0.6141199469566345, + "learning_rate": 0.0001, + "loss": 65.5593, + "step": 12910 + }, + { + "epoch": 2.0862316061444055, + "grad_norm": 0.6364370584487915, + "learning_rate": 0.0001, + "loss": 65.3899, + "step": 12920 + }, + { + "epoch": 2.087846430229507, + "grad_norm": 0.5948928594589233, + "learning_rate": 0.0001, + "loss": 63.5976, + "step": 12930 + }, + { + "epoch": 2.089461254314608, + "grad_norm": 0.630434513092041, + "learning_rate": 0.0001, + "loss": 66.764, + "step": 12940 + }, + { + "epoch": 2.091076078399709, + "grad_norm": 0.6178410649299622, + "learning_rate": 0.0001, + "loss": 66.3948, + "step": 12950 + }, + { + "epoch": 2.0926909024848106, + "grad_norm": 0.7695903778076172, + "learning_rate": 0.0001, + "loss": 68.2218, + "step": 12960 + }, + { + "epoch": 2.0943057265699117, + "grad_norm": 0.6712848544120789, + "learning_rate": 0.0001, + "loss": 66.8801, + "step": 12970 + }, + { + "epoch": 2.095920550655013, + "grad_norm": 0.7133443355560303, + "learning_rate": 0.0001, + "loss": 65.343, + "step": 12980 + }, + { + "epoch": 2.097535374740114, + "grad_norm": 0.6836514472961426, + "learning_rate": 0.0001, + "loss": 64.4993, + "step": 12990 + }, + { + "epoch": 2.0991501988252157, + "grad_norm": 0.6734977960586548, + "learning_rate": 0.0001, + "loss": 67.6078, + "step": 13000 + }, + { + "epoch": 2.1007650229103167, + "grad_norm": 0.590630054473877, + "learning_rate": 0.0001, + "loss": 65.4567, + "step": 13010 + }, + { + "epoch": 2.102379846995418, + "grad_norm": 0.7174671292304993, + "learning_rate": 0.0001, + "loss": 66.2743, + "step": 13020 + }, + { + "epoch": 2.1039946710805193, + "grad_norm": 0.5266520977020264, + "learning_rate": 0.0001, + "loss": 65.0063, + "step": 13030 + }, + { + "epoch": 2.1056094951656203, + "grad_norm": 0.550865113735199, + "learning_rate": 0.0001, + "loss": 67.0641, + "step": 13040 + }, + { + "epoch": 2.1072243192507214, + "grad_norm": 0.7976025938987732, + "learning_rate": 0.0001, + "loss": 67.0135, + "step": 13050 + }, + { + "epoch": 2.108839143335823, + "grad_norm": 0.7592310905456543, + "learning_rate": 0.0001, + "loss": 65.9875, + "step": 13060 + }, + { + "epoch": 2.110453967420924, + "grad_norm": 0.7613953948020935, + "learning_rate": 0.0001, + "loss": 66.0958, + "step": 13070 + }, + { + "epoch": 2.1120687915060254, + "grad_norm": 0.6951309442520142, + "learning_rate": 0.0001, + "loss": 65.949, + "step": 13080 + }, + { + "epoch": 2.1136836155911265, + "grad_norm": 0.6439896821975708, + "learning_rate": 0.0001, + "loss": 66.4056, + "step": 13090 + }, + { + "epoch": 2.115298439676228, + "grad_norm": 0.6922518610954285, + "learning_rate": 0.0001, + "loss": 67.537, + "step": 13100 + }, + { + "epoch": 2.116913263761329, + "grad_norm": 0.5330061316490173, + "learning_rate": 0.0001, + "loss": 67.0438, + "step": 13110 + }, + { + "epoch": 2.11852808784643, + "grad_norm": 0.6507022380828857, + "learning_rate": 0.0001, + "loss": 66.9908, + "step": 13120 + }, + { + "epoch": 2.1201429119315316, + "grad_norm": 0.994196891784668, + "learning_rate": 0.0001, + "loss": 63.9818, + "step": 13130 + }, + { + "epoch": 2.1217577360166326, + "grad_norm": 0.7431663870811462, + "learning_rate": 0.0001, + "loss": 65.8259, + "step": 13140 + }, + { + "epoch": 2.1233725601017337, + "grad_norm": 0.5852991938591003, + "learning_rate": 0.0001, + "loss": 65.918, + "step": 13150 + }, + { + "epoch": 2.124987384186835, + "grad_norm": 0.551194429397583, + "learning_rate": 0.0001, + "loss": 64.9907, + "step": 13160 + }, + { + "epoch": 2.1266022082719362, + "grad_norm": 0.6882205605506897, + "learning_rate": 0.0001, + "loss": 66.7266, + "step": 13170 + }, + { + "epoch": 2.1282170323570377, + "grad_norm": 0.8906047940254211, + "learning_rate": 0.0001, + "loss": 65.3441, + "step": 13180 + }, + { + "epoch": 2.129831856442139, + "grad_norm": 0.5340781211853027, + "learning_rate": 0.0001, + "loss": 65.6772, + "step": 13190 + }, + { + "epoch": 2.1314466805272403, + "grad_norm": 0.7657933831214905, + "learning_rate": 0.0001, + "loss": 65.3888, + "step": 13200 + }, + { + "epoch": 2.1330615046123413, + "grad_norm": 1.2086169719696045, + "learning_rate": 0.0001, + "loss": 65.7214, + "step": 13210 + }, + { + "epoch": 2.1346763286974424, + "grad_norm": 0.6794751286506653, + "learning_rate": 0.0001, + "loss": 66.6761, + "step": 13220 + }, + { + "epoch": 2.136291152782544, + "grad_norm": 0.5939100384712219, + "learning_rate": 0.0001, + "loss": 64.1858, + "step": 13230 + }, + { + "epoch": 2.137905976867645, + "grad_norm": 0.581163763999939, + "learning_rate": 0.0001, + "loss": 65.9456, + "step": 13240 + }, + { + "epoch": 2.139520800952746, + "grad_norm": 0.676103949546814, + "learning_rate": 0.0001, + "loss": 65.1919, + "step": 13250 + }, + { + "epoch": 2.1411356250378475, + "grad_norm": 0.6052015423774719, + "learning_rate": 0.0001, + "loss": 67.9872, + "step": 13260 + }, + { + "epoch": 2.1427504491229485, + "grad_norm": 0.5938939452171326, + "learning_rate": 0.0001, + "loss": 64.7395, + "step": 13270 + }, + { + "epoch": 2.14436527320805, + "grad_norm": 0.563115656375885, + "learning_rate": 0.0001, + "loss": 63.9757, + "step": 13280 + }, + { + "epoch": 2.145980097293151, + "grad_norm": 0.5397975444793701, + "learning_rate": 0.0001, + "loss": 65.6596, + "step": 13290 + }, + { + "epoch": 2.1475949213782526, + "grad_norm": 0.6397805213928223, + "learning_rate": 0.0001, + "loss": 67.0618, + "step": 13300 + }, + { + "epoch": 2.1492097454633536, + "grad_norm": 0.678053081035614, + "learning_rate": 0.0001, + "loss": 65.8194, + "step": 13310 + }, + { + "epoch": 2.1508245695484547, + "grad_norm": 0.6067008972167969, + "learning_rate": 0.0001, + "loss": 65.9035, + "step": 13320 + }, + { + "epoch": 2.152439393633556, + "grad_norm": 0.6560337543487549, + "learning_rate": 0.0001, + "loss": 64.9907, + "step": 13330 + }, + { + "epoch": 2.1540542177186572, + "grad_norm": 0.5570183396339417, + "learning_rate": 0.0001, + "loss": 64.5903, + "step": 13340 + }, + { + "epoch": 2.1556690418037583, + "grad_norm": 0.6721981167793274, + "learning_rate": 0.0001, + "loss": 66.5618, + "step": 13350 + }, + { + "epoch": 2.15728386588886, + "grad_norm": 0.7251304388046265, + "learning_rate": 0.0001, + "loss": 66.724, + "step": 13360 + }, + { + "epoch": 2.158898689973961, + "grad_norm": 0.5928503274917603, + "learning_rate": 0.0001, + "loss": 66.6991, + "step": 13370 + }, + { + "epoch": 2.1605135140590623, + "grad_norm": 0.7684288620948792, + "learning_rate": 0.0001, + "loss": 66.4064, + "step": 13380 + }, + { + "epoch": 2.1621283381441634, + "grad_norm": 0.5499702095985413, + "learning_rate": 0.0001, + "loss": 66.7671, + "step": 13390 + }, + { + "epoch": 2.163743162229265, + "grad_norm": 0.5267577767372131, + "learning_rate": 0.0001, + "loss": 65.8112, + "step": 13400 + }, + { + "epoch": 2.165357986314366, + "grad_norm": 0.6214171648025513, + "learning_rate": 0.0001, + "loss": 65.851, + "step": 13410 + }, + { + "epoch": 2.166972810399467, + "grad_norm": 0.5952336192131042, + "learning_rate": 0.0001, + "loss": 65.4166, + "step": 13420 + }, + { + "epoch": 2.1685876344845685, + "grad_norm": 1.3454428911209106, + "learning_rate": 0.0001, + "loss": 64.7546, + "step": 13430 + }, + { + "epoch": 2.1702024585696695, + "grad_norm": 0.6586198210716248, + "learning_rate": 0.0001, + "loss": 65.6818, + "step": 13440 + }, + { + "epoch": 2.1718172826547706, + "grad_norm": 0.8028526306152344, + "learning_rate": 0.0001, + "loss": 67.2677, + "step": 13450 + }, + { + "epoch": 2.173432106739872, + "grad_norm": 0.5949039459228516, + "learning_rate": 0.0001, + "loss": 66.994, + "step": 13460 + }, + { + "epoch": 2.175046930824973, + "grad_norm": 0.6666626930236816, + "learning_rate": 0.0001, + "loss": 66.6509, + "step": 13470 + }, + { + "epoch": 2.1766617549100746, + "grad_norm": 0.5985739827156067, + "learning_rate": 0.0001, + "loss": 65.9211, + "step": 13480 + }, + { + "epoch": 2.1782765789951757, + "grad_norm": 0.6771738529205322, + "learning_rate": 0.0001, + "loss": 65.3477, + "step": 13490 + }, + { + "epoch": 2.1798914030802767, + "grad_norm": 0.6147419214248657, + "learning_rate": 0.0001, + "loss": 64.4142, + "step": 13500 + }, + { + "epoch": 2.1815062271653782, + "grad_norm": 0.6774442791938782, + "learning_rate": 0.0001, + "loss": 64.5831, + "step": 13510 + }, + { + "epoch": 2.1831210512504793, + "grad_norm": 0.5778859853744507, + "learning_rate": 0.0001, + "loss": 64.4449, + "step": 13520 + }, + { + "epoch": 2.184735875335581, + "grad_norm": 0.5697338581085205, + "learning_rate": 0.0001, + "loss": 67.1627, + "step": 13530 + }, + { + "epoch": 2.186350699420682, + "grad_norm": 1.2444179058074951, + "learning_rate": 0.0001, + "loss": 64.7959, + "step": 13540 + }, + { + "epoch": 2.187965523505783, + "grad_norm": 0.6489098072052002, + "learning_rate": 0.0001, + "loss": 65.9926, + "step": 13550 + }, + { + "epoch": 2.1895803475908844, + "grad_norm": 0.6871664524078369, + "learning_rate": 0.0001, + "loss": 67.681, + "step": 13560 + }, + { + "epoch": 2.1911951716759854, + "grad_norm": 0.5581620931625366, + "learning_rate": 0.0001, + "loss": 65.6707, + "step": 13570 + }, + { + "epoch": 2.192809995761087, + "grad_norm": 0.522402286529541, + "learning_rate": 0.0001, + "loss": 65.9014, + "step": 13580 + }, + { + "epoch": 2.194424819846188, + "grad_norm": 0.6149793267250061, + "learning_rate": 0.0001, + "loss": 64.6348, + "step": 13590 + }, + { + "epoch": 2.196039643931289, + "grad_norm": 0.5616180896759033, + "learning_rate": 0.0001, + "loss": 66.3635, + "step": 13600 + }, + { + "epoch": 2.1976544680163905, + "grad_norm": 1.024154543876648, + "learning_rate": 0.0001, + "loss": 66.7156, + "step": 13610 + }, + { + "epoch": 2.1992692921014916, + "grad_norm": 1.035205602645874, + "learning_rate": 0.0001, + "loss": 65.996, + "step": 13620 + }, + { + "epoch": 2.200884116186593, + "grad_norm": 0.5354496240615845, + "learning_rate": 0.0001, + "loss": 66.6107, + "step": 13630 + }, + { + "epoch": 2.202498940271694, + "grad_norm": 0.5648348331451416, + "learning_rate": 0.0001, + "loss": 66.473, + "step": 13640 + }, + { + "epoch": 2.204113764356795, + "grad_norm": 0.6247662901878357, + "learning_rate": 0.0001, + "loss": 66.576, + "step": 13650 + }, + { + "epoch": 2.2057285884418967, + "grad_norm": 0.5871408581733704, + "learning_rate": 0.0001, + "loss": 66.6254, + "step": 13660 + }, + { + "epoch": 2.2073434125269977, + "grad_norm": 0.6394441723823547, + "learning_rate": 0.0001, + "loss": 66.2924, + "step": 13670 + }, + { + "epoch": 2.2089582366120992, + "grad_norm": 0.6017318367958069, + "learning_rate": 0.0001, + "loss": 66.6313, + "step": 13680 + }, + { + "epoch": 2.2105730606972003, + "grad_norm": 0.5696585774421692, + "learning_rate": 0.0001, + "loss": 65.9198, + "step": 13690 + }, + { + "epoch": 2.2121878847823013, + "grad_norm": 0.5966867804527283, + "learning_rate": 0.0001, + "loss": 67.507, + "step": 13700 + }, + { + "epoch": 2.213802708867403, + "grad_norm": 0.6163555979728699, + "learning_rate": 0.0001, + "loss": 67.2444, + "step": 13710 + }, + { + "epoch": 2.215417532952504, + "grad_norm": 0.5670737624168396, + "learning_rate": 0.0001, + "loss": 66.5747, + "step": 13720 + }, + { + "epoch": 2.2170323570376054, + "grad_norm": 0.6289104223251343, + "learning_rate": 0.0001, + "loss": 65.6289, + "step": 13730 + }, + { + "epoch": 2.2186471811227064, + "grad_norm": 0.5649837255477905, + "learning_rate": 0.0001, + "loss": 66.0953, + "step": 13740 + }, + { + "epoch": 2.2202620052078075, + "grad_norm": 0.6066703200340271, + "learning_rate": 0.0001, + "loss": 65.9425, + "step": 13750 + }, + { + "epoch": 2.221876829292909, + "grad_norm": 0.6202117204666138, + "learning_rate": 0.0001, + "loss": 65.3268, + "step": 13760 + }, + { + "epoch": 2.22349165337801, + "grad_norm": 0.6883425116539001, + "learning_rate": 0.0001, + "loss": 64.1026, + "step": 13770 + }, + { + "epoch": 2.2251064774631115, + "grad_norm": 0.5564731955528259, + "learning_rate": 0.0001, + "loss": 65.1203, + "step": 13780 + }, + { + "epoch": 2.2267213015482126, + "grad_norm": 0.48229852318763733, + "learning_rate": 0.0001, + "loss": 65.2423, + "step": 13790 + }, + { + "epoch": 2.2283361256333136, + "grad_norm": 0.549240231513977, + "learning_rate": 0.0001, + "loss": 68.3811, + "step": 13800 + }, + { + "epoch": 2.229950949718415, + "grad_norm": 0.7217127084732056, + "learning_rate": 0.0001, + "loss": 64.9966, + "step": 13810 + }, + { + "epoch": 2.231565773803516, + "grad_norm": 0.8491377830505371, + "learning_rate": 0.0001, + "loss": 65.6468, + "step": 13820 + }, + { + "epoch": 2.2331805978886177, + "grad_norm": 0.5460603833198547, + "learning_rate": 0.0001, + "loss": 65.192, + "step": 13830 + }, + { + "epoch": 2.2347954219737187, + "grad_norm": 0.5784180164337158, + "learning_rate": 0.0001, + "loss": 67.3488, + "step": 13840 + }, + { + "epoch": 2.2364102460588198, + "grad_norm": 0.7575398087501526, + "learning_rate": 0.0001, + "loss": 65.6969, + "step": 13850 + }, + { + "epoch": 2.2380250701439213, + "grad_norm": 0.5111517310142517, + "learning_rate": 0.0001, + "loss": 66.482, + "step": 13860 + }, + { + "epoch": 2.2396398942290223, + "grad_norm": 0.6383832693099976, + "learning_rate": 0.0001, + "loss": 67.4894, + "step": 13870 + }, + { + "epoch": 2.241254718314124, + "grad_norm": 0.5922625064849854, + "learning_rate": 0.0001, + "loss": 66.1718, + "step": 13880 + }, + { + "epoch": 2.242869542399225, + "grad_norm": 5.395338535308838, + "learning_rate": 0.0001, + "loss": 64.7385, + "step": 13890 + }, + { + "epoch": 2.244484366484326, + "grad_norm": 1.4203481674194336, + "learning_rate": 0.0001, + "loss": 64.7676, + "step": 13900 + }, + { + "epoch": 2.2460991905694274, + "grad_norm": 0.6692256331443787, + "learning_rate": 0.0001, + "loss": 65.856, + "step": 13910 + }, + { + "epoch": 2.2477140146545285, + "grad_norm": 0.5868324041366577, + "learning_rate": 0.0001, + "loss": 66.1823, + "step": 13920 + }, + { + "epoch": 2.24932883873963, + "grad_norm": 0.6828433275222778, + "learning_rate": 0.0001, + "loss": 65.7601, + "step": 13930 + }, + { + "epoch": 2.250943662824731, + "grad_norm": 0.5108211040496826, + "learning_rate": 0.0001, + "loss": 63.413, + "step": 13940 + }, + { + "epoch": 2.252558486909832, + "grad_norm": 0.5538185238838196, + "learning_rate": 0.0001, + "loss": 64.9088, + "step": 13950 + }, + { + "epoch": 2.2541733109949336, + "grad_norm": 0.7211924195289612, + "learning_rate": 0.0001, + "loss": 66.0293, + "step": 13960 + }, + { + "epoch": 2.2557881350800346, + "grad_norm": 0.5632925033569336, + "learning_rate": 0.0001, + "loss": 64.8261, + "step": 13970 + }, + { + "epoch": 2.257402959165136, + "grad_norm": 0.574968159198761, + "learning_rate": 0.0001, + "loss": 65.2667, + "step": 13980 + }, + { + "epoch": 2.259017783250237, + "grad_norm": 0.5919519066810608, + "learning_rate": 0.0001, + "loss": 66.2643, + "step": 13990 + }, + { + "epoch": 2.260632607335338, + "grad_norm": 0.6732460260391235, + "learning_rate": 0.0001, + "loss": 64.7524, + "step": 14000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.7142098270936367e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6a151f546555c78ebc502dd7b703b665f0672d6f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35235df8408202e91954a585851d27187bafaf4e1d75c4e63c51fbf721136ff5 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-14000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/latest b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/latest new file mode 100644 index 0000000000000000000000000000000000000000..76800d601529826c2ec8d7453b614b895cf74f2a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/latest @@ -0,0 +1 @@ +global_step16000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..69d2986a602a05d28c98e9b9d39fee53bd59bde0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c71e5aa6b9e26ebdb59422dde106e5946a658c517f796d3e9e13478d6fb3a709 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..cd0b491b36b49257c5e8d36bf8c2cf87d4987b50 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21c885cdbbdff10493fee650243d288a03c2e3144959398bc20778546b15adfb +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..0f67cc92859c1043b9a92e2e7a11d821c7fc1baa --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f16aa5a51d7b5d73253c03d58a684e9dac6c4c808345d15ca739047a2fe73a0a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..8d7a5f5587dca11ba3c70d617f0f2c3e4d077911 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1471ec565666778fe168edd6b017da10c78ae94936ca9c6519378f1a375f7c46 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..570bbed91c52e0d6b9f86172c6b86b5551d39e59 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:537bd14cae990af886f3ba2e151b1a6fc5f300d36d5a587693f186e626eec00f +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..ba9173e353a522c3272e4f182f811fc79bfddeda --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71746b158a6b96668fdb685cf9b17de306686a1357fedc07a6cecb21f3a60f96 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..16a041e2391cb1b1ea1acb008cbcbc9c9e57a8a6 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f1329f013e9147aeb65996115f22f6bfe798f6541f185dd37f9e605a81e8d1d +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..5d5476bad87153b658c156710c6930733496d832 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e3c7a2c88fd745483dfd95c97c02c24a473a2285d67958763d28cef66f89604 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..e105875a050fe5e3e01c4ecad2e5d383acc2018d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17b80c7cc1ac55a54368abb28e974605d695767f522f1e619f0b9fe10ac9406a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..40b93c36797b53b455f19ea4c1bc67022918d62f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a427e77702ae9121e994cdda9f4f553997fc61824162d506f30f54b748c208a5 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..11996174af092b85ef481eb018552569dd6a64a3 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/trainer_state.json @@ -0,0 +1,11234 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.583597424355584, + "eval_steps": 2000, + "global_step": 16000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0016148240851012292, + "grad_norm": 43.90101623535156, + "learning_rate": 0.0001, + "loss": 307.2084, + "step": 10 + }, + { + "epoch": 0.0032296481702024584, + "grad_norm": 19.106313705444336, + "learning_rate": 0.0001, + "loss": 320.2794, + "step": 20 + }, + { + "epoch": 0.004844472255303688, + "grad_norm": 5.771847248077393, + "learning_rate": 0.0001, + "loss": 247.4977, + "step": 30 + }, + { + "epoch": 0.006459296340404917, + "grad_norm": 2.630659580230713, + "learning_rate": 0.0001, + "loss": 229.9593, + "step": 40 + }, + { + "epoch": 0.008074120425506147, + "grad_norm": 2.487499475479126, + "learning_rate": 0.0001, + "loss": 228.65, + "step": 50 + }, + { + "epoch": 0.009688944510607376, + "grad_norm": 2.078263521194458, + "learning_rate": 0.0001, + "loss": 227.648, + "step": 60 + }, + { + "epoch": 0.011303768595708604, + "grad_norm": 3.105417490005493, + "learning_rate": 0.0001, + "loss": 227.9345, + "step": 70 + }, + { + "epoch": 0.012918592680809833, + "grad_norm": 2.2145848274230957, + "learning_rate": 0.0001, + "loss": 227.3662, + "step": 80 + }, + { + "epoch": 0.014533416765911064, + "grad_norm": 1.9644615650177002, + "learning_rate": 0.0001, + "loss": 226.9145, + "step": 90 + }, + { + "epoch": 0.016148240851012293, + "grad_norm": 1.867382287979126, + "learning_rate": 0.0001, + "loss": 227.5703, + "step": 100 + }, + { + "epoch": 0.017763064936113522, + "grad_norm": 1.4613710641860962, + "learning_rate": 0.0001, + "loss": 227.1107, + "step": 110 + }, + { + "epoch": 0.01937788902121475, + "grad_norm": 2.8953869342803955, + "learning_rate": 0.0001, + "loss": 226.8185, + "step": 120 + }, + { + "epoch": 0.02099271310631598, + "grad_norm": 1.8446482419967651, + "learning_rate": 0.0001, + "loss": 226.9624, + "step": 130 + }, + { + "epoch": 0.02260753719141721, + "grad_norm": 1.3048362731933594, + "learning_rate": 0.0001, + "loss": 226.8404, + "step": 140 + }, + { + "epoch": 0.024222361276518438, + "grad_norm": 1.7100074291229248, + "learning_rate": 0.0001, + "loss": 226.1071, + "step": 150 + }, + { + "epoch": 0.025837185361619667, + "grad_norm": 1.7875330448150635, + "learning_rate": 0.0001, + "loss": 226.0764, + "step": 160 + }, + { + "epoch": 0.0274520094467209, + "grad_norm": 1.7503769397735596, + "learning_rate": 0.0001, + "loss": 226.0772, + "step": 170 + }, + { + "epoch": 0.029066833531822128, + "grad_norm": 2.7083983421325684, + "learning_rate": 0.0001, + "loss": 224.7667, + "step": 180 + }, + { + "epoch": 0.030681657616923357, + "grad_norm": 1.8192371129989624, + "learning_rate": 0.0001, + "loss": 223.6494, + "step": 190 + }, + { + "epoch": 0.032296481702024586, + "grad_norm": 3.210358142852783, + "learning_rate": 0.0001, + "loss": 223.3545, + "step": 200 + }, + { + "epoch": 0.03391130578712581, + "grad_norm": 5.819669246673584, + "learning_rate": 0.0001, + "loss": 222.1732, + "step": 210 + }, + { + "epoch": 0.035526129872227044, + "grad_norm": 4.185793399810791, + "learning_rate": 0.0001, + "loss": 221.6866, + "step": 220 + }, + { + "epoch": 0.03714095395732828, + "grad_norm": 2.7514216899871826, + "learning_rate": 0.0001, + "loss": 221.0769, + "step": 230 + }, + { + "epoch": 0.0387557780424295, + "grad_norm": 1.5478988885879517, + "learning_rate": 0.0001, + "loss": 219.7232, + "step": 240 + }, + { + "epoch": 0.040370602127530734, + "grad_norm": 1.4958223104476929, + "learning_rate": 0.0001, + "loss": 218.5454, + "step": 250 + }, + { + "epoch": 0.04198542621263196, + "grad_norm": 6.36635684967041, + "learning_rate": 0.0001, + "loss": 218.5061, + "step": 260 + }, + { + "epoch": 0.04360025029773319, + "grad_norm": 3.029601812362671, + "learning_rate": 0.0001, + "loss": 216.3466, + "step": 270 + }, + { + "epoch": 0.04521507438283442, + "grad_norm": 2.6436100006103516, + "learning_rate": 0.0001, + "loss": 214.0003, + "step": 280 + }, + { + "epoch": 0.04682989846793565, + "grad_norm": 5.329655647277832, + "learning_rate": 0.0001, + "loss": 212.9169, + "step": 290 + }, + { + "epoch": 0.048444722553036876, + "grad_norm": 4.930875301361084, + "learning_rate": 0.0001, + "loss": 209.7737, + "step": 300 + }, + { + "epoch": 0.05005954663813811, + "grad_norm": 18.902585983276367, + "learning_rate": 0.0001, + "loss": 209.4267, + "step": 310 + }, + { + "epoch": 0.051674370723239334, + "grad_norm": 5.364975929260254, + "learning_rate": 0.0001, + "loss": 206.49, + "step": 320 + }, + { + "epoch": 0.053289194808340566, + "grad_norm": 3.1421146392822266, + "learning_rate": 0.0001, + "loss": 204.8132, + "step": 330 + }, + { + "epoch": 0.0549040188934418, + "grad_norm": 3.399320602416992, + "learning_rate": 0.0001, + "loss": 202.4785, + "step": 340 + }, + { + "epoch": 0.056518842978543024, + "grad_norm": 5.137377738952637, + "learning_rate": 0.0001, + "loss": 199.5143, + "step": 350 + }, + { + "epoch": 0.058133667063644257, + "grad_norm": 5.4967522621154785, + "learning_rate": 0.0001, + "loss": 196.2809, + "step": 360 + }, + { + "epoch": 0.05974849114874548, + "grad_norm": 6.435070037841797, + "learning_rate": 0.0001, + "loss": 196.8831, + "step": 370 + }, + { + "epoch": 0.061363315233846714, + "grad_norm": 5.7334370613098145, + "learning_rate": 0.0001, + "loss": 193.141, + "step": 380 + }, + { + "epoch": 0.06297813931894794, + "grad_norm": 3.6522979736328125, + "learning_rate": 0.0001, + "loss": 191.5819, + "step": 390 + }, + { + "epoch": 0.06459296340404917, + "grad_norm": 2.614051103591919, + "learning_rate": 0.0001, + "loss": 190.046, + "step": 400 + }, + { + "epoch": 0.0662077874891504, + "grad_norm": 4.845924377441406, + "learning_rate": 0.0001, + "loss": 188.7121, + "step": 410 + }, + { + "epoch": 0.06782261157425162, + "grad_norm": 4.319681644439697, + "learning_rate": 0.0001, + "loss": 185.3697, + "step": 420 + }, + { + "epoch": 0.06943743565935286, + "grad_norm": 2.7018792629241943, + "learning_rate": 0.0001, + "loss": 183.5171, + "step": 430 + }, + { + "epoch": 0.07105225974445409, + "grad_norm": 2.6407113075256348, + "learning_rate": 0.0001, + "loss": 181.245, + "step": 440 + }, + { + "epoch": 0.07266708382955532, + "grad_norm": 3.314251184463501, + "learning_rate": 0.0001, + "loss": 179.3874, + "step": 450 + }, + { + "epoch": 0.07428190791465655, + "grad_norm": 2.635136365890503, + "learning_rate": 0.0001, + "loss": 177.8257, + "step": 460 + }, + { + "epoch": 0.07589673199975777, + "grad_norm": 5.227738857269287, + "learning_rate": 0.0001, + "loss": 176.4699, + "step": 470 + }, + { + "epoch": 0.077511556084859, + "grad_norm": 4.998415470123291, + "learning_rate": 0.0001, + "loss": 175.7601, + "step": 480 + }, + { + "epoch": 0.07912638016996024, + "grad_norm": 2.946892261505127, + "learning_rate": 0.0001, + "loss": 173.936, + "step": 490 + }, + { + "epoch": 0.08074120425506147, + "grad_norm": 4.20358943939209, + "learning_rate": 0.0001, + "loss": 172.065, + "step": 500 + }, + { + "epoch": 0.08235602834016269, + "grad_norm": 3.125765562057495, + "learning_rate": 0.0001, + "loss": 171.3727, + "step": 510 + }, + { + "epoch": 0.08397085242526392, + "grad_norm": 2.4870612621307373, + "learning_rate": 0.0001, + "loss": 169.9, + "step": 520 + }, + { + "epoch": 0.08558567651036515, + "grad_norm": 3.917384147644043, + "learning_rate": 0.0001, + "loss": 168.5049, + "step": 530 + }, + { + "epoch": 0.08720050059546638, + "grad_norm": 2.302142858505249, + "learning_rate": 0.0001, + "loss": 167.2027, + "step": 540 + }, + { + "epoch": 0.08881532468056762, + "grad_norm": 4.794841289520264, + "learning_rate": 0.0001, + "loss": 166.0887, + "step": 550 + }, + { + "epoch": 0.09043014876566884, + "grad_norm": 2.200291395187378, + "learning_rate": 0.0001, + "loss": 164.5882, + "step": 560 + }, + { + "epoch": 0.09204497285077007, + "grad_norm": 1.9572436809539795, + "learning_rate": 0.0001, + "loss": 163.3007, + "step": 570 + }, + { + "epoch": 0.0936597969358713, + "grad_norm": 3.443633556365967, + "learning_rate": 0.0001, + "loss": 160.6805, + "step": 580 + }, + { + "epoch": 0.09527462102097253, + "grad_norm": 2.6079020500183105, + "learning_rate": 0.0001, + "loss": 160.3372, + "step": 590 + }, + { + "epoch": 0.09688944510607375, + "grad_norm": 2.4401192665100098, + "learning_rate": 0.0001, + "loss": 160.0956, + "step": 600 + }, + { + "epoch": 0.09850426919117498, + "grad_norm": 2.686101198196411, + "learning_rate": 0.0001, + "loss": 157.6138, + "step": 610 + }, + { + "epoch": 0.10011909327627622, + "grad_norm": 2.594151496887207, + "learning_rate": 0.0001, + "loss": 156.0726, + "step": 620 + }, + { + "epoch": 0.10173391736137745, + "grad_norm": 2.509141683578491, + "learning_rate": 0.0001, + "loss": 155.9923, + "step": 630 + }, + { + "epoch": 0.10334874144647867, + "grad_norm": 2.4077377319335938, + "learning_rate": 0.0001, + "loss": 155.214, + "step": 640 + }, + { + "epoch": 0.1049635655315799, + "grad_norm": 4.083552837371826, + "learning_rate": 0.0001, + "loss": 152.8257, + "step": 650 + }, + { + "epoch": 0.10657838961668113, + "grad_norm": 2.7188713550567627, + "learning_rate": 0.0001, + "loss": 151.875, + "step": 660 + }, + { + "epoch": 0.10819321370178236, + "grad_norm": 2.41287899017334, + "learning_rate": 0.0001, + "loss": 151.935, + "step": 670 + }, + { + "epoch": 0.1098080377868836, + "grad_norm": 3.995708465576172, + "learning_rate": 0.0001, + "loss": 151.0069, + "step": 680 + }, + { + "epoch": 0.11142286187198482, + "grad_norm": 3.217705488204956, + "learning_rate": 0.0001, + "loss": 149.5922, + "step": 690 + }, + { + "epoch": 0.11303768595708605, + "grad_norm": 3.4349842071533203, + "learning_rate": 0.0001, + "loss": 148.0044, + "step": 700 + }, + { + "epoch": 0.11465251004218728, + "grad_norm": 2.106865167617798, + "learning_rate": 0.0001, + "loss": 147.8404, + "step": 710 + }, + { + "epoch": 0.11626733412728851, + "grad_norm": 1.8729510307312012, + "learning_rate": 0.0001, + "loss": 145.1299, + "step": 720 + }, + { + "epoch": 0.11788215821238973, + "grad_norm": 4.14620304107666, + "learning_rate": 0.0001, + "loss": 144.3928, + "step": 730 + }, + { + "epoch": 0.11949698229749096, + "grad_norm": 2.202505350112915, + "learning_rate": 0.0001, + "loss": 144.8357, + "step": 740 + }, + { + "epoch": 0.1211118063825922, + "grad_norm": 2.201470375061035, + "learning_rate": 0.0001, + "loss": 143.2202, + "step": 750 + }, + { + "epoch": 0.12272663046769343, + "grad_norm": 1.5977853536605835, + "learning_rate": 0.0001, + "loss": 143.3537, + "step": 760 + }, + { + "epoch": 0.12434145455279466, + "grad_norm": 2.4403927326202393, + "learning_rate": 0.0001, + "loss": 141.2914, + "step": 770 + }, + { + "epoch": 0.12595627863789588, + "grad_norm": 1.6895846128463745, + "learning_rate": 0.0001, + "loss": 139.9639, + "step": 780 + }, + { + "epoch": 0.12757110272299713, + "grad_norm": 3.4370665550231934, + "learning_rate": 0.0001, + "loss": 140.715, + "step": 790 + }, + { + "epoch": 0.12918592680809834, + "grad_norm": 2.1449077129364014, + "learning_rate": 0.0001, + "loss": 139.2094, + "step": 800 + }, + { + "epoch": 0.13080075089319956, + "grad_norm": 2.4578912258148193, + "learning_rate": 0.0001, + "loss": 137.6724, + "step": 810 + }, + { + "epoch": 0.1324155749783008, + "grad_norm": 1.8994814157485962, + "learning_rate": 0.0001, + "loss": 136.9551, + "step": 820 + }, + { + "epoch": 0.13403039906340203, + "grad_norm": 2.6531031131744385, + "learning_rate": 0.0001, + "loss": 136.5812, + "step": 830 + }, + { + "epoch": 0.13564522314850325, + "grad_norm": 3.8680083751678467, + "learning_rate": 0.0001, + "loss": 134.3655, + "step": 840 + }, + { + "epoch": 0.1372600472336045, + "grad_norm": 1.9594208002090454, + "learning_rate": 0.0001, + "loss": 135.4958, + "step": 850 + }, + { + "epoch": 0.1388748713187057, + "grad_norm": 1.5475000143051147, + "learning_rate": 0.0001, + "loss": 135.2142, + "step": 860 + }, + { + "epoch": 0.14048969540380696, + "grad_norm": 1.6569288969039917, + "learning_rate": 0.0001, + "loss": 133.4162, + "step": 870 + }, + { + "epoch": 0.14210451948890818, + "grad_norm": 1.6724224090576172, + "learning_rate": 0.0001, + "loss": 133.7519, + "step": 880 + }, + { + "epoch": 0.1437193435740094, + "grad_norm": 1.7299500703811646, + "learning_rate": 0.0001, + "loss": 131.471, + "step": 890 + }, + { + "epoch": 0.14533416765911064, + "grad_norm": 1.9444470405578613, + "learning_rate": 0.0001, + "loss": 128.5966, + "step": 900 + }, + { + "epoch": 0.14694899174421186, + "grad_norm": 2.404723644256592, + "learning_rate": 0.0001, + "loss": 129.9824, + "step": 910 + }, + { + "epoch": 0.1485638158293131, + "grad_norm": 2.1493897438049316, + "learning_rate": 0.0001, + "loss": 128.0219, + "step": 920 + }, + { + "epoch": 0.15017863991441432, + "grad_norm": 2.8861117362976074, + "learning_rate": 0.0001, + "loss": 128.5297, + "step": 930 + }, + { + "epoch": 0.15179346399951554, + "grad_norm": 2.546415090560913, + "learning_rate": 0.0001, + "loss": 126.6817, + "step": 940 + }, + { + "epoch": 0.1534082880846168, + "grad_norm": 1.727663278579712, + "learning_rate": 0.0001, + "loss": 125.0413, + "step": 950 + }, + { + "epoch": 0.155023112169718, + "grad_norm": 3.7156920433044434, + "learning_rate": 0.0001, + "loss": 124.8085, + "step": 960 + }, + { + "epoch": 0.15663793625481925, + "grad_norm": 3.0576517581939697, + "learning_rate": 0.0001, + "loss": 125.9107, + "step": 970 + }, + { + "epoch": 0.15825276033992047, + "grad_norm": 1.9951574802398682, + "learning_rate": 0.0001, + "loss": 123.5359, + "step": 980 + }, + { + "epoch": 0.1598675844250217, + "grad_norm": 1.4620885848999023, + "learning_rate": 0.0001, + "loss": 121.9363, + "step": 990 + }, + { + "epoch": 0.16148240851012294, + "grad_norm": 1.7640169858932495, + "learning_rate": 0.0001, + "loss": 121.1764, + "step": 1000 + }, + { + "epoch": 0.16309723259522416, + "grad_norm": 2.921750545501709, + "learning_rate": 0.0001, + "loss": 119.6931, + "step": 1010 + }, + { + "epoch": 0.16471205668032537, + "grad_norm": 1.5623588562011719, + "learning_rate": 0.0001, + "loss": 119.097, + "step": 1020 + }, + { + "epoch": 0.16632688076542662, + "grad_norm": 2.2386627197265625, + "learning_rate": 0.0001, + "loss": 120.9224, + "step": 1030 + }, + { + "epoch": 0.16794170485052784, + "grad_norm": 1.3213791847229004, + "learning_rate": 0.0001, + "loss": 119.5649, + "step": 1040 + }, + { + "epoch": 0.16955652893562909, + "grad_norm": 1.5255440473556519, + "learning_rate": 0.0001, + "loss": 118.1524, + "step": 1050 + }, + { + "epoch": 0.1711713530207303, + "grad_norm": 1.6038514375686646, + "learning_rate": 0.0001, + "loss": 118.6307, + "step": 1060 + }, + { + "epoch": 0.17278617710583152, + "grad_norm": 2.356739044189453, + "learning_rate": 0.0001, + "loss": 116.8472, + "step": 1070 + }, + { + "epoch": 0.17440100119093277, + "grad_norm": 2.5326743125915527, + "learning_rate": 0.0001, + "loss": 115.6625, + "step": 1080 + }, + { + "epoch": 0.176015825276034, + "grad_norm": 1.2823396921157837, + "learning_rate": 0.0001, + "loss": 115.3312, + "step": 1090 + }, + { + "epoch": 0.17763064936113523, + "grad_norm": 1.981788158416748, + "learning_rate": 0.0001, + "loss": 116.2257, + "step": 1100 + }, + { + "epoch": 0.17924547344623645, + "grad_norm": 1.540440559387207, + "learning_rate": 0.0001, + "loss": 114.5031, + "step": 1110 + }, + { + "epoch": 0.18086029753133767, + "grad_norm": 1.7667800188064575, + "learning_rate": 0.0001, + "loss": 113.7663, + "step": 1120 + }, + { + "epoch": 0.18247512161643892, + "grad_norm": 2.2793102264404297, + "learning_rate": 0.0001, + "loss": 115.219, + "step": 1130 + }, + { + "epoch": 0.18408994570154014, + "grad_norm": 2.132046699523926, + "learning_rate": 0.0001, + "loss": 112.8482, + "step": 1140 + }, + { + "epoch": 0.18570476978664135, + "grad_norm": 1.3153988122940063, + "learning_rate": 0.0001, + "loss": 111.4546, + "step": 1150 + }, + { + "epoch": 0.1873195938717426, + "grad_norm": 1.5246292352676392, + "learning_rate": 0.0001, + "loss": 112.4348, + "step": 1160 + }, + { + "epoch": 0.18893441795684382, + "grad_norm": 1.365096092224121, + "learning_rate": 0.0001, + "loss": 109.755, + "step": 1170 + }, + { + "epoch": 0.19054924204194507, + "grad_norm": 1.2777953147888184, + "learning_rate": 0.0001, + "loss": 111.0318, + "step": 1180 + }, + { + "epoch": 0.19216406612704628, + "grad_norm": 1.8672829866409302, + "learning_rate": 0.0001, + "loss": 109.1648, + "step": 1190 + }, + { + "epoch": 0.1937788902121475, + "grad_norm": 1.709987759590149, + "learning_rate": 0.0001, + "loss": 108.1616, + "step": 1200 + }, + { + "epoch": 0.19539371429724875, + "grad_norm": 1.176277995109558, + "learning_rate": 0.0001, + "loss": 109.3591, + "step": 1210 + }, + { + "epoch": 0.19700853838234997, + "grad_norm": 1.4181180000305176, + "learning_rate": 0.0001, + "loss": 110.0216, + "step": 1220 + }, + { + "epoch": 0.19862336246745121, + "grad_norm": 1.3246116638183594, + "learning_rate": 0.0001, + "loss": 107.9769, + "step": 1230 + }, + { + "epoch": 0.20023818655255243, + "grad_norm": 1.2501498460769653, + "learning_rate": 0.0001, + "loss": 108.2601, + "step": 1240 + }, + { + "epoch": 0.20185301063765365, + "grad_norm": 1.2277404069900513, + "learning_rate": 0.0001, + "loss": 107.1258, + "step": 1250 + }, + { + "epoch": 0.2034678347227549, + "grad_norm": 1.8064894676208496, + "learning_rate": 0.0001, + "loss": 105.6278, + "step": 1260 + }, + { + "epoch": 0.20508265880785612, + "grad_norm": 1.708897352218628, + "learning_rate": 0.0001, + "loss": 106.9269, + "step": 1270 + }, + { + "epoch": 0.20669748289295733, + "grad_norm": 1.4507478475570679, + "learning_rate": 0.0001, + "loss": 106.9236, + "step": 1280 + }, + { + "epoch": 0.20831230697805858, + "grad_norm": 1.4904310703277588, + "learning_rate": 0.0001, + "loss": 106.0653, + "step": 1290 + }, + { + "epoch": 0.2099271310631598, + "grad_norm": 1.2637165784835815, + "learning_rate": 0.0001, + "loss": 104.8665, + "step": 1300 + }, + { + "epoch": 0.21154195514826105, + "grad_norm": 1.4085133075714111, + "learning_rate": 0.0001, + "loss": 104.8891, + "step": 1310 + }, + { + "epoch": 0.21315677923336226, + "grad_norm": 1.5710629224777222, + "learning_rate": 0.0001, + "loss": 104.3009, + "step": 1320 + }, + { + "epoch": 0.21477160331846348, + "grad_norm": 1.3634663820266724, + "learning_rate": 0.0001, + "loss": 104.4545, + "step": 1330 + }, + { + "epoch": 0.21638642740356473, + "grad_norm": 1.5565859079360962, + "learning_rate": 0.0001, + "loss": 103.6205, + "step": 1340 + }, + { + "epoch": 0.21800125148866595, + "grad_norm": 1.4491888284683228, + "learning_rate": 0.0001, + "loss": 104.0892, + "step": 1350 + }, + { + "epoch": 0.2196160755737672, + "grad_norm": 1.7609063386917114, + "learning_rate": 0.0001, + "loss": 104.5177, + "step": 1360 + }, + { + "epoch": 0.2212308996588684, + "grad_norm": 1.6723753213882446, + "learning_rate": 0.0001, + "loss": 104.5552, + "step": 1370 + }, + { + "epoch": 0.22284572374396963, + "grad_norm": 1.3781665563583374, + "learning_rate": 0.0001, + "loss": 102.1022, + "step": 1380 + }, + { + "epoch": 0.22446054782907088, + "grad_norm": 1.5397366285324097, + "learning_rate": 0.0001, + "loss": 102.5882, + "step": 1390 + }, + { + "epoch": 0.2260753719141721, + "grad_norm": 1.4567879438400269, + "learning_rate": 0.0001, + "loss": 101.3781, + "step": 1400 + }, + { + "epoch": 0.22769019599927334, + "grad_norm": 1.2882301807403564, + "learning_rate": 0.0001, + "loss": 100.9378, + "step": 1410 + }, + { + "epoch": 0.22930502008437456, + "grad_norm": 1.4510424137115479, + "learning_rate": 0.0001, + "loss": 101.5383, + "step": 1420 + }, + { + "epoch": 0.23091984416947578, + "grad_norm": 2.8331449031829834, + "learning_rate": 0.0001, + "loss": 100.2773, + "step": 1430 + }, + { + "epoch": 0.23253466825457703, + "grad_norm": 1.363616943359375, + "learning_rate": 0.0001, + "loss": 101.646, + "step": 1440 + }, + { + "epoch": 0.23414949233967824, + "grad_norm": 1.2184548377990723, + "learning_rate": 0.0001, + "loss": 101.1133, + "step": 1450 + }, + { + "epoch": 0.23576431642477946, + "grad_norm": 2.235656976699829, + "learning_rate": 0.0001, + "loss": 100.2044, + "step": 1460 + }, + { + "epoch": 0.2373791405098807, + "grad_norm": 1.2334418296813965, + "learning_rate": 0.0001, + "loss": 99.4031, + "step": 1470 + }, + { + "epoch": 0.23899396459498193, + "grad_norm": 1.2394684553146362, + "learning_rate": 0.0001, + "loss": 101.1247, + "step": 1480 + }, + { + "epoch": 0.24060878868008317, + "grad_norm": 1.5218186378479004, + "learning_rate": 0.0001, + "loss": 100.4413, + "step": 1490 + }, + { + "epoch": 0.2422236127651844, + "grad_norm": 1.2112364768981934, + "learning_rate": 0.0001, + "loss": 99.8722, + "step": 1500 + }, + { + "epoch": 0.2438384368502856, + "grad_norm": 1.2277610301971436, + "learning_rate": 0.0001, + "loss": 99.0276, + "step": 1510 + }, + { + "epoch": 0.24545326093538686, + "grad_norm": 1.1005053520202637, + "learning_rate": 0.0001, + "loss": 98.6764, + "step": 1520 + }, + { + "epoch": 0.24706808502048808, + "grad_norm": 1.0754456520080566, + "learning_rate": 0.0001, + "loss": 97.8968, + "step": 1530 + }, + { + "epoch": 0.24868290910558932, + "grad_norm": 1.0695950984954834, + "learning_rate": 0.0001, + "loss": 98.4032, + "step": 1540 + }, + { + "epoch": 0.25029773319069054, + "grad_norm": 1.5098627805709839, + "learning_rate": 0.0001, + "loss": 97.9391, + "step": 1550 + }, + { + "epoch": 0.25191255727579176, + "grad_norm": 1.006681203842163, + "learning_rate": 0.0001, + "loss": 96.5525, + "step": 1560 + }, + { + "epoch": 0.253527381360893, + "grad_norm": 1.3987332582473755, + "learning_rate": 0.0001, + "loss": 98.383, + "step": 1570 + }, + { + "epoch": 0.25514220544599425, + "grad_norm": 1.290887713432312, + "learning_rate": 0.0001, + "loss": 99.0901, + "step": 1580 + }, + { + "epoch": 0.25675702953109547, + "grad_norm": 0.991177499294281, + "learning_rate": 0.0001, + "loss": 97.7093, + "step": 1590 + }, + { + "epoch": 0.2583718536161967, + "grad_norm": 1.147647738456726, + "learning_rate": 0.0001, + "loss": 96.9181, + "step": 1600 + }, + { + "epoch": 0.2599866777012979, + "grad_norm": 1.1756514310836792, + "learning_rate": 0.0001, + "loss": 98.3935, + "step": 1610 + }, + { + "epoch": 0.2616015017863991, + "grad_norm": 1.1708602905273438, + "learning_rate": 0.0001, + "loss": 96.715, + "step": 1620 + }, + { + "epoch": 0.2632163258715004, + "grad_norm": 1.4577715396881104, + "learning_rate": 0.0001, + "loss": 97.2971, + "step": 1630 + }, + { + "epoch": 0.2648311499566016, + "grad_norm": 1.1172800064086914, + "learning_rate": 0.0001, + "loss": 97.6494, + "step": 1640 + }, + { + "epoch": 0.26644597404170284, + "grad_norm": 1.1491258144378662, + "learning_rate": 0.0001, + "loss": 97.5503, + "step": 1650 + }, + { + "epoch": 0.26806079812680406, + "grad_norm": 1.2188555002212524, + "learning_rate": 0.0001, + "loss": 95.3423, + "step": 1660 + }, + { + "epoch": 0.2696756222119053, + "grad_norm": 1.8595614433288574, + "learning_rate": 0.0001, + "loss": 96.0112, + "step": 1670 + }, + { + "epoch": 0.2712904462970065, + "grad_norm": 1.1887950897216797, + "learning_rate": 0.0001, + "loss": 95.4939, + "step": 1680 + }, + { + "epoch": 0.27290527038210777, + "grad_norm": 1.1777839660644531, + "learning_rate": 0.0001, + "loss": 95.9619, + "step": 1690 + }, + { + "epoch": 0.274520094467209, + "grad_norm": 1.1241233348846436, + "learning_rate": 0.0001, + "loss": 96.2834, + "step": 1700 + }, + { + "epoch": 0.2761349185523102, + "grad_norm": 1.0900121927261353, + "learning_rate": 0.0001, + "loss": 94.7518, + "step": 1710 + }, + { + "epoch": 0.2777497426374114, + "grad_norm": 1.0615205764770508, + "learning_rate": 0.0001, + "loss": 95.8032, + "step": 1720 + }, + { + "epoch": 0.27936456672251264, + "grad_norm": 1.2626168727874756, + "learning_rate": 0.0001, + "loss": 93.9454, + "step": 1730 + }, + { + "epoch": 0.2809793908076139, + "grad_norm": 0.8869941830635071, + "learning_rate": 0.0001, + "loss": 95.4085, + "step": 1740 + }, + { + "epoch": 0.28259421489271513, + "grad_norm": 1.0350773334503174, + "learning_rate": 0.0001, + "loss": 95.7954, + "step": 1750 + }, + { + "epoch": 0.28420903897781635, + "grad_norm": 1.364768147468567, + "learning_rate": 0.0001, + "loss": 93.6447, + "step": 1760 + }, + { + "epoch": 0.28582386306291757, + "grad_norm": 1.0115883350372314, + "learning_rate": 0.0001, + "loss": 93.7905, + "step": 1770 + }, + { + "epoch": 0.2874386871480188, + "grad_norm": 0.9393435120582581, + "learning_rate": 0.0001, + "loss": 94.3108, + "step": 1780 + }, + { + "epoch": 0.28905351123312006, + "grad_norm": 1.0769627094268799, + "learning_rate": 0.0001, + "loss": 92.566, + "step": 1790 + }, + { + "epoch": 0.2906683353182213, + "grad_norm": 2.526201009750366, + "learning_rate": 0.0001, + "loss": 92.8161, + "step": 1800 + }, + { + "epoch": 0.2922831594033225, + "grad_norm": 1.3066468238830566, + "learning_rate": 0.0001, + "loss": 92.0459, + "step": 1810 + }, + { + "epoch": 0.2938979834884237, + "grad_norm": 1.121713638305664, + "learning_rate": 0.0001, + "loss": 93.5032, + "step": 1820 + }, + { + "epoch": 0.29551280757352494, + "grad_norm": 1.3744316101074219, + "learning_rate": 0.0001, + "loss": 94.2623, + "step": 1830 + }, + { + "epoch": 0.2971276316586262, + "grad_norm": 1.1561540365219116, + "learning_rate": 0.0001, + "loss": 94.1611, + "step": 1840 + }, + { + "epoch": 0.29874245574372743, + "grad_norm": 1.3947464227676392, + "learning_rate": 0.0001, + "loss": 93.5962, + "step": 1850 + }, + { + "epoch": 0.30035727982882865, + "grad_norm": 1.0011566877365112, + "learning_rate": 0.0001, + "loss": 92.0199, + "step": 1860 + }, + { + "epoch": 0.30197210391392987, + "grad_norm": 0.9978747963905334, + "learning_rate": 0.0001, + "loss": 91.0945, + "step": 1870 + }, + { + "epoch": 0.3035869279990311, + "grad_norm": 0.92522132396698, + "learning_rate": 0.0001, + "loss": 93.274, + "step": 1880 + }, + { + "epoch": 0.30520175208413236, + "grad_norm": 1.0089120864868164, + "learning_rate": 0.0001, + "loss": 91.9308, + "step": 1890 + }, + { + "epoch": 0.3068165761692336, + "grad_norm": 1.0535134077072144, + "learning_rate": 0.0001, + "loss": 92.563, + "step": 1900 + }, + { + "epoch": 0.3084314002543348, + "grad_norm": 0.9782275557518005, + "learning_rate": 0.0001, + "loss": 90.3486, + "step": 1910 + }, + { + "epoch": 0.310046224339436, + "grad_norm": 1.1433162689208984, + "learning_rate": 0.0001, + "loss": 90.4977, + "step": 1920 + }, + { + "epoch": 0.31166104842453723, + "grad_norm": 0.98392653465271, + "learning_rate": 0.0001, + "loss": 92.6529, + "step": 1930 + }, + { + "epoch": 0.3132758725096385, + "grad_norm": 0.8538148403167725, + "learning_rate": 0.0001, + "loss": 90.6859, + "step": 1940 + }, + { + "epoch": 0.3148906965947397, + "grad_norm": 1.019405484199524, + "learning_rate": 0.0001, + "loss": 91.0028, + "step": 1950 + }, + { + "epoch": 0.31650552067984095, + "grad_norm": 0.9828067421913147, + "learning_rate": 0.0001, + "loss": 89.9826, + "step": 1960 + }, + { + "epoch": 0.31812034476494216, + "grad_norm": 0.9725450277328491, + "learning_rate": 0.0001, + "loss": 91.7342, + "step": 1970 + }, + { + "epoch": 0.3197351688500434, + "grad_norm": 1.0139408111572266, + "learning_rate": 0.0001, + "loss": 90.5362, + "step": 1980 + }, + { + "epoch": 0.3213499929351446, + "grad_norm": 1.2493853569030762, + "learning_rate": 0.0001, + "loss": 92.7222, + "step": 1990 + }, + { + "epoch": 0.3229648170202459, + "grad_norm": 1.0152915716171265, + "learning_rate": 0.0001, + "loss": 92.2271, + "step": 2000 + }, + { + "epoch": 0.3245796411053471, + "grad_norm": 0.9745819568634033, + "learning_rate": 0.0001, + "loss": 88.7034, + "step": 2010 + }, + { + "epoch": 0.3261944651904483, + "grad_norm": 1.009624719619751, + "learning_rate": 0.0001, + "loss": 90.7428, + "step": 2020 + }, + { + "epoch": 0.32780928927554953, + "grad_norm": 0.910858690738678, + "learning_rate": 0.0001, + "loss": 89.8199, + "step": 2030 + }, + { + "epoch": 0.32942411336065075, + "grad_norm": 1.1745878458023071, + "learning_rate": 0.0001, + "loss": 90.8267, + "step": 2040 + }, + { + "epoch": 0.331038937445752, + "grad_norm": 0.9409273266792297, + "learning_rate": 0.0001, + "loss": 90.2937, + "step": 2050 + }, + { + "epoch": 0.33265376153085324, + "grad_norm": 1.4152942895889282, + "learning_rate": 0.0001, + "loss": 90.3965, + "step": 2060 + }, + { + "epoch": 0.33426858561595446, + "grad_norm": 0.9529693722724915, + "learning_rate": 0.0001, + "loss": 90.4739, + "step": 2070 + }, + { + "epoch": 0.3358834097010557, + "grad_norm": 0.8531096577644348, + "learning_rate": 0.0001, + "loss": 90.9301, + "step": 2080 + }, + { + "epoch": 0.3374982337861569, + "grad_norm": 0.9314167499542236, + "learning_rate": 0.0001, + "loss": 90.4235, + "step": 2090 + }, + { + "epoch": 0.33911305787125817, + "grad_norm": 0.9253250360488892, + "learning_rate": 0.0001, + "loss": 88.1893, + "step": 2100 + }, + { + "epoch": 0.3407278819563594, + "grad_norm": 1.773053526878357, + "learning_rate": 0.0001, + "loss": 88.3417, + "step": 2110 + }, + { + "epoch": 0.3423427060414606, + "grad_norm": 0.9432951211929321, + "learning_rate": 0.0001, + "loss": 89.7141, + "step": 2120 + }, + { + "epoch": 0.34395753012656183, + "grad_norm": 3.8847527503967285, + "learning_rate": 0.0001, + "loss": 89.1009, + "step": 2130 + }, + { + "epoch": 0.34557235421166305, + "grad_norm": 1.0309821367263794, + "learning_rate": 0.0001, + "loss": 89.2965, + "step": 2140 + }, + { + "epoch": 0.3471871782967643, + "grad_norm": 0.9590495824813843, + "learning_rate": 0.0001, + "loss": 89.1734, + "step": 2150 + }, + { + "epoch": 0.34880200238186554, + "grad_norm": 0.9943560361862183, + "learning_rate": 0.0001, + "loss": 85.425, + "step": 2160 + }, + { + "epoch": 0.35041682646696676, + "grad_norm": 1.3322339057922363, + "learning_rate": 0.0001, + "loss": 89.7791, + "step": 2170 + }, + { + "epoch": 0.352031650552068, + "grad_norm": 1.0212253332138062, + "learning_rate": 0.0001, + "loss": 89.3771, + "step": 2180 + }, + { + "epoch": 0.3536464746371692, + "grad_norm": 1.0446089506149292, + "learning_rate": 0.0001, + "loss": 88.6894, + "step": 2190 + }, + { + "epoch": 0.35526129872227047, + "grad_norm": 1.05767822265625, + "learning_rate": 0.0001, + "loss": 89.5341, + "step": 2200 + }, + { + "epoch": 0.3568761228073717, + "grad_norm": 1.0373681783676147, + "learning_rate": 0.0001, + "loss": 89.7173, + "step": 2210 + }, + { + "epoch": 0.3584909468924729, + "grad_norm": 0.9098039269447327, + "learning_rate": 0.0001, + "loss": 89.2116, + "step": 2220 + }, + { + "epoch": 0.3601057709775741, + "grad_norm": 1.028130292892456, + "learning_rate": 0.0001, + "loss": 88.6627, + "step": 2230 + }, + { + "epoch": 0.36172059506267534, + "grad_norm": 1.0295171737670898, + "learning_rate": 0.0001, + "loss": 89.608, + "step": 2240 + }, + { + "epoch": 0.36333541914777656, + "grad_norm": 0.9720722436904907, + "learning_rate": 0.0001, + "loss": 88.3624, + "step": 2250 + }, + { + "epoch": 0.36495024323287784, + "grad_norm": 0.839654803276062, + "learning_rate": 0.0001, + "loss": 87.2983, + "step": 2260 + }, + { + "epoch": 0.36656506731797905, + "grad_norm": 1.1600219011306763, + "learning_rate": 0.0001, + "loss": 86.6004, + "step": 2270 + }, + { + "epoch": 0.3681798914030803, + "grad_norm": 1.0513924360275269, + "learning_rate": 0.0001, + "loss": 86.6062, + "step": 2280 + }, + { + "epoch": 0.3697947154881815, + "grad_norm": 0.9728474617004395, + "learning_rate": 0.0001, + "loss": 86.1856, + "step": 2290 + }, + { + "epoch": 0.3714095395732827, + "grad_norm": 1.0196386575698853, + "learning_rate": 0.0001, + "loss": 87.6088, + "step": 2300 + }, + { + "epoch": 0.373024363658384, + "grad_norm": 1.0439444780349731, + "learning_rate": 0.0001, + "loss": 85.2374, + "step": 2310 + }, + { + "epoch": 0.3746391877434852, + "grad_norm": 1.0020275115966797, + "learning_rate": 0.0001, + "loss": 86.0596, + "step": 2320 + }, + { + "epoch": 0.3762540118285864, + "grad_norm": 1.0107691287994385, + "learning_rate": 0.0001, + "loss": 86.9513, + "step": 2330 + }, + { + "epoch": 0.37786883591368764, + "grad_norm": 1.680018663406372, + "learning_rate": 0.0001, + "loss": 87.8885, + "step": 2340 + }, + { + "epoch": 0.37948365999878886, + "grad_norm": 0.9181709885597229, + "learning_rate": 0.0001, + "loss": 86.7662, + "step": 2350 + }, + { + "epoch": 0.38109848408389013, + "grad_norm": 0.9598278403282166, + "learning_rate": 0.0001, + "loss": 88.3775, + "step": 2360 + }, + { + "epoch": 0.38271330816899135, + "grad_norm": 1.0707905292510986, + "learning_rate": 0.0001, + "loss": 86.0997, + "step": 2370 + }, + { + "epoch": 0.38432813225409257, + "grad_norm": 1.125797152519226, + "learning_rate": 0.0001, + "loss": 85.8117, + "step": 2380 + }, + { + "epoch": 0.3859429563391938, + "grad_norm": 1.0251439809799194, + "learning_rate": 0.0001, + "loss": 86.0806, + "step": 2390 + }, + { + "epoch": 0.387557780424295, + "grad_norm": 1.0193125009536743, + "learning_rate": 0.0001, + "loss": 87.264, + "step": 2400 + }, + { + "epoch": 0.3891726045093963, + "grad_norm": 0.9309751391410828, + "learning_rate": 0.0001, + "loss": 85.9187, + "step": 2410 + }, + { + "epoch": 0.3907874285944975, + "grad_norm": 1.1025906801223755, + "learning_rate": 0.0001, + "loss": 86.4365, + "step": 2420 + }, + { + "epoch": 0.3924022526795987, + "grad_norm": 0.9880627393722534, + "learning_rate": 0.0001, + "loss": 86.2303, + "step": 2430 + }, + { + "epoch": 0.39401707676469994, + "grad_norm": 0.9520611763000488, + "learning_rate": 0.0001, + "loss": 84.9035, + "step": 2440 + }, + { + "epoch": 0.39563190084980115, + "grad_norm": 0.8641896843910217, + "learning_rate": 0.0001, + "loss": 85.4754, + "step": 2450 + }, + { + "epoch": 0.39724672493490243, + "grad_norm": 1.1162904500961304, + "learning_rate": 0.0001, + "loss": 85.6767, + "step": 2460 + }, + { + "epoch": 0.39886154902000365, + "grad_norm": 0.9073636531829834, + "learning_rate": 0.0001, + "loss": 85.4198, + "step": 2470 + }, + { + "epoch": 0.40047637310510487, + "grad_norm": 1.1135166883468628, + "learning_rate": 0.0001, + "loss": 86.0884, + "step": 2480 + }, + { + "epoch": 0.4020911971902061, + "grad_norm": 0.9273980259895325, + "learning_rate": 0.0001, + "loss": 85.5473, + "step": 2490 + }, + { + "epoch": 0.4037060212753073, + "grad_norm": 0.8653831481933594, + "learning_rate": 0.0001, + "loss": 87.279, + "step": 2500 + }, + { + "epoch": 0.4053208453604086, + "grad_norm": 0.8535159826278687, + "learning_rate": 0.0001, + "loss": 86.7072, + "step": 2510 + }, + { + "epoch": 0.4069356694455098, + "grad_norm": 0.8864684700965881, + "learning_rate": 0.0001, + "loss": 86.441, + "step": 2520 + }, + { + "epoch": 0.408550493530611, + "grad_norm": 0.8042080998420715, + "learning_rate": 0.0001, + "loss": 83.1706, + "step": 2530 + }, + { + "epoch": 0.41016531761571223, + "grad_norm": 1.0486373901367188, + "learning_rate": 0.0001, + "loss": 83.54, + "step": 2540 + }, + { + "epoch": 0.41178014170081345, + "grad_norm": 0.8087959289550781, + "learning_rate": 0.0001, + "loss": 85.7615, + "step": 2550 + }, + { + "epoch": 0.41339496578591467, + "grad_norm": 1.5032018423080444, + "learning_rate": 0.0001, + "loss": 85.7368, + "step": 2560 + }, + { + "epoch": 0.41500978987101594, + "grad_norm": 1.0786182880401611, + "learning_rate": 0.0001, + "loss": 86.3484, + "step": 2570 + }, + { + "epoch": 0.41662461395611716, + "grad_norm": 0.8744294047355652, + "learning_rate": 0.0001, + "loss": 83.5945, + "step": 2580 + }, + { + "epoch": 0.4182394380412184, + "grad_norm": 0.7840721607208252, + "learning_rate": 0.0001, + "loss": 86.3831, + "step": 2590 + }, + { + "epoch": 0.4198542621263196, + "grad_norm": 0.7977836728096008, + "learning_rate": 0.0001, + "loss": 83.6314, + "step": 2600 + }, + { + "epoch": 0.4214690862114208, + "grad_norm": 0.892484188079834, + "learning_rate": 0.0001, + "loss": 86.4223, + "step": 2610 + }, + { + "epoch": 0.4230839102965221, + "grad_norm": 0.912995457649231, + "learning_rate": 0.0001, + "loss": 83.8971, + "step": 2620 + }, + { + "epoch": 0.4246987343816233, + "grad_norm": 0.9566861987113953, + "learning_rate": 0.0001, + "loss": 85.291, + "step": 2630 + }, + { + "epoch": 0.42631355846672453, + "grad_norm": 0.8324154615402222, + "learning_rate": 0.0001, + "loss": 84.6233, + "step": 2640 + }, + { + "epoch": 0.42792838255182575, + "grad_norm": 0.9490360617637634, + "learning_rate": 0.0001, + "loss": 81.799, + "step": 2650 + }, + { + "epoch": 0.42954320663692697, + "grad_norm": 0.9744566679000854, + "learning_rate": 0.0001, + "loss": 85.4837, + "step": 2660 + }, + { + "epoch": 0.43115803072202824, + "grad_norm": 0.941614031791687, + "learning_rate": 0.0001, + "loss": 82.838, + "step": 2670 + }, + { + "epoch": 0.43277285480712946, + "grad_norm": 1.338364601135254, + "learning_rate": 0.0001, + "loss": 84.4224, + "step": 2680 + }, + { + "epoch": 0.4343876788922307, + "grad_norm": 0.8643816113471985, + "learning_rate": 0.0001, + "loss": 84.3959, + "step": 2690 + }, + { + "epoch": 0.4360025029773319, + "grad_norm": 1.0231473445892334, + "learning_rate": 0.0001, + "loss": 82.3144, + "step": 2700 + }, + { + "epoch": 0.4376173270624331, + "grad_norm": 0.9089650511741638, + "learning_rate": 0.0001, + "loss": 83.0326, + "step": 2710 + }, + { + "epoch": 0.4392321511475344, + "grad_norm": 0.893628716468811, + "learning_rate": 0.0001, + "loss": 83.0981, + "step": 2720 + }, + { + "epoch": 0.4408469752326356, + "grad_norm": 0.869076132774353, + "learning_rate": 0.0001, + "loss": 84.2571, + "step": 2730 + }, + { + "epoch": 0.4424617993177368, + "grad_norm": 0.7964235544204712, + "learning_rate": 0.0001, + "loss": 83.3329, + "step": 2740 + }, + { + "epoch": 0.44407662340283804, + "grad_norm": 3.1645138263702393, + "learning_rate": 0.0001, + "loss": 83.7232, + "step": 2750 + }, + { + "epoch": 0.44569144748793926, + "grad_norm": 0.934644877910614, + "learning_rate": 0.0001, + "loss": 85.0688, + "step": 2760 + }, + { + "epoch": 0.44730627157304054, + "grad_norm": 0.8120711445808411, + "learning_rate": 0.0001, + "loss": 82.7706, + "step": 2770 + }, + { + "epoch": 0.44892109565814176, + "grad_norm": 0.8161435127258301, + "learning_rate": 0.0001, + "loss": 84.4826, + "step": 2780 + }, + { + "epoch": 0.450535919743243, + "grad_norm": 0.8659493327140808, + "learning_rate": 0.0001, + "loss": 82.6222, + "step": 2790 + }, + { + "epoch": 0.4521507438283442, + "grad_norm": 0.9306702613830566, + "learning_rate": 0.0001, + "loss": 83.5897, + "step": 2800 + }, + { + "epoch": 0.4537655679134454, + "grad_norm": 2.2544431686401367, + "learning_rate": 0.0001, + "loss": 82.9603, + "step": 2810 + }, + { + "epoch": 0.4553803919985467, + "grad_norm": 0.8868395686149597, + "learning_rate": 0.0001, + "loss": 82.6674, + "step": 2820 + }, + { + "epoch": 0.4569952160836479, + "grad_norm": 0.9336455464363098, + "learning_rate": 0.0001, + "loss": 84.6974, + "step": 2830 + }, + { + "epoch": 0.4586100401687491, + "grad_norm": 0.8814165592193604, + "learning_rate": 0.0001, + "loss": 83.9329, + "step": 2840 + }, + { + "epoch": 0.46022486425385034, + "grad_norm": 0.8125660419464111, + "learning_rate": 0.0001, + "loss": 84.6168, + "step": 2850 + }, + { + "epoch": 0.46183968833895156, + "grad_norm": 0.9447190165519714, + "learning_rate": 0.0001, + "loss": 82.4098, + "step": 2860 + }, + { + "epoch": 0.4634545124240528, + "grad_norm": 0.8080969452857971, + "learning_rate": 0.0001, + "loss": 83.4076, + "step": 2870 + }, + { + "epoch": 0.46506933650915405, + "grad_norm": 1.0586694478988647, + "learning_rate": 0.0001, + "loss": 83.3295, + "step": 2880 + }, + { + "epoch": 0.46668416059425527, + "grad_norm": 0.8207664489746094, + "learning_rate": 0.0001, + "loss": 84.5056, + "step": 2890 + }, + { + "epoch": 0.4682989846793565, + "grad_norm": 1.0647571086883545, + "learning_rate": 0.0001, + "loss": 83.0766, + "step": 2900 + }, + { + "epoch": 0.4699138087644577, + "grad_norm": 0.9416915774345398, + "learning_rate": 0.0001, + "loss": 83.8166, + "step": 2910 + }, + { + "epoch": 0.4715286328495589, + "grad_norm": 0.7479362487792969, + "learning_rate": 0.0001, + "loss": 82.7507, + "step": 2920 + }, + { + "epoch": 0.4731434569346602, + "grad_norm": 1.0775514841079712, + "learning_rate": 0.0001, + "loss": 84.4641, + "step": 2930 + }, + { + "epoch": 0.4747582810197614, + "grad_norm": 0.8736841082572937, + "learning_rate": 0.0001, + "loss": 82.0451, + "step": 2940 + }, + { + "epoch": 0.47637310510486264, + "grad_norm": 0.9500495791435242, + "learning_rate": 0.0001, + "loss": 82.6425, + "step": 2950 + }, + { + "epoch": 0.47798792918996386, + "grad_norm": 0.8551210165023804, + "learning_rate": 0.0001, + "loss": 81.1971, + "step": 2960 + }, + { + "epoch": 0.4796027532750651, + "grad_norm": 0.8149814605712891, + "learning_rate": 0.0001, + "loss": 82.7919, + "step": 2970 + }, + { + "epoch": 0.48121757736016635, + "grad_norm": 0.9012249112129211, + "learning_rate": 0.0001, + "loss": 81.975, + "step": 2980 + }, + { + "epoch": 0.48283240144526757, + "grad_norm": 1.3784620761871338, + "learning_rate": 0.0001, + "loss": 81.541, + "step": 2990 + }, + { + "epoch": 0.4844472255303688, + "grad_norm": 0.7921801209449768, + "learning_rate": 0.0001, + "loss": 83.6215, + "step": 3000 + }, + { + "epoch": 0.48606204961547, + "grad_norm": 0.9051580429077148, + "learning_rate": 0.0001, + "loss": 82.4048, + "step": 3010 + }, + { + "epoch": 0.4876768737005712, + "grad_norm": 1.0717192888259888, + "learning_rate": 0.0001, + "loss": 82.4079, + "step": 3020 + }, + { + "epoch": 0.4892916977856725, + "grad_norm": 0.8971781730651855, + "learning_rate": 0.0001, + "loss": 83.3668, + "step": 3030 + }, + { + "epoch": 0.4909065218707737, + "grad_norm": 0.9635320901870728, + "learning_rate": 0.0001, + "loss": 82.8244, + "step": 3040 + }, + { + "epoch": 0.49252134595587493, + "grad_norm": 1.1009949445724487, + "learning_rate": 0.0001, + "loss": 82.2174, + "step": 3050 + }, + { + "epoch": 0.49413617004097615, + "grad_norm": 0.9805473685264587, + "learning_rate": 0.0001, + "loss": 83.7897, + "step": 3060 + }, + { + "epoch": 0.49575099412607737, + "grad_norm": 1.2107361555099487, + "learning_rate": 0.0001, + "loss": 82.5108, + "step": 3070 + }, + { + "epoch": 0.49736581821117865, + "grad_norm": 0.9171829223632812, + "learning_rate": 0.0001, + "loss": 80.9126, + "step": 3080 + }, + { + "epoch": 0.49898064229627986, + "grad_norm": 0.8093295097351074, + "learning_rate": 0.0001, + "loss": 83.8147, + "step": 3090 + }, + { + "epoch": 0.5005954663813811, + "grad_norm": 1.2166653871536255, + "learning_rate": 0.0001, + "loss": 82.0288, + "step": 3100 + }, + { + "epoch": 0.5022102904664824, + "grad_norm": 1.0471502542495728, + "learning_rate": 0.0001, + "loss": 82.0706, + "step": 3110 + }, + { + "epoch": 0.5038251145515835, + "grad_norm": 1.007645845413208, + "learning_rate": 0.0001, + "loss": 81.7387, + "step": 3120 + }, + { + "epoch": 0.5054399386366848, + "grad_norm": 0.7252638339996338, + "learning_rate": 0.0001, + "loss": 82.9107, + "step": 3130 + }, + { + "epoch": 0.507054762721786, + "grad_norm": 0.8565683364868164, + "learning_rate": 0.0001, + "loss": 80.2935, + "step": 3140 + }, + { + "epoch": 0.5086695868068872, + "grad_norm": 0.9905404448509216, + "learning_rate": 0.0001, + "loss": 82.5201, + "step": 3150 + }, + { + "epoch": 0.5102844108919885, + "grad_norm": 0.887428343296051, + "learning_rate": 0.0001, + "loss": 80.8528, + "step": 3160 + }, + { + "epoch": 0.5118992349770897, + "grad_norm": 1.062942624092102, + "learning_rate": 0.0001, + "loss": 80.7035, + "step": 3170 + }, + { + "epoch": 0.5135140590621909, + "grad_norm": 1.203599214553833, + "learning_rate": 0.0001, + "loss": 81.2521, + "step": 3180 + }, + { + "epoch": 0.5151288831472921, + "grad_norm": 0.8452383875846863, + "learning_rate": 0.0001, + "loss": 81.6079, + "step": 3190 + }, + { + "epoch": 0.5167437072323934, + "grad_norm": 1.2530587911605835, + "learning_rate": 0.0001, + "loss": 82.2148, + "step": 3200 + }, + { + "epoch": 0.5183585313174947, + "grad_norm": 0.9351627826690674, + "learning_rate": 0.0001, + "loss": 82.7857, + "step": 3210 + }, + { + "epoch": 0.5199733554025958, + "grad_norm": 1.0128077268600464, + "learning_rate": 0.0001, + "loss": 81.0532, + "step": 3220 + }, + { + "epoch": 0.5215881794876971, + "grad_norm": 0.9316917061805725, + "learning_rate": 0.0001, + "loss": 80.7443, + "step": 3230 + }, + { + "epoch": 0.5232030035727983, + "grad_norm": 1.502705693244934, + "learning_rate": 0.0001, + "loss": 80.0608, + "step": 3240 + }, + { + "epoch": 0.5248178276578995, + "grad_norm": 0.8166837692260742, + "learning_rate": 0.0001, + "loss": 81.2013, + "step": 3250 + }, + { + "epoch": 0.5264326517430008, + "grad_norm": 0.8210042119026184, + "learning_rate": 0.0001, + "loss": 80.2491, + "step": 3260 + }, + { + "epoch": 0.528047475828102, + "grad_norm": 0.7854771018028259, + "learning_rate": 0.0001, + "loss": 81.1017, + "step": 3270 + }, + { + "epoch": 0.5296622999132032, + "grad_norm": 0.8530565500259399, + "learning_rate": 0.0001, + "loss": 80.8123, + "step": 3280 + }, + { + "epoch": 0.5312771239983044, + "grad_norm": 0.9446498155593872, + "learning_rate": 0.0001, + "loss": 80.0029, + "step": 3290 + }, + { + "epoch": 0.5328919480834057, + "grad_norm": 0.9646537899971008, + "learning_rate": 0.0001, + "loss": 81.5413, + "step": 3300 + }, + { + "epoch": 0.5345067721685068, + "grad_norm": 0.9262019395828247, + "learning_rate": 0.0001, + "loss": 80.9399, + "step": 3310 + }, + { + "epoch": 0.5361215962536081, + "grad_norm": 0.8515486121177673, + "learning_rate": 0.0001, + "loss": 80.9421, + "step": 3320 + }, + { + "epoch": 0.5377364203387094, + "grad_norm": 1.679321050643921, + "learning_rate": 0.0001, + "loss": 82.4851, + "step": 3330 + }, + { + "epoch": 0.5393512444238105, + "grad_norm": 0.8289812207221985, + "learning_rate": 0.0001, + "loss": 80.5664, + "step": 3340 + }, + { + "epoch": 0.5409660685089118, + "grad_norm": 0.7463789582252502, + "learning_rate": 0.0001, + "loss": 81.5193, + "step": 3350 + }, + { + "epoch": 0.542580892594013, + "grad_norm": 0.8332446217536926, + "learning_rate": 0.0001, + "loss": 80.8091, + "step": 3360 + }, + { + "epoch": 0.5441957166791143, + "grad_norm": 0.7612847089767456, + "learning_rate": 0.0001, + "loss": 79.8893, + "step": 3370 + }, + { + "epoch": 0.5458105407642155, + "grad_norm": 1.0557948350906372, + "learning_rate": 0.0001, + "loss": 81.4305, + "step": 3380 + }, + { + "epoch": 0.5474253648493167, + "grad_norm": 0.7975788116455078, + "learning_rate": 0.0001, + "loss": 80.0105, + "step": 3390 + }, + { + "epoch": 0.549040188934418, + "grad_norm": 0.9049227237701416, + "learning_rate": 0.0001, + "loss": 81.2689, + "step": 3400 + }, + { + "epoch": 0.5506550130195191, + "grad_norm": 1.1195895671844482, + "learning_rate": 0.0001, + "loss": 80.7163, + "step": 3410 + }, + { + "epoch": 0.5522698371046204, + "grad_norm": 0.7506540417671204, + "learning_rate": 0.0001, + "loss": 79.0855, + "step": 3420 + }, + { + "epoch": 0.5538846611897217, + "grad_norm": 0.73989337682724, + "learning_rate": 0.0001, + "loss": 81.1865, + "step": 3430 + }, + { + "epoch": 0.5554994852748228, + "grad_norm": 0.8418401479721069, + "learning_rate": 0.0001, + "loss": 79.5714, + "step": 3440 + }, + { + "epoch": 0.5571143093599241, + "grad_norm": 0.9807226061820984, + "learning_rate": 0.0001, + "loss": 78.5699, + "step": 3450 + }, + { + "epoch": 0.5587291334450253, + "grad_norm": 1.2890545129776, + "learning_rate": 0.0001, + "loss": 81.0128, + "step": 3460 + }, + { + "epoch": 0.5603439575301266, + "grad_norm": 0.8593361973762512, + "learning_rate": 0.0001, + "loss": 80.1638, + "step": 3470 + }, + { + "epoch": 0.5619587816152278, + "grad_norm": 0.7766354084014893, + "learning_rate": 0.0001, + "loss": 81.5597, + "step": 3480 + }, + { + "epoch": 0.563573605700329, + "grad_norm": 0.7785626649856567, + "learning_rate": 0.0001, + "loss": 79.3862, + "step": 3490 + }, + { + "epoch": 0.5651884297854303, + "grad_norm": 0.8429279327392578, + "learning_rate": 0.0001, + "loss": 76.6294, + "step": 3500 + }, + { + "epoch": 0.5668032538705314, + "grad_norm": 0.8141765594482422, + "learning_rate": 0.0001, + "loss": 80.2584, + "step": 3510 + }, + { + "epoch": 0.5684180779556327, + "grad_norm": 0.7065490484237671, + "learning_rate": 0.0001, + "loss": 79.3626, + "step": 3520 + }, + { + "epoch": 0.570032902040734, + "grad_norm": 0.7002031803131104, + "learning_rate": 0.0001, + "loss": 79.283, + "step": 3530 + }, + { + "epoch": 0.5716477261258351, + "grad_norm": 0.8715317845344543, + "learning_rate": 0.0001, + "loss": 80.3145, + "step": 3540 + }, + { + "epoch": 0.5732625502109364, + "grad_norm": 0.9584882855415344, + "learning_rate": 0.0001, + "loss": 78.7443, + "step": 3550 + }, + { + "epoch": 0.5748773742960376, + "grad_norm": 0.7908946871757507, + "learning_rate": 0.0001, + "loss": 79.1906, + "step": 3560 + }, + { + "epoch": 0.5764921983811389, + "grad_norm": 0.8416746258735657, + "learning_rate": 0.0001, + "loss": 77.8172, + "step": 3570 + }, + { + "epoch": 0.5781070224662401, + "grad_norm": 0.823667049407959, + "learning_rate": 0.0001, + "loss": 81.864, + "step": 3580 + }, + { + "epoch": 0.5797218465513413, + "grad_norm": 1.4282617568969727, + "learning_rate": 0.0001, + "loss": 79.9052, + "step": 3590 + }, + { + "epoch": 0.5813366706364426, + "grad_norm": 0.8747813701629639, + "learning_rate": 0.0001, + "loss": 78.8234, + "step": 3600 + }, + { + "epoch": 0.5829514947215437, + "grad_norm": 0.8415511846542358, + "learning_rate": 0.0001, + "loss": 78.8171, + "step": 3610 + }, + { + "epoch": 0.584566318806645, + "grad_norm": 0.9014492034912109, + "learning_rate": 0.0001, + "loss": 80.5011, + "step": 3620 + }, + { + "epoch": 0.5861811428917463, + "grad_norm": 0.7535204887390137, + "learning_rate": 0.0001, + "loss": 79.4895, + "step": 3630 + }, + { + "epoch": 0.5877959669768474, + "grad_norm": 0.8196508884429932, + "learning_rate": 0.0001, + "loss": 79.7101, + "step": 3640 + }, + { + "epoch": 0.5894107910619487, + "grad_norm": 1.5975492000579834, + "learning_rate": 0.0001, + "loss": 79.0687, + "step": 3650 + }, + { + "epoch": 0.5910256151470499, + "grad_norm": 0.9251692295074463, + "learning_rate": 0.0001, + "loss": 80.2377, + "step": 3660 + }, + { + "epoch": 0.5926404392321512, + "grad_norm": 0.7143557071685791, + "learning_rate": 0.0001, + "loss": 78.0671, + "step": 3670 + }, + { + "epoch": 0.5942552633172524, + "grad_norm": 0.7876290678977966, + "learning_rate": 0.0001, + "loss": 79.1455, + "step": 3680 + }, + { + "epoch": 0.5958700874023536, + "grad_norm": 1.5361895561218262, + "learning_rate": 0.0001, + "loss": 78.1253, + "step": 3690 + }, + { + "epoch": 0.5974849114874549, + "grad_norm": 0.7637040019035339, + "learning_rate": 0.0001, + "loss": 77.4658, + "step": 3700 + }, + { + "epoch": 0.599099735572556, + "grad_norm": 0.8166250586509705, + "learning_rate": 0.0001, + "loss": 79.5806, + "step": 3710 + }, + { + "epoch": 0.6007145596576573, + "grad_norm": 1.2767380475997925, + "learning_rate": 0.0001, + "loss": 79.1894, + "step": 3720 + }, + { + "epoch": 0.6023293837427586, + "grad_norm": 1.3048640489578247, + "learning_rate": 0.0001, + "loss": 78.0593, + "step": 3730 + }, + { + "epoch": 0.6039442078278597, + "grad_norm": 0.7945445775985718, + "learning_rate": 0.0001, + "loss": 79.7623, + "step": 3740 + }, + { + "epoch": 0.605559031912961, + "grad_norm": 0.7366836667060852, + "learning_rate": 0.0001, + "loss": 77.9571, + "step": 3750 + }, + { + "epoch": 0.6071738559980622, + "grad_norm": 0.9126115441322327, + "learning_rate": 0.0001, + "loss": 79.9502, + "step": 3760 + }, + { + "epoch": 0.6087886800831634, + "grad_norm": 0.800355851650238, + "learning_rate": 0.0001, + "loss": 78.7244, + "step": 3770 + }, + { + "epoch": 0.6104035041682647, + "grad_norm": 0.8290980458259583, + "learning_rate": 0.0001, + "loss": 78.7724, + "step": 3780 + }, + { + "epoch": 0.6120183282533659, + "grad_norm": 0.8512316942214966, + "learning_rate": 0.0001, + "loss": 78.6277, + "step": 3790 + }, + { + "epoch": 0.6136331523384672, + "grad_norm": 0.8909353613853455, + "learning_rate": 0.0001, + "loss": 78.5956, + "step": 3800 + }, + { + "epoch": 0.6152479764235683, + "grad_norm": 0.8328800797462463, + "learning_rate": 0.0001, + "loss": 78.1323, + "step": 3810 + }, + { + "epoch": 0.6168628005086696, + "grad_norm": 0.7950423359870911, + "learning_rate": 0.0001, + "loss": 78.7529, + "step": 3820 + }, + { + "epoch": 0.6184776245937709, + "grad_norm": 0.9988545179367065, + "learning_rate": 0.0001, + "loss": 80.7727, + "step": 3830 + }, + { + "epoch": 0.620092448678872, + "grad_norm": 0.7243085503578186, + "learning_rate": 0.0001, + "loss": 76.7616, + "step": 3840 + }, + { + "epoch": 0.6217072727639733, + "grad_norm": 0.7674855589866638, + "learning_rate": 0.0001, + "loss": 79.1938, + "step": 3850 + }, + { + "epoch": 0.6233220968490745, + "grad_norm": 0.8979483842849731, + "learning_rate": 0.0001, + "loss": 79.1522, + "step": 3860 + }, + { + "epoch": 0.6249369209341757, + "grad_norm": 0.7759745717048645, + "learning_rate": 0.0001, + "loss": 77.7001, + "step": 3870 + }, + { + "epoch": 0.626551745019277, + "grad_norm": 0.9940371513366699, + "learning_rate": 0.0001, + "loss": 79.8712, + "step": 3880 + }, + { + "epoch": 0.6281665691043782, + "grad_norm": 0.8103354573249817, + "learning_rate": 0.0001, + "loss": 77.5229, + "step": 3890 + }, + { + "epoch": 0.6297813931894795, + "grad_norm": 0.9188411831855774, + "learning_rate": 0.0001, + "loss": 78.2933, + "step": 3900 + }, + { + "epoch": 0.6313962172745806, + "grad_norm": 0.823017954826355, + "learning_rate": 0.0001, + "loss": 77.8692, + "step": 3910 + }, + { + "epoch": 0.6330110413596819, + "grad_norm": 0.8833358287811279, + "learning_rate": 0.0001, + "loss": 78.4465, + "step": 3920 + }, + { + "epoch": 0.634625865444783, + "grad_norm": 0.6880685687065125, + "learning_rate": 0.0001, + "loss": 78.8881, + "step": 3930 + }, + { + "epoch": 0.6362406895298843, + "grad_norm": 0.778455913066864, + "learning_rate": 0.0001, + "loss": 76.0277, + "step": 3940 + }, + { + "epoch": 0.6378555136149856, + "grad_norm": 0.8132772445678711, + "learning_rate": 0.0001, + "loss": 79.1284, + "step": 3950 + }, + { + "epoch": 0.6394703377000868, + "grad_norm": 0.7703788876533508, + "learning_rate": 0.0001, + "loss": 76.9013, + "step": 3960 + }, + { + "epoch": 0.641085161785188, + "grad_norm": 0.7876151204109192, + "learning_rate": 0.0001, + "loss": 77.1354, + "step": 3970 + }, + { + "epoch": 0.6426999858702892, + "grad_norm": 0.8027064800262451, + "learning_rate": 0.0001, + "loss": 77.4936, + "step": 3980 + }, + { + "epoch": 0.6443148099553905, + "grad_norm": 0.8261818885803223, + "learning_rate": 0.0001, + "loss": 76.7634, + "step": 3990 + }, + { + "epoch": 0.6459296340404918, + "grad_norm": 0.7024852633476257, + "learning_rate": 0.0001, + "loss": 79.951, + "step": 4000 + }, + { + "epoch": 0.6475444581255929, + "grad_norm": 1.015307068824768, + "learning_rate": 0.0001, + "loss": 76.0354, + "step": 4010 + }, + { + "epoch": 0.6491592822106942, + "grad_norm": 0.715346097946167, + "learning_rate": 0.0001, + "loss": 78.9021, + "step": 4020 + }, + { + "epoch": 0.6507741062957954, + "grad_norm": 0.8994929790496826, + "learning_rate": 0.0001, + "loss": 77.0587, + "step": 4030 + }, + { + "epoch": 0.6523889303808966, + "grad_norm": 0.7683436274528503, + "learning_rate": 0.0001, + "loss": 78.6943, + "step": 4040 + }, + { + "epoch": 0.6540037544659979, + "grad_norm": 0.7273952960968018, + "learning_rate": 0.0001, + "loss": 77.8396, + "step": 4050 + }, + { + "epoch": 0.6556185785510991, + "grad_norm": 1.0676878690719604, + "learning_rate": 0.0001, + "loss": 79.3449, + "step": 4060 + }, + { + "epoch": 0.6572334026362003, + "grad_norm": 0.6916168928146362, + "learning_rate": 0.0001, + "loss": 76.5462, + "step": 4070 + }, + { + "epoch": 0.6588482267213015, + "grad_norm": 0.6466719508171082, + "learning_rate": 0.0001, + "loss": 77.9354, + "step": 4080 + }, + { + "epoch": 0.6604630508064028, + "grad_norm": 0.7406665086746216, + "learning_rate": 0.0001, + "loss": 76.0135, + "step": 4090 + }, + { + "epoch": 0.662077874891504, + "grad_norm": 0.7104353904724121, + "learning_rate": 0.0001, + "loss": 78.3727, + "step": 4100 + }, + { + "epoch": 0.6636926989766052, + "grad_norm": 0.8328452110290527, + "learning_rate": 0.0001, + "loss": 77.5354, + "step": 4110 + }, + { + "epoch": 0.6653075230617065, + "grad_norm": 0.7012260556221008, + "learning_rate": 0.0001, + "loss": 78.7406, + "step": 4120 + }, + { + "epoch": 0.6669223471468076, + "grad_norm": 1.0436632633209229, + "learning_rate": 0.0001, + "loss": 78.0063, + "step": 4130 + }, + { + "epoch": 0.6685371712319089, + "grad_norm": 0.8471246361732483, + "learning_rate": 0.0001, + "loss": 77.8871, + "step": 4140 + }, + { + "epoch": 0.6701519953170102, + "grad_norm": 0.7378994822502136, + "learning_rate": 0.0001, + "loss": 76.9109, + "step": 4150 + }, + { + "epoch": 0.6717668194021114, + "grad_norm": 0.7778745889663696, + "learning_rate": 0.0001, + "loss": 78.3374, + "step": 4160 + }, + { + "epoch": 0.6733816434872126, + "grad_norm": 0.7841216325759888, + "learning_rate": 0.0001, + "loss": 78.1657, + "step": 4170 + }, + { + "epoch": 0.6749964675723138, + "grad_norm": 1.0027995109558105, + "learning_rate": 0.0001, + "loss": 76.6232, + "step": 4180 + }, + { + "epoch": 0.6766112916574151, + "grad_norm": 1.0034840106964111, + "learning_rate": 0.0001, + "loss": 79.5785, + "step": 4190 + }, + { + "epoch": 0.6782261157425163, + "grad_norm": 0.8399350643157959, + "learning_rate": 0.0001, + "loss": 76.8101, + "step": 4200 + }, + { + "epoch": 0.6798409398276175, + "grad_norm": 0.8864424824714661, + "learning_rate": 0.0001, + "loss": 77.8671, + "step": 4210 + }, + { + "epoch": 0.6814557639127188, + "grad_norm": 0.8322188258171082, + "learning_rate": 0.0001, + "loss": 77.6597, + "step": 4220 + }, + { + "epoch": 0.68307058799782, + "grad_norm": 0.8962367177009583, + "learning_rate": 0.0001, + "loss": 77.7324, + "step": 4230 + }, + { + "epoch": 0.6846854120829212, + "grad_norm": 0.9727205038070679, + "learning_rate": 0.0001, + "loss": 77.8531, + "step": 4240 + }, + { + "epoch": 0.6863002361680225, + "grad_norm": 0.956584095954895, + "learning_rate": 0.0001, + "loss": 78.709, + "step": 4250 + }, + { + "epoch": 0.6879150602531237, + "grad_norm": 0.8531600832939148, + "learning_rate": 0.0001, + "loss": 78.7755, + "step": 4260 + }, + { + "epoch": 0.6895298843382249, + "grad_norm": 0.8059844374656677, + "learning_rate": 0.0001, + "loss": 77.1636, + "step": 4270 + }, + { + "epoch": 0.6911447084233261, + "grad_norm": 0.8373456597328186, + "learning_rate": 0.0001, + "loss": 78.6561, + "step": 4280 + }, + { + "epoch": 0.6927595325084274, + "grad_norm": 0.7819933891296387, + "learning_rate": 0.0001, + "loss": 77.6061, + "step": 4290 + }, + { + "epoch": 0.6943743565935286, + "grad_norm": 0.8829817771911621, + "learning_rate": 0.0001, + "loss": 77.3387, + "step": 4300 + }, + { + "epoch": 0.6959891806786298, + "grad_norm": 0.791772723197937, + "learning_rate": 0.0001, + "loss": 77.4368, + "step": 4310 + }, + { + "epoch": 0.6976040047637311, + "grad_norm": 0.9949231147766113, + "learning_rate": 0.0001, + "loss": 73.4359, + "step": 4320 + }, + { + "epoch": 0.6992188288488322, + "grad_norm": 0.8676188588142395, + "learning_rate": 0.0001, + "loss": 78.4836, + "step": 4330 + }, + { + "epoch": 0.7008336529339335, + "grad_norm": 1.1704007387161255, + "learning_rate": 0.0001, + "loss": 76.5091, + "step": 4340 + }, + { + "epoch": 0.7024484770190348, + "grad_norm": 0.8877903819084167, + "learning_rate": 0.0001, + "loss": 78.021, + "step": 4350 + }, + { + "epoch": 0.704063301104136, + "grad_norm": 1.381914496421814, + "learning_rate": 0.0001, + "loss": 77.8544, + "step": 4360 + }, + { + "epoch": 0.7056781251892372, + "grad_norm": 0.835023045539856, + "learning_rate": 0.0001, + "loss": 76.1552, + "step": 4370 + }, + { + "epoch": 0.7072929492743384, + "grad_norm": 0.9814149141311646, + "learning_rate": 0.0001, + "loss": 76.7646, + "step": 4380 + }, + { + "epoch": 0.7089077733594397, + "grad_norm": 0.8012816309928894, + "learning_rate": 0.0001, + "loss": 75.794, + "step": 4390 + }, + { + "epoch": 0.7105225974445409, + "grad_norm": 0.8340234160423279, + "learning_rate": 0.0001, + "loss": 77.4417, + "step": 4400 + }, + { + "epoch": 0.7121374215296421, + "grad_norm": 0.7257950305938721, + "learning_rate": 0.0001, + "loss": 77.1807, + "step": 4410 + }, + { + "epoch": 0.7137522456147434, + "grad_norm": 0.7342708706855774, + "learning_rate": 0.0001, + "loss": 75.5091, + "step": 4420 + }, + { + "epoch": 0.7153670696998445, + "grad_norm": 0.8298904895782471, + "learning_rate": 0.0001, + "loss": 76.6651, + "step": 4430 + }, + { + "epoch": 0.7169818937849458, + "grad_norm": 0.8564396500587463, + "learning_rate": 0.0001, + "loss": 76.0055, + "step": 4440 + }, + { + "epoch": 0.7185967178700471, + "grad_norm": 0.9145296812057495, + "learning_rate": 0.0001, + "loss": 76.0369, + "step": 4450 + }, + { + "epoch": 0.7202115419551482, + "grad_norm": 0.8074191212654114, + "learning_rate": 0.0001, + "loss": 77.4436, + "step": 4460 + }, + { + "epoch": 0.7218263660402495, + "grad_norm": 0.8212682008743286, + "learning_rate": 0.0001, + "loss": 76.3129, + "step": 4470 + }, + { + "epoch": 0.7234411901253507, + "grad_norm": 0.751323938369751, + "learning_rate": 0.0001, + "loss": 77.603, + "step": 4480 + }, + { + "epoch": 0.725056014210452, + "grad_norm": 0.9664170145988464, + "learning_rate": 0.0001, + "loss": 77.481, + "step": 4490 + }, + { + "epoch": 0.7266708382955531, + "grad_norm": 0.7369455695152283, + "learning_rate": 0.0001, + "loss": 76.3238, + "step": 4500 + }, + { + "epoch": 0.7282856623806544, + "grad_norm": 0.8448513746261597, + "learning_rate": 0.0001, + "loss": 76.5679, + "step": 4510 + }, + { + "epoch": 0.7299004864657557, + "grad_norm": 0.8605676293373108, + "learning_rate": 0.0001, + "loss": 75.7667, + "step": 4520 + }, + { + "epoch": 0.7315153105508568, + "grad_norm": 0.8034664392471313, + "learning_rate": 0.0001, + "loss": 76.2163, + "step": 4530 + }, + { + "epoch": 0.7331301346359581, + "grad_norm": 0.7846826910972595, + "learning_rate": 0.0001, + "loss": 73.5979, + "step": 4540 + }, + { + "epoch": 0.7347449587210593, + "grad_norm": 0.7422606348991394, + "learning_rate": 0.0001, + "loss": 76.2334, + "step": 4550 + }, + { + "epoch": 0.7363597828061605, + "grad_norm": 1.4478483200073242, + "learning_rate": 0.0001, + "loss": 77.6168, + "step": 4560 + }, + { + "epoch": 0.7379746068912618, + "grad_norm": 2.0957767963409424, + "learning_rate": 0.0001, + "loss": 76.3199, + "step": 4570 + }, + { + "epoch": 0.739589430976363, + "grad_norm": 0.6757177114486694, + "learning_rate": 0.0001, + "loss": 77.2923, + "step": 4580 + }, + { + "epoch": 0.7412042550614643, + "grad_norm": 0.9281920790672302, + "learning_rate": 0.0001, + "loss": 76.9963, + "step": 4590 + }, + { + "epoch": 0.7428190791465654, + "grad_norm": 0.8702669143676758, + "learning_rate": 0.0001, + "loss": 76.2553, + "step": 4600 + }, + { + "epoch": 0.7444339032316667, + "grad_norm": 0.7976512312889099, + "learning_rate": 0.0001, + "loss": 74.7622, + "step": 4610 + }, + { + "epoch": 0.746048727316768, + "grad_norm": 0.9116189479827881, + "learning_rate": 0.0001, + "loss": 76.3198, + "step": 4620 + }, + { + "epoch": 0.7476635514018691, + "grad_norm": 0.6239303946495056, + "learning_rate": 0.0001, + "loss": 75.7678, + "step": 4630 + }, + { + "epoch": 0.7492783754869704, + "grad_norm": 0.8380976915359497, + "learning_rate": 0.0001, + "loss": 76.8305, + "step": 4640 + }, + { + "epoch": 0.7508931995720716, + "grad_norm": 0.7394141554832458, + "learning_rate": 0.0001, + "loss": 76.5033, + "step": 4650 + }, + { + "epoch": 0.7525080236571728, + "grad_norm": 1.0694036483764648, + "learning_rate": 0.0001, + "loss": 75.0084, + "step": 4660 + }, + { + "epoch": 0.7541228477422741, + "grad_norm": 0.7530115246772766, + "learning_rate": 0.0001, + "loss": 74.4977, + "step": 4670 + }, + { + "epoch": 0.7557376718273753, + "grad_norm": 1.9561909437179565, + "learning_rate": 0.0001, + "loss": 74.7077, + "step": 4680 + }, + { + "epoch": 0.7573524959124766, + "grad_norm": 0.625341534614563, + "learning_rate": 0.0001, + "loss": 76.1238, + "step": 4690 + }, + { + "epoch": 0.7589673199975777, + "grad_norm": 0.9790500998497009, + "learning_rate": 0.0001, + "loss": 76.3997, + "step": 4700 + }, + { + "epoch": 0.760582144082679, + "grad_norm": 0.7087915539741516, + "learning_rate": 0.0001, + "loss": 75.6893, + "step": 4710 + }, + { + "epoch": 0.7621969681677803, + "grad_norm": 1.0995875597000122, + "learning_rate": 0.0001, + "loss": 74.8588, + "step": 4720 + }, + { + "epoch": 0.7638117922528814, + "grad_norm": 1.0309056043624878, + "learning_rate": 0.0001, + "loss": 73.5934, + "step": 4730 + }, + { + "epoch": 0.7654266163379827, + "grad_norm": 1.0132921934127808, + "learning_rate": 0.0001, + "loss": 73.9868, + "step": 4740 + }, + { + "epoch": 0.7670414404230839, + "grad_norm": 0.7138039469718933, + "learning_rate": 0.0001, + "loss": 76.4459, + "step": 4750 + }, + { + "epoch": 0.7686562645081851, + "grad_norm": 0.7055657505989075, + "learning_rate": 0.0001, + "loss": 75.1068, + "step": 4760 + }, + { + "epoch": 0.7702710885932864, + "grad_norm": 0.7398490905761719, + "learning_rate": 0.0001, + "loss": 75.3697, + "step": 4770 + }, + { + "epoch": 0.7718859126783876, + "grad_norm": 0.6836295127868652, + "learning_rate": 0.0001, + "loss": 77.4988, + "step": 4780 + }, + { + "epoch": 0.7735007367634888, + "grad_norm": 0.7552574276924133, + "learning_rate": 0.0001, + "loss": 75.149, + "step": 4790 + }, + { + "epoch": 0.77511556084859, + "grad_norm": 0.7930851578712463, + "learning_rate": 0.0001, + "loss": 73.8661, + "step": 4800 + }, + { + "epoch": 0.7767303849336913, + "grad_norm": 0.7549886107444763, + "learning_rate": 0.0001, + "loss": 75.042, + "step": 4810 + }, + { + "epoch": 0.7783452090187926, + "grad_norm": 0.9932214617729187, + "learning_rate": 0.0001, + "loss": 75.6619, + "step": 4820 + }, + { + "epoch": 0.7799600331038937, + "grad_norm": 0.7685471773147583, + "learning_rate": 0.0001, + "loss": 75.8404, + "step": 4830 + }, + { + "epoch": 0.781574857188995, + "grad_norm": 0.828737199306488, + "learning_rate": 0.0001, + "loss": 75.5839, + "step": 4840 + }, + { + "epoch": 0.7831896812740962, + "grad_norm": 0.8575420379638672, + "learning_rate": 0.0001, + "loss": 75.7889, + "step": 4850 + }, + { + "epoch": 0.7848045053591974, + "grad_norm": 0.7053834199905396, + "learning_rate": 0.0001, + "loss": 75.9372, + "step": 4860 + }, + { + "epoch": 0.7864193294442987, + "grad_norm": 0.7861505150794983, + "learning_rate": 0.0001, + "loss": 76.1532, + "step": 4870 + }, + { + "epoch": 0.7880341535293999, + "grad_norm": 0.8357386589050293, + "learning_rate": 0.0001, + "loss": 75.8536, + "step": 4880 + }, + { + "epoch": 0.7896489776145011, + "grad_norm": 3.448834180831909, + "learning_rate": 0.0001, + "loss": 74.1794, + "step": 4890 + }, + { + "epoch": 0.7912638016996023, + "grad_norm": 0.813522458076477, + "learning_rate": 0.0001, + "loss": 74.5202, + "step": 4900 + }, + { + "epoch": 0.7928786257847036, + "grad_norm": 0.840230405330658, + "learning_rate": 0.0001, + "loss": 74.6634, + "step": 4910 + }, + { + "epoch": 0.7944934498698049, + "grad_norm": 0.7855745553970337, + "learning_rate": 0.0001, + "loss": 73.5369, + "step": 4920 + }, + { + "epoch": 0.796108273954906, + "grad_norm": 0.7708212733268738, + "learning_rate": 0.0001, + "loss": 74.6225, + "step": 4930 + }, + { + "epoch": 0.7977230980400073, + "grad_norm": 0.7140830159187317, + "learning_rate": 0.0001, + "loss": 75.0585, + "step": 4940 + }, + { + "epoch": 0.7993379221251085, + "grad_norm": 0.7960936427116394, + "learning_rate": 0.0001, + "loss": 74.0574, + "step": 4950 + }, + { + "epoch": 0.8009527462102097, + "grad_norm": 0.7269165515899658, + "learning_rate": 0.0001, + "loss": 74.0379, + "step": 4960 + }, + { + "epoch": 0.802567570295311, + "grad_norm": 0.8505013585090637, + "learning_rate": 0.0001, + "loss": 74.7448, + "step": 4970 + }, + { + "epoch": 0.8041823943804122, + "grad_norm": 0.8935654759407043, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 4980 + }, + { + "epoch": 0.8057972184655134, + "grad_norm": 0.9447595477104187, + "learning_rate": 0.0001, + "loss": 75.9674, + "step": 4990 + }, + { + "epoch": 0.8074120425506146, + "grad_norm": 1.6739715337753296, + "learning_rate": 0.0001, + "loss": 73.9035, + "step": 5000 + }, + { + "epoch": 0.8090268666357159, + "grad_norm": 0.861902117729187, + "learning_rate": 0.0001, + "loss": 74.906, + "step": 5010 + }, + { + "epoch": 0.8106416907208172, + "grad_norm": 0.8328655362129211, + "learning_rate": 0.0001, + "loss": 74.1493, + "step": 5020 + }, + { + "epoch": 0.8122565148059183, + "grad_norm": 0.8193135261535645, + "learning_rate": 0.0001, + "loss": 74.6614, + "step": 5030 + }, + { + "epoch": 0.8138713388910196, + "grad_norm": 0.6954565048217773, + "learning_rate": 0.0001, + "loss": 73.7366, + "step": 5040 + }, + { + "epoch": 0.8154861629761208, + "grad_norm": 0.7381289005279541, + "learning_rate": 0.0001, + "loss": 73.5086, + "step": 5050 + }, + { + "epoch": 0.817100987061222, + "grad_norm": 1.090443730354309, + "learning_rate": 0.0001, + "loss": 75.0695, + "step": 5060 + }, + { + "epoch": 0.8187158111463233, + "grad_norm": 1.2544814348220825, + "learning_rate": 0.0001, + "loss": 72.89, + "step": 5070 + }, + { + "epoch": 0.8203306352314245, + "grad_norm": 0.632146418094635, + "learning_rate": 0.0001, + "loss": 74.8604, + "step": 5080 + }, + { + "epoch": 0.8219454593165257, + "grad_norm": 1.1038259267807007, + "learning_rate": 0.0001, + "loss": 74.7354, + "step": 5090 + }, + { + "epoch": 0.8235602834016269, + "grad_norm": 0.6886508464813232, + "learning_rate": 0.0001, + "loss": 75.4243, + "step": 5100 + }, + { + "epoch": 0.8251751074867282, + "grad_norm": 0.8367568850517273, + "learning_rate": 0.0001, + "loss": 75.2732, + "step": 5110 + }, + { + "epoch": 0.8267899315718293, + "grad_norm": 0.9302715063095093, + "learning_rate": 0.0001, + "loss": 75.7924, + "step": 5120 + }, + { + "epoch": 0.8284047556569306, + "grad_norm": 0.6871219873428345, + "learning_rate": 0.0001, + "loss": 74.2905, + "step": 5130 + }, + { + "epoch": 0.8300195797420319, + "grad_norm": 0.7319445013999939, + "learning_rate": 0.0001, + "loss": 73.7774, + "step": 5140 + }, + { + "epoch": 0.831634403827133, + "grad_norm": 0.7184522151947021, + "learning_rate": 0.0001, + "loss": 73.0554, + "step": 5150 + }, + { + "epoch": 0.8332492279122343, + "grad_norm": 0.8462343811988831, + "learning_rate": 0.0001, + "loss": 73.7615, + "step": 5160 + }, + { + "epoch": 0.8348640519973355, + "grad_norm": 0.9436241984367371, + "learning_rate": 0.0001, + "loss": 73.9464, + "step": 5170 + }, + { + "epoch": 0.8364788760824368, + "grad_norm": 0.9079630374908447, + "learning_rate": 0.0001, + "loss": 73.2727, + "step": 5180 + }, + { + "epoch": 0.838093700167538, + "grad_norm": 0.8280741572380066, + "learning_rate": 0.0001, + "loss": 73.9751, + "step": 5190 + }, + { + "epoch": 0.8397085242526392, + "grad_norm": 0.6475157141685486, + "learning_rate": 0.0001, + "loss": 74.7105, + "step": 5200 + }, + { + "epoch": 0.8413233483377405, + "grad_norm": 0.7090312838554382, + "learning_rate": 0.0001, + "loss": 73.6489, + "step": 5210 + }, + { + "epoch": 0.8429381724228416, + "grad_norm": 1.20967698097229, + "learning_rate": 0.0001, + "loss": 75.1335, + "step": 5220 + }, + { + "epoch": 0.8445529965079429, + "grad_norm": 2.053532600402832, + "learning_rate": 0.0001, + "loss": 75.3111, + "step": 5230 + }, + { + "epoch": 0.8461678205930442, + "grad_norm": 0.7563302516937256, + "learning_rate": 0.0001, + "loss": 72.9935, + "step": 5240 + }, + { + "epoch": 0.8477826446781453, + "grad_norm": 0.7801489233970642, + "learning_rate": 0.0001, + "loss": 74.7856, + "step": 5250 + }, + { + "epoch": 0.8493974687632466, + "grad_norm": 0.737870454788208, + "learning_rate": 0.0001, + "loss": 75.6281, + "step": 5260 + }, + { + "epoch": 0.8510122928483478, + "grad_norm": 0.704352617263794, + "learning_rate": 0.0001, + "loss": 75.4598, + "step": 5270 + }, + { + "epoch": 0.8526271169334491, + "grad_norm": 0.6452057361602783, + "learning_rate": 0.0001, + "loss": 74.0341, + "step": 5280 + }, + { + "epoch": 0.8542419410185503, + "grad_norm": 0.6863471865653992, + "learning_rate": 0.0001, + "loss": 74.3727, + "step": 5290 + }, + { + "epoch": 0.8558567651036515, + "grad_norm": 9.756170272827148, + "learning_rate": 0.0001, + "loss": 75.5231, + "step": 5300 + }, + { + "epoch": 0.8574715891887528, + "grad_norm": 0.8624544739723206, + "learning_rate": 0.0001, + "loss": 74.7821, + "step": 5310 + }, + { + "epoch": 0.8590864132738539, + "grad_norm": 0.6637155413627625, + "learning_rate": 0.0001, + "loss": 74.1153, + "step": 5320 + }, + { + "epoch": 0.8607012373589552, + "grad_norm": 1.0764445066452026, + "learning_rate": 0.0001, + "loss": 73.7208, + "step": 5330 + }, + { + "epoch": 0.8623160614440565, + "grad_norm": 0.8567569255828857, + "learning_rate": 0.0001, + "loss": 73.1399, + "step": 5340 + }, + { + "epoch": 0.8639308855291576, + "grad_norm": 0.7681576013565063, + "learning_rate": 0.0001, + "loss": 73.7577, + "step": 5350 + }, + { + "epoch": 0.8655457096142589, + "grad_norm": 0.9336384534835815, + "learning_rate": 0.0001, + "loss": 74.1022, + "step": 5360 + }, + { + "epoch": 0.8671605336993601, + "grad_norm": 0.8603724837303162, + "learning_rate": 0.0001, + "loss": 75.9579, + "step": 5370 + }, + { + "epoch": 0.8687753577844614, + "grad_norm": 0.77194744348526, + "learning_rate": 0.0001, + "loss": 72.4799, + "step": 5380 + }, + { + "epoch": 0.8703901818695626, + "grad_norm": 0.7480376958847046, + "learning_rate": 0.0001, + "loss": 76.1257, + "step": 5390 + }, + { + "epoch": 0.8720050059546638, + "grad_norm": 0.7841023206710815, + "learning_rate": 0.0001, + "loss": 73.1512, + "step": 5400 + }, + { + "epoch": 0.8736198300397651, + "grad_norm": 0.8741798996925354, + "learning_rate": 0.0001, + "loss": 74.2151, + "step": 5410 + }, + { + "epoch": 0.8752346541248662, + "grad_norm": 0.9762082099914551, + "learning_rate": 0.0001, + "loss": 76.7138, + "step": 5420 + }, + { + "epoch": 0.8768494782099675, + "grad_norm": 0.7670348286628723, + "learning_rate": 0.0001, + "loss": 72.9567, + "step": 5430 + }, + { + "epoch": 0.8784643022950688, + "grad_norm": 0.7449469566345215, + "learning_rate": 0.0001, + "loss": 74.8841, + "step": 5440 + }, + { + "epoch": 0.8800791263801699, + "grad_norm": 0.6984641551971436, + "learning_rate": 0.0001, + "loss": 73.8906, + "step": 5450 + }, + { + "epoch": 0.8816939504652712, + "grad_norm": 0.7854361534118652, + "learning_rate": 0.0001, + "loss": 74.626, + "step": 5460 + }, + { + "epoch": 0.8833087745503724, + "grad_norm": 0.9584240317344666, + "learning_rate": 0.0001, + "loss": 74.7048, + "step": 5470 + }, + { + "epoch": 0.8849235986354737, + "grad_norm": 0.6040284037590027, + "learning_rate": 0.0001, + "loss": 74.7194, + "step": 5480 + }, + { + "epoch": 0.8865384227205749, + "grad_norm": 0.7756215929985046, + "learning_rate": 0.0001, + "loss": 73.8178, + "step": 5490 + }, + { + "epoch": 0.8881532468056761, + "grad_norm": 0.9070586562156677, + "learning_rate": 0.0001, + "loss": 72.8309, + "step": 5500 + }, + { + "epoch": 0.8897680708907774, + "grad_norm": 0.8464986681938171, + "learning_rate": 0.0001, + "loss": 73.8847, + "step": 5510 + }, + { + "epoch": 0.8913828949758785, + "grad_norm": 0.7968605160713196, + "learning_rate": 0.0001, + "loss": 73.1691, + "step": 5520 + }, + { + "epoch": 0.8929977190609798, + "grad_norm": 0.9934583306312561, + "learning_rate": 0.0001, + "loss": 76.2662, + "step": 5530 + }, + { + "epoch": 0.8946125431460811, + "grad_norm": 0.7272152900695801, + "learning_rate": 0.0001, + "loss": 75.5295, + "step": 5540 + }, + { + "epoch": 0.8962273672311822, + "grad_norm": 0.857345461845398, + "learning_rate": 0.0001, + "loss": 75.3703, + "step": 5550 + }, + { + "epoch": 0.8978421913162835, + "grad_norm": 0.713918924331665, + "learning_rate": 0.0001, + "loss": 73.4563, + "step": 5560 + }, + { + "epoch": 0.8994570154013847, + "grad_norm": 0.759233832359314, + "learning_rate": 0.0001, + "loss": 73.2744, + "step": 5570 + }, + { + "epoch": 0.901071839486486, + "grad_norm": 0.7085754871368408, + "learning_rate": 0.0001, + "loss": 74.7792, + "step": 5580 + }, + { + "epoch": 0.9026866635715872, + "grad_norm": 0.7370456457138062, + "learning_rate": 0.0001, + "loss": 74.4216, + "step": 5590 + }, + { + "epoch": 0.9043014876566884, + "grad_norm": 0.779746949672699, + "learning_rate": 0.0001, + "loss": 75.9822, + "step": 5600 + }, + { + "epoch": 0.9059163117417897, + "grad_norm": 0.7915763854980469, + "learning_rate": 0.0001, + "loss": 74.3706, + "step": 5610 + }, + { + "epoch": 0.9075311358268908, + "grad_norm": 0.7886125445365906, + "learning_rate": 0.0001, + "loss": 72.9453, + "step": 5620 + }, + { + "epoch": 0.9091459599119921, + "grad_norm": 0.7804751992225647, + "learning_rate": 0.0001, + "loss": 73.514, + "step": 5630 + }, + { + "epoch": 0.9107607839970934, + "grad_norm": 0.7168806195259094, + "learning_rate": 0.0001, + "loss": 71.4159, + "step": 5640 + }, + { + "epoch": 0.9123756080821945, + "grad_norm": 0.7553626894950867, + "learning_rate": 0.0001, + "loss": 74.4353, + "step": 5650 + }, + { + "epoch": 0.9139904321672958, + "grad_norm": 0.6688916683197021, + "learning_rate": 0.0001, + "loss": 74.3786, + "step": 5660 + }, + { + "epoch": 0.915605256252397, + "grad_norm": 0.7425532341003418, + "learning_rate": 0.0001, + "loss": 73.3355, + "step": 5670 + }, + { + "epoch": 0.9172200803374982, + "grad_norm": 0.6473882794380188, + "learning_rate": 0.0001, + "loss": 74.2315, + "step": 5680 + }, + { + "epoch": 0.9188349044225994, + "grad_norm": 0.7222439050674438, + "learning_rate": 0.0001, + "loss": 74.2885, + "step": 5690 + }, + { + "epoch": 0.9204497285077007, + "grad_norm": 1.5716948509216309, + "learning_rate": 0.0001, + "loss": 74.6739, + "step": 5700 + }, + { + "epoch": 0.922064552592802, + "grad_norm": 0.8077759742736816, + "learning_rate": 0.0001, + "loss": 72.6617, + "step": 5710 + }, + { + "epoch": 0.9236793766779031, + "grad_norm": 0.6948631405830383, + "learning_rate": 0.0001, + "loss": 73.6247, + "step": 5720 + }, + { + "epoch": 0.9252942007630044, + "grad_norm": 0.8823757767677307, + "learning_rate": 0.0001, + "loss": 73.2536, + "step": 5730 + }, + { + "epoch": 0.9269090248481056, + "grad_norm": 0.7066764235496521, + "learning_rate": 0.0001, + "loss": 74.1748, + "step": 5740 + }, + { + "epoch": 0.9285238489332068, + "grad_norm": 0.6979793906211853, + "learning_rate": 0.0001, + "loss": 72.3189, + "step": 5750 + }, + { + "epoch": 0.9301386730183081, + "grad_norm": 0.6961462497711182, + "learning_rate": 0.0001, + "loss": 74.6196, + "step": 5760 + }, + { + "epoch": 0.9317534971034093, + "grad_norm": 0.7797656655311584, + "learning_rate": 0.0001, + "loss": 74.5472, + "step": 5770 + }, + { + "epoch": 0.9333683211885105, + "grad_norm": 1.0671838521957397, + "learning_rate": 0.0001, + "loss": 73.241, + "step": 5780 + }, + { + "epoch": 0.9349831452736117, + "grad_norm": 0.7566089034080505, + "learning_rate": 0.0001, + "loss": 73.6784, + "step": 5790 + }, + { + "epoch": 0.936597969358713, + "grad_norm": 0.8563273549079895, + "learning_rate": 0.0001, + "loss": 73.6871, + "step": 5800 + }, + { + "epoch": 0.9382127934438143, + "grad_norm": 0.7885987162590027, + "learning_rate": 0.0001, + "loss": 73.4447, + "step": 5810 + }, + { + "epoch": 0.9398276175289154, + "grad_norm": 0.8751112818717957, + "learning_rate": 0.0001, + "loss": 74.0531, + "step": 5820 + }, + { + "epoch": 0.9414424416140167, + "grad_norm": 0.6663336157798767, + "learning_rate": 0.0001, + "loss": 73.0557, + "step": 5830 + }, + { + "epoch": 0.9430572656991179, + "grad_norm": 0.7993099689483643, + "learning_rate": 0.0001, + "loss": 74.4467, + "step": 5840 + }, + { + "epoch": 0.9446720897842191, + "grad_norm": 0.6987188458442688, + "learning_rate": 0.0001, + "loss": 74.0327, + "step": 5850 + }, + { + "epoch": 0.9462869138693204, + "grad_norm": 0.8458583354949951, + "learning_rate": 0.0001, + "loss": 73.1757, + "step": 5860 + }, + { + "epoch": 0.9479017379544216, + "grad_norm": 0.6372436881065369, + "learning_rate": 0.0001, + "loss": 73.1067, + "step": 5870 + }, + { + "epoch": 0.9495165620395228, + "grad_norm": 0.7447753548622131, + "learning_rate": 0.0001, + "loss": 75.5461, + "step": 5880 + }, + { + "epoch": 0.951131386124624, + "grad_norm": 0.6633971929550171, + "learning_rate": 0.0001, + "loss": 72.9966, + "step": 5890 + }, + { + "epoch": 0.9527462102097253, + "grad_norm": 0.8576784729957581, + "learning_rate": 0.0001, + "loss": 71.5712, + "step": 5900 + }, + { + "epoch": 0.9543610342948265, + "grad_norm": 1.4696853160858154, + "learning_rate": 0.0001, + "loss": 74.7893, + "step": 5910 + }, + { + "epoch": 0.9559758583799277, + "grad_norm": 0.611210286617279, + "learning_rate": 0.0001, + "loss": 74.3426, + "step": 5920 + }, + { + "epoch": 0.957590682465029, + "grad_norm": 0.5947410464286804, + "learning_rate": 0.0001, + "loss": 71.2563, + "step": 5930 + }, + { + "epoch": 0.9592055065501301, + "grad_norm": 0.7169992923736572, + "learning_rate": 0.0001, + "loss": 71.6049, + "step": 5940 + }, + { + "epoch": 0.9608203306352314, + "grad_norm": 0.6773818731307983, + "learning_rate": 0.0001, + "loss": 75.0328, + "step": 5950 + }, + { + "epoch": 0.9624351547203327, + "grad_norm": 0.6714807748794556, + "learning_rate": 0.0001, + "loss": 71.5105, + "step": 5960 + }, + { + "epoch": 0.9640499788054339, + "grad_norm": 1.0064337253570557, + "learning_rate": 0.0001, + "loss": 75.8887, + "step": 5970 + }, + { + "epoch": 0.9656648028905351, + "grad_norm": 0.9802227020263672, + "learning_rate": 0.0001, + "loss": 73.5655, + "step": 5980 + }, + { + "epoch": 0.9672796269756363, + "grad_norm": 0.7097306251525879, + "learning_rate": 0.0001, + "loss": 73.298, + "step": 5990 + }, + { + "epoch": 0.9688944510607376, + "grad_norm": 0.6680361032485962, + "learning_rate": 0.0001, + "loss": 73.8873, + "step": 6000 + }, + { + "epoch": 0.9705092751458388, + "grad_norm": 1.1302506923675537, + "learning_rate": 0.0001, + "loss": 73.3328, + "step": 6010 + }, + { + "epoch": 0.97212409923094, + "grad_norm": 0.7163954377174377, + "learning_rate": 0.0001, + "loss": 70.5955, + "step": 6020 + }, + { + "epoch": 0.9737389233160413, + "grad_norm": 0.8182240128517151, + "learning_rate": 0.0001, + "loss": 72.6397, + "step": 6030 + }, + { + "epoch": 0.9753537474011424, + "grad_norm": 0.6752052903175354, + "learning_rate": 0.0001, + "loss": 73.1188, + "step": 6040 + }, + { + "epoch": 0.9769685714862437, + "grad_norm": 0.8829680681228638, + "learning_rate": 0.0001, + "loss": 75.7346, + "step": 6050 + }, + { + "epoch": 0.978583395571345, + "grad_norm": 0.9254767894744873, + "learning_rate": 0.0001, + "loss": 73.6221, + "step": 6060 + }, + { + "epoch": 0.9801982196564462, + "grad_norm": 0.7527495622634888, + "learning_rate": 0.0001, + "loss": 73.948, + "step": 6070 + }, + { + "epoch": 0.9818130437415474, + "grad_norm": 1.0135066509246826, + "learning_rate": 0.0001, + "loss": 72.8929, + "step": 6080 + }, + { + "epoch": 0.9834278678266486, + "grad_norm": 0.7093394994735718, + "learning_rate": 0.0001, + "loss": 73.5826, + "step": 6090 + }, + { + "epoch": 0.9850426919117499, + "grad_norm": 0.6651231646537781, + "learning_rate": 0.0001, + "loss": 73.167, + "step": 6100 + }, + { + "epoch": 0.9866575159968511, + "grad_norm": 0.720552921295166, + "learning_rate": 0.0001, + "loss": 74.8985, + "step": 6110 + }, + { + "epoch": 0.9882723400819523, + "grad_norm": 1.7821632623672485, + "learning_rate": 0.0001, + "loss": 74.1414, + "step": 6120 + }, + { + "epoch": 0.9898871641670536, + "grad_norm": 0.6405150890350342, + "learning_rate": 0.0001, + "loss": 73.5989, + "step": 6130 + }, + { + "epoch": 0.9915019882521547, + "grad_norm": 0.8207482695579529, + "learning_rate": 0.0001, + "loss": 73.0977, + "step": 6140 + }, + { + "epoch": 0.993116812337256, + "grad_norm": 0.7320757508277893, + "learning_rate": 0.0001, + "loss": 72.0241, + "step": 6150 + }, + { + "epoch": 0.9947316364223573, + "grad_norm": 0.9522950649261475, + "learning_rate": 0.0001, + "loss": 73.9831, + "step": 6160 + }, + { + "epoch": 0.9963464605074585, + "grad_norm": 0.7437159419059753, + "learning_rate": 0.0001, + "loss": 72.1054, + "step": 6170 + }, + { + "epoch": 0.9979612845925597, + "grad_norm": 0.7035623788833618, + "learning_rate": 0.0001, + "loss": 73.959, + "step": 6180 + }, + { + "epoch": 0.9995761086776609, + "grad_norm": 0.7559277415275574, + "learning_rate": 0.0001, + "loss": 71.4651, + "step": 6190 + }, + { + "epoch": 1.001130376859571, + "grad_norm": 0.852342426776886, + "learning_rate": 0.0001, + "loss": 71.8844, + "step": 6200 + }, + { + "epoch": 1.002745200944672, + "grad_norm": 0.8018911480903625, + "learning_rate": 0.0001, + "loss": 74.5093, + "step": 6210 + }, + { + "epoch": 1.0043600250297733, + "grad_norm": 0.6613730192184448, + "learning_rate": 0.0001, + "loss": 71.9953, + "step": 6220 + }, + { + "epoch": 1.0059748491148746, + "grad_norm": 0.9815977215766907, + "learning_rate": 0.0001, + "loss": 70.8632, + "step": 6230 + }, + { + "epoch": 1.0075896731999758, + "grad_norm": 0.781581699848175, + "learning_rate": 0.0001, + "loss": 72.3603, + "step": 6240 + }, + { + "epoch": 1.009204497285077, + "grad_norm": 0.8079054355621338, + "learning_rate": 0.0001, + "loss": 73.6153, + "step": 6250 + }, + { + "epoch": 1.0108193213701782, + "grad_norm": 0.8356053233146667, + "learning_rate": 0.0001, + "loss": 73.1562, + "step": 6260 + }, + { + "epoch": 1.0124341454552794, + "grad_norm": 0.7107778787612915, + "learning_rate": 0.0001, + "loss": 74.5276, + "step": 6270 + }, + { + "epoch": 1.0140489695403807, + "grad_norm": 0.8601720929145813, + "learning_rate": 0.0001, + "loss": 70.103, + "step": 6280 + }, + { + "epoch": 1.015663793625482, + "grad_norm": 0.675837516784668, + "learning_rate": 0.0001, + "loss": 72.6102, + "step": 6290 + }, + { + "epoch": 1.0172786177105833, + "grad_norm": 0.7160016298294067, + "learning_rate": 0.0001, + "loss": 72.2404, + "step": 6300 + }, + { + "epoch": 1.0188934417956843, + "grad_norm": 0.7286407351493835, + "learning_rate": 0.0001, + "loss": 72.7149, + "step": 6310 + }, + { + "epoch": 1.0205082658807856, + "grad_norm": 0.8183980584144592, + "learning_rate": 0.0001, + "loss": 73.5827, + "step": 6320 + }, + { + "epoch": 1.0221230899658869, + "grad_norm": 0.7300626039505005, + "learning_rate": 0.0001, + "loss": 72.3459, + "step": 6330 + }, + { + "epoch": 1.0237379140509881, + "grad_norm": 0.7222613096237183, + "learning_rate": 0.0001, + "loss": 71.881, + "step": 6340 + }, + { + "epoch": 1.0253527381360894, + "grad_norm": 0.7324674725532532, + "learning_rate": 0.0001, + "loss": 74.4781, + "step": 6350 + }, + { + "epoch": 1.0269675622211905, + "grad_norm": 0.9173555374145508, + "learning_rate": 0.0001, + "loss": 74.3222, + "step": 6360 + }, + { + "epoch": 1.0285823863062917, + "grad_norm": 0.8404964208602905, + "learning_rate": 0.0001, + "loss": 71.8959, + "step": 6370 + }, + { + "epoch": 1.030197210391393, + "grad_norm": 0.8735223412513733, + "learning_rate": 0.0001, + "loss": 72.7251, + "step": 6380 + }, + { + "epoch": 1.0318120344764943, + "grad_norm": 0.8467233777046204, + "learning_rate": 0.0001, + "loss": 72.5218, + "step": 6390 + }, + { + "epoch": 1.0334268585615956, + "grad_norm": 0.673565149307251, + "learning_rate": 0.0001, + "loss": 70.6392, + "step": 6400 + }, + { + "epoch": 1.0350416826466966, + "grad_norm": 0.6558571457862854, + "learning_rate": 0.0001, + "loss": 73.5305, + "step": 6410 + }, + { + "epoch": 1.0366565067317979, + "grad_norm": 0.7251600623130798, + "learning_rate": 0.0001, + "loss": 72.76, + "step": 6420 + }, + { + "epoch": 1.0382713308168992, + "grad_norm": 0.7458305358886719, + "learning_rate": 0.0001, + "loss": 73.6812, + "step": 6430 + }, + { + "epoch": 1.0398861549020004, + "grad_norm": 1.493802547454834, + "learning_rate": 0.0001, + "loss": 74.2455, + "step": 6440 + }, + { + "epoch": 1.0415009789871017, + "grad_norm": 0.894632875919342, + "learning_rate": 0.0001, + "loss": 74.2072, + "step": 6450 + }, + { + "epoch": 1.0431158030722028, + "grad_norm": 0.7142218947410583, + "learning_rate": 0.0001, + "loss": 71.4293, + "step": 6460 + }, + { + "epoch": 1.044730627157304, + "grad_norm": 0.702639639377594, + "learning_rate": 0.0001, + "loss": 71.5383, + "step": 6470 + }, + { + "epoch": 1.0463454512424053, + "grad_norm": 0.9378212094306946, + "learning_rate": 0.0001, + "loss": 72.8916, + "step": 6480 + }, + { + "epoch": 1.0479602753275066, + "grad_norm": 0.8298363089561462, + "learning_rate": 0.0001, + "loss": 72.3344, + "step": 6490 + }, + { + "epoch": 1.0495750994126078, + "grad_norm": 0.6617016792297363, + "learning_rate": 0.0001, + "loss": 71.0487, + "step": 6500 + }, + { + "epoch": 1.051189923497709, + "grad_norm": 0.638520359992981, + "learning_rate": 0.0001, + "loss": 72.2583, + "step": 6510 + }, + { + "epoch": 1.0528047475828102, + "grad_norm": 0.7101534008979797, + "learning_rate": 0.0001, + "loss": 70.8697, + "step": 6520 + }, + { + "epoch": 1.0544195716679114, + "grad_norm": 0.9425339698791504, + "learning_rate": 0.0001, + "loss": 71.0641, + "step": 6530 + }, + { + "epoch": 1.0560343957530127, + "grad_norm": 0.6572297811508179, + "learning_rate": 0.0001, + "loss": 72.6265, + "step": 6540 + }, + { + "epoch": 1.057649219838114, + "grad_norm": 0.745622456073761, + "learning_rate": 0.0001, + "loss": 73.0587, + "step": 6550 + }, + { + "epoch": 1.059264043923215, + "grad_norm": 0.7667096257209778, + "learning_rate": 0.0001, + "loss": 72.1779, + "step": 6560 + }, + { + "epoch": 1.0608788680083163, + "grad_norm": 0.7322410941123962, + "learning_rate": 0.0001, + "loss": 71.5503, + "step": 6570 + }, + { + "epoch": 1.0624936920934176, + "grad_norm": 0.5739616751670837, + "learning_rate": 0.0001, + "loss": 72.3846, + "step": 6580 + }, + { + "epoch": 1.0641085161785189, + "grad_norm": 0.7149553894996643, + "learning_rate": 0.0001, + "loss": 71.5459, + "step": 6590 + }, + { + "epoch": 1.0657233402636201, + "grad_norm": 0.6634597778320312, + "learning_rate": 0.0001, + "loss": 72.2528, + "step": 6600 + }, + { + "epoch": 1.0673381643487212, + "grad_norm": 0.7208803296089172, + "learning_rate": 0.0001, + "loss": 71.1115, + "step": 6610 + }, + { + "epoch": 1.0689529884338225, + "grad_norm": 0.9845868945121765, + "learning_rate": 0.0001, + "loss": 72.6418, + "step": 6620 + }, + { + "epoch": 1.0705678125189237, + "grad_norm": 0.9322167634963989, + "learning_rate": 0.0001, + "loss": 71.937, + "step": 6630 + }, + { + "epoch": 1.072182636604025, + "grad_norm": 0.6323093771934509, + "learning_rate": 0.0001, + "loss": 71.1537, + "step": 6640 + }, + { + "epoch": 1.0737974606891263, + "grad_norm": 0.6476802229881287, + "learning_rate": 0.0001, + "loss": 73.7811, + "step": 6650 + }, + { + "epoch": 1.0754122847742273, + "grad_norm": 0.7882822751998901, + "learning_rate": 0.0001, + "loss": 71.9548, + "step": 6660 + }, + { + "epoch": 1.0770271088593286, + "grad_norm": 0.7651168704032898, + "learning_rate": 0.0001, + "loss": 73.4066, + "step": 6670 + }, + { + "epoch": 1.07864193294443, + "grad_norm": 0.7536904215812683, + "learning_rate": 0.0001, + "loss": 72.1063, + "step": 6680 + }, + { + "epoch": 1.0802567570295312, + "grad_norm": 0.7908427119255066, + "learning_rate": 0.0001, + "loss": 72.6374, + "step": 6690 + }, + { + "epoch": 1.0818715811146324, + "grad_norm": 0.7796691060066223, + "learning_rate": 0.0001, + "loss": 72.2938, + "step": 6700 + }, + { + "epoch": 1.0834864051997335, + "grad_norm": 0.8512274622917175, + "learning_rate": 0.0001, + "loss": 71.556, + "step": 6710 + }, + { + "epoch": 1.0851012292848348, + "grad_norm": 0.7285851836204529, + "learning_rate": 0.0001, + "loss": 72.406, + "step": 6720 + }, + { + "epoch": 1.086716053369936, + "grad_norm": 0.6613008975982666, + "learning_rate": 0.0001, + "loss": 70.6509, + "step": 6730 + }, + { + "epoch": 1.0883308774550373, + "grad_norm": 0.7158271074295044, + "learning_rate": 0.0001, + "loss": 71.8434, + "step": 6740 + }, + { + "epoch": 1.0899457015401384, + "grad_norm": 0.8635967373847961, + "learning_rate": 0.0001, + "loss": 71.8279, + "step": 6750 + }, + { + "epoch": 1.0915605256252396, + "grad_norm": 0.7020159959793091, + "learning_rate": 0.0001, + "loss": 72.2417, + "step": 6760 + }, + { + "epoch": 1.093175349710341, + "grad_norm": 0.6389853954315186, + "learning_rate": 0.0001, + "loss": 72.1879, + "step": 6770 + }, + { + "epoch": 1.0947901737954422, + "grad_norm": 0.6460697650909424, + "learning_rate": 0.0001, + "loss": 72.3356, + "step": 6780 + }, + { + "epoch": 1.0964049978805435, + "grad_norm": 0.7152006030082703, + "learning_rate": 0.0001, + "loss": 71.3725, + "step": 6790 + }, + { + "epoch": 1.0980198219656445, + "grad_norm": 1.0050997734069824, + "learning_rate": 0.0001, + "loss": 72.5452, + "step": 6800 + }, + { + "epoch": 1.0996346460507458, + "grad_norm": 0.6967180371284485, + "learning_rate": 0.0001, + "loss": 72.0832, + "step": 6810 + }, + { + "epoch": 1.101249470135847, + "grad_norm": 0.7472837567329407, + "learning_rate": 0.0001, + "loss": 72.8122, + "step": 6820 + }, + { + "epoch": 1.1028642942209483, + "grad_norm": 0.8198382258415222, + "learning_rate": 0.0001, + "loss": 72.5116, + "step": 6830 + }, + { + "epoch": 1.1044791183060496, + "grad_norm": 0.8112207651138306, + "learning_rate": 0.0001, + "loss": 72.4218, + "step": 6840 + }, + { + "epoch": 1.1060939423911507, + "grad_norm": 0.8412114977836609, + "learning_rate": 0.0001, + "loss": 71.0198, + "step": 6850 + }, + { + "epoch": 1.107708766476252, + "grad_norm": 0.6730719804763794, + "learning_rate": 0.0001, + "loss": 71.2045, + "step": 6860 + }, + { + "epoch": 1.1093235905613532, + "grad_norm": 0.6895949244499207, + "learning_rate": 0.0001, + "loss": 72.3056, + "step": 6870 + }, + { + "epoch": 1.1109384146464545, + "grad_norm": 2.476609945297241, + "learning_rate": 0.0001, + "loss": 70.3865, + "step": 6880 + }, + { + "epoch": 1.1125532387315558, + "grad_norm": 0.7285001873970032, + "learning_rate": 0.0001, + "loss": 71.5761, + "step": 6890 + }, + { + "epoch": 1.1141680628166568, + "grad_norm": 0.7546679973602295, + "learning_rate": 0.0001, + "loss": 72.5979, + "step": 6900 + }, + { + "epoch": 1.115782886901758, + "grad_norm": 0.8974841237068176, + "learning_rate": 0.0001, + "loss": 74.5888, + "step": 6910 + }, + { + "epoch": 1.1173977109868594, + "grad_norm": 0.8785758018493652, + "learning_rate": 0.0001, + "loss": 71.662, + "step": 6920 + }, + { + "epoch": 1.1190125350719606, + "grad_norm": 0.6805328130722046, + "learning_rate": 0.0001, + "loss": 72.201, + "step": 6930 + }, + { + "epoch": 1.120627359157062, + "grad_norm": 0.8132315278053284, + "learning_rate": 0.0001, + "loss": 72.7473, + "step": 6940 + }, + { + "epoch": 1.122242183242163, + "grad_norm": 1.1625338792800903, + "learning_rate": 0.0001, + "loss": 70.5987, + "step": 6950 + }, + { + "epoch": 1.1238570073272642, + "grad_norm": 1.0984737873077393, + "learning_rate": 0.0001, + "loss": 72.7335, + "step": 6960 + }, + { + "epoch": 1.1254718314123655, + "grad_norm": 0.6798779368400574, + "learning_rate": 0.0001, + "loss": 72.1942, + "step": 6970 + }, + { + "epoch": 1.1270866554974668, + "grad_norm": 0.7546010613441467, + "learning_rate": 0.0001, + "loss": 70.8014, + "step": 6980 + }, + { + "epoch": 1.128701479582568, + "grad_norm": 1.0238053798675537, + "learning_rate": 0.0001, + "loss": 72.5617, + "step": 6990 + }, + { + "epoch": 1.130316303667669, + "grad_norm": 0.6535419821739197, + "learning_rate": 0.0001, + "loss": 70.2193, + "step": 7000 + }, + { + "epoch": 1.1319311277527704, + "grad_norm": 0.7354230284690857, + "learning_rate": 0.0001, + "loss": 72.0685, + "step": 7010 + }, + { + "epoch": 1.1335459518378717, + "grad_norm": 0.7848799824714661, + "learning_rate": 0.0001, + "loss": 72.2869, + "step": 7020 + }, + { + "epoch": 1.135160775922973, + "grad_norm": 0.6519262790679932, + "learning_rate": 0.0001, + "loss": 72.2023, + "step": 7030 + }, + { + "epoch": 1.1367756000080742, + "grad_norm": 0.8428511023521423, + "learning_rate": 0.0001, + "loss": 71.3396, + "step": 7040 + }, + { + "epoch": 1.1383904240931753, + "grad_norm": 0.8873165249824524, + "learning_rate": 0.0001, + "loss": 69.8284, + "step": 7050 + }, + { + "epoch": 1.1400052481782765, + "grad_norm": 0.9141374230384827, + "learning_rate": 0.0001, + "loss": 70.3666, + "step": 7060 + }, + { + "epoch": 1.1416200722633778, + "grad_norm": 0.6556552648544312, + "learning_rate": 0.0001, + "loss": 71.1316, + "step": 7070 + }, + { + "epoch": 1.143234896348479, + "grad_norm": 0.6736201047897339, + "learning_rate": 0.0001, + "loss": 70.2601, + "step": 7080 + }, + { + "epoch": 1.1448497204335804, + "grad_norm": 0.6965599656105042, + "learning_rate": 0.0001, + "loss": 71.9409, + "step": 7090 + }, + { + "epoch": 1.1464645445186814, + "grad_norm": 1.0600581169128418, + "learning_rate": 0.0001, + "loss": 72.7344, + "step": 7100 + }, + { + "epoch": 1.1480793686037827, + "grad_norm": 0.7501764297485352, + "learning_rate": 0.0001, + "loss": 72.3129, + "step": 7110 + }, + { + "epoch": 1.149694192688884, + "grad_norm": 0.7105675935745239, + "learning_rate": 0.0001, + "loss": 71.1356, + "step": 7120 + }, + { + "epoch": 1.1513090167739852, + "grad_norm": 0.7446026802062988, + "learning_rate": 0.0001, + "loss": 69.8208, + "step": 7130 + }, + { + "epoch": 1.1529238408590865, + "grad_norm": 0.6664865016937256, + "learning_rate": 0.0001, + "loss": 72.2466, + "step": 7140 + }, + { + "epoch": 1.1545386649441876, + "grad_norm": 1.7590374946594238, + "learning_rate": 0.0001, + "loss": 71.4987, + "step": 7150 + }, + { + "epoch": 1.1561534890292888, + "grad_norm": 0.7941271066665649, + "learning_rate": 0.0001, + "loss": 70.787, + "step": 7160 + }, + { + "epoch": 1.15776831311439, + "grad_norm": 0.7473573088645935, + "learning_rate": 0.0001, + "loss": 71.2516, + "step": 7170 + }, + { + "epoch": 1.1593831371994914, + "grad_norm": 0.7589637041091919, + "learning_rate": 0.0001, + "loss": 71.4781, + "step": 7180 + }, + { + "epoch": 1.1609979612845926, + "grad_norm": 0.8148510456085205, + "learning_rate": 0.0001, + "loss": 71.1491, + "step": 7190 + }, + { + "epoch": 1.1626127853696937, + "grad_norm": 0.6803365349769592, + "learning_rate": 0.0001, + "loss": 71.7937, + "step": 7200 + }, + { + "epoch": 1.164227609454795, + "grad_norm": 0.7314755320549011, + "learning_rate": 0.0001, + "loss": 70.6566, + "step": 7210 + }, + { + "epoch": 1.1658424335398963, + "grad_norm": 0.6494631171226501, + "learning_rate": 0.0001, + "loss": 70.5057, + "step": 7220 + }, + { + "epoch": 1.1674572576249975, + "grad_norm": 0.6076239943504333, + "learning_rate": 0.0001, + "loss": 72.2985, + "step": 7230 + }, + { + "epoch": 1.1690720817100988, + "grad_norm": 0.6650206446647644, + "learning_rate": 0.0001, + "loss": 71.0719, + "step": 7240 + }, + { + "epoch": 1.1706869057951999, + "grad_norm": 0.6926266551017761, + "learning_rate": 0.0001, + "loss": 70.3714, + "step": 7250 + }, + { + "epoch": 1.1723017298803011, + "grad_norm": 0.6534716486930847, + "learning_rate": 0.0001, + "loss": 71.3194, + "step": 7260 + }, + { + "epoch": 1.1739165539654024, + "grad_norm": 0.7194528579711914, + "learning_rate": 0.0001, + "loss": 72.3277, + "step": 7270 + }, + { + "epoch": 1.1755313780505037, + "grad_norm": 0.7496437430381775, + "learning_rate": 0.0001, + "loss": 71.1136, + "step": 7280 + }, + { + "epoch": 1.177146202135605, + "grad_norm": 0.690522313117981, + "learning_rate": 0.0001, + "loss": 70.065, + "step": 7290 + }, + { + "epoch": 1.178761026220706, + "grad_norm": 0.6966914534568787, + "learning_rate": 0.0001, + "loss": 73.1137, + "step": 7300 + }, + { + "epoch": 1.1803758503058073, + "grad_norm": 0.7304641604423523, + "learning_rate": 0.0001, + "loss": 71.2466, + "step": 7310 + }, + { + "epoch": 1.1819906743909085, + "grad_norm": 0.7564008235931396, + "learning_rate": 0.0001, + "loss": 71.4147, + "step": 7320 + }, + { + "epoch": 1.1836054984760098, + "grad_norm": 0.61886066198349, + "learning_rate": 0.0001, + "loss": 70.3297, + "step": 7330 + }, + { + "epoch": 1.185220322561111, + "grad_norm": 0.6281574964523315, + "learning_rate": 0.0001, + "loss": 70.9607, + "step": 7340 + }, + { + "epoch": 1.1868351466462121, + "grad_norm": 0.6583316326141357, + "learning_rate": 0.0001, + "loss": 70.3443, + "step": 7350 + }, + { + "epoch": 1.1884499707313134, + "grad_norm": 1.8159942626953125, + "learning_rate": 0.0001, + "loss": 71.4792, + "step": 7360 + }, + { + "epoch": 1.1900647948164147, + "grad_norm": 0.7253140807151794, + "learning_rate": 0.0001, + "loss": 72.2182, + "step": 7370 + }, + { + "epoch": 1.191679618901516, + "grad_norm": 0.9141016602516174, + "learning_rate": 0.0001, + "loss": 68.7443, + "step": 7380 + }, + { + "epoch": 1.193294442986617, + "grad_norm": 0.7392430901527405, + "learning_rate": 0.0001, + "loss": 69.1888, + "step": 7390 + }, + { + "epoch": 1.1949092670717183, + "grad_norm": 0.7713924646377563, + "learning_rate": 0.0001, + "loss": 69.1301, + "step": 7400 + }, + { + "epoch": 1.1965240911568196, + "grad_norm": 0.6000798344612122, + "learning_rate": 0.0001, + "loss": 71.2576, + "step": 7410 + }, + { + "epoch": 1.1981389152419208, + "grad_norm": 0.687111496925354, + "learning_rate": 0.0001, + "loss": 70.2919, + "step": 7420 + }, + { + "epoch": 1.1997537393270221, + "grad_norm": 1.1718226671218872, + "learning_rate": 0.0001, + "loss": 71.9498, + "step": 7430 + }, + { + "epoch": 1.2013685634121232, + "grad_norm": 0.7176038026809692, + "learning_rate": 0.0001, + "loss": 71.91, + "step": 7440 + }, + { + "epoch": 1.2029833874972244, + "grad_norm": 0.7705256938934326, + "learning_rate": 0.0001, + "loss": 70.1241, + "step": 7450 + }, + { + "epoch": 1.2045982115823257, + "grad_norm": 0.8004609942436218, + "learning_rate": 0.0001, + "loss": 69.5121, + "step": 7460 + }, + { + "epoch": 1.206213035667427, + "grad_norm": 0.6708329319953918, + "learning_rate": 0.0001, + "loss": 73.0173, + "step": 7470 + }, + { + "epoch": 1.2078278597525283, + "grad_norm": 0.8777983784675598, + "learning_rate": 0.0001, + "loss": 71.6843, + "step": 7480 + }, + { + "epoch": 1.2094426838376293, + "grad_norm": 0.7683864831924438, + "learning_rate": 0.0001, + "loss": 71.6914, + "step": 7490 + }, + { + "epoch": 1.2110575079227306, + "grad_norm": 0.8022210597991943, + "learning_rate": 0.0001, + "loss": 69.1724, + "step": 7500 + }, + { + "epoch": 1.2126723320078319, + "grad_norm": 0.7250640392303467, + "learning_rate": 0.0001, + "loss": 70.6568, + "step": 7510 + }, + { + "epoch": 1.2142871560929331, + "grad_norm": 0.5625214576721191, + "learning_rate": 0.0001, + "loss": 69.4849, + "step": 7520 + }, + { + "epoch": 1.2159019801780344, + "grad_norm": 0.7980468273162842, + "learning_rate": 0.0001, + "loss": 70.8164, + "step": 7530 + }, + { + "epoch": 1.2175168042631355, + "grad_norm": 0.6900951862335205, + "learning_rate": 0.0001, + "loss": 71.1719, + "step": 7540 + }, + { + "epoch": 1.2191316283482367, + "grad_norm": 0.6284971833229065, + "learning_rate": 0.0001, + "loss": 70.1751, + "step": 7550 + }, + { + "epoch": 1.220746452433338, + "grad_norm": 0.6641798615455627, + "learning_rate": 0.0001, + "loss": 71.9043, + "step": 7560 + }, + { + "epoch": 1.2223612765184393, + "grad_norm": 0.7439492344856262, + "learning_rate": 0.0001, + "loss": 70.7286, + "step": 7570 + }, + { + "epoch": 1.2239761006035406, + "grad_norm": 0.8018355369567871, + "learning_rate": 0.0001, + "loss": 71.638, + "step": 7580 + }, + { + "epoch": 1.2255909246886416, + "grad_norm": 0.7318522930145264, + "learning_rate": 0.0001, + "loss": 70.0867, + "step": 7590 + }, + { + "epoch": 1.2272057487737429, + "grad_norm": 0.6799285411834717, + "learning_rate": 0.0001, + "loss": 71.3427, + "step": 7600 + }, + { + "epoch": 1.2288205728588442, + "grad_norm": 0.6796864867210388, + "learning_rate": 0.0001, + "loss": 71.1698, + "step": 7610 + }, + { + "epoch": 1.2304353969439454, + "grad_norm": 0.7256141304969788, + "learning_rate": 0.0001, + "loss": 71.1051, + "step": 7620 + }, + { + "epoch": 1.2320502210290467, + "grad_norm": 0.8318199515342712, + "learning_rate": 0.0001, + "loss": 71.3179, + "step": 7630 + }, + { + "epoch": 1.2336650451141478, + "grad_norm": 0.6594699025154114, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 7640 + }, + { + "epoch": 1.235279869199249, + "grad_norm": 0.7900195717811584, + "learning_rate": 0.0001, + "loss": 72.0139, + "step": 7650 + }, + { + "epoch": 1.2368946932843503, + "grad_norm": 0.7845672965049744, + "learning_rate": 0.0001, + "loss": 70.808, + "step": 7660 + }, + { + "epoch": 1.2385095173694516, + "grad_norm": 0.6598045229911804, + "learning_rate": 0.0001, + "loss": 69.8953, + "step": 7670 + }, + { + "epoch": 1.2401243414545529, + "grad_norm": 1.3222978115081787, + "learning_rate": 0.0001, + "loss": 68.787, + "step": 7680 + }, + { + "epoch": 1.241739165539654, + "grad_norm": 0.6660508513450623, + "learning_rate": 0.0001, + "loss": 71.5831, + "step": 7690 + }, + { + "epoch": 1.2433539896247552, + "grad_norm": 0.6305698156356812, + "learning_rate": 0.0001, + "loss": 70.3232, + "step": 7700 + }, + { + "epoch": 1.2449688137098565, + "grad_norm": 1.2648323774337769, + "learning_rate": 0.0001, + "loss": 67.8901, + "step": 7710 + }, + { + "epoch": 1.2465836377949577, + "grad_norm": 0.7177619934082031, + "learning_rate": 0.0001, + "loss": 69.6582, + "step": 7720 + }, + { + "epoch": 1.248198461880059, + "grad_norm": 0.6716091632843018, + "learning_rate": 0.0001, + "loss": 70.2034, + "step": 7730 + }, + { + "epoch": 1.24981328596516, + "grad_norm": 0.6050158143043518, + "learning_rate": 0.0001, + "loss": 69.3942, + "step": 7740 + }, + { + "epoch": 1.2514281100502613, + "grad_norm": 0.7001014947891235, + "learning_rate": 0.0001, + "loss": 69.5971, + "step": 7750 + }, + { + "epoch": 1.2530429341353626, + "grad_norm": 0.7221280932426453, + "learning_rate": 0.0001, + "loss": 71.3651, + "step": 7760 + }, + { + "epoch": 1.2546577582204639, + "grad_norm": 0.9130340218544006, + "learning_rate": 0.0001, + "loss": 68.3038, + "step": 7770 + }, + { + "epoch": 1.2562725823055652, + "grad_norm": 0.7703199982643127, + "learning_rate": 0.0001, + "loss": 71.6127, + "step": 7780 + }, + { + "epoch": 1.2578874063906662, + "grad_norm": 0.6334695816040039, + "learning_rate": 0.0001, + "loss": 68.9074, + "step": 7790 + }, + { + "epoch": 1.2595022304757675, + "grad_norm": 0.7074242234230042, + "learning_rate": 0.0001, + "loss": 70.0569, + "step": 7800 + }, + { + "epoch": 1.2611170545608688, + "grad_norm": 0.6956025958061218, + "learning_rate": 0.0001, + "loss": 69.892, + "step": 7810 + }, + { + "epoch": 1.26273187864597, + "grad_norm": 0.7235708236694336, + "learning_rate": 0.0001, + "loss": 71.1777, + "step": 7820 + }, + { + "epoch": 1.2643467027310713, + "grad_norm": 0.6157007813453674, + "learning_rate": 0.0001, + "loss": 71.5091, + "step": 7830 + }, + { + "epoch": 1.2659615268161724, + "grad_norm": 0.5352533459663391, + "learning_rate": 0.0001, + "loss": 70.083, + "step": 7840 + }, + { + "epoch": 1.2675763509012736, + "grad_norm": 0.7171182632446289, + "learning_rate": 0.0001, + "loss": 68.4585, + "step": 7850 + }, + { + "epoch": 1.269191174986375, + "grad_norm": 0.7503113746643066, + "learning_rate": 0.0001, + "loss": 70.4206, + "step": 7860 + }, + { + "epoch": 1.2708059990714762, + "grad_norm": 0.6552693843841553, + "learning_rate": 0.0001, + "loss": 70.8455, + "step": 7870 + }, + { + "epoch": 1.2724208231565775, + "grad_norm": 0.7173179984092712, + "learning_rate": 0.0001, + "loss": 68.82, + "step": 7880 + }, + { + "epoch": 1.2740356472416785, + "grad_norm": 0.7352942824363708, + "learning_rate": 0.0001, + "loss": 70.4974, + "step": 7890 + }, + { + "epoch": 1.2756504713267798, + "grad_norm": 0.6120197176933289, + "learning_rate": 0.0001, + "loss": 72.1638, + "step": 7900 + }, + { + "epoch": 1.277265295411881, + "grad_norm": 0.6271925568580627, + "learning_rate": 0.0001, + "loss": 69.3267, + "step": 7910 + }, + { + "epoch": 1.2788801194969823, + "grad_norm": 0.752636194229126, + "learning_rate": 0.0001, + "loss": 70.6761, + "step": 7920 + }, + { + "epoch": 1.2804949435820836, + "grad_norm": 0.7307374477386475, + "learning_rate": 0.0001, + "loss": 70.2909, + "step": 7930 + }, + { + "epoch": 1.2821097676671847, + "grad_norm": 0.5977495908737183, + "learning_rate": 0.0001, + "loss": 72.2969, + "step": 7940 + }, + { + "epoch": 1.283724591752286, + "grad_norm": 0.7927457094192505, + "learning_rate": 0.0001, + "loss": 70.3784, + "step": 7950 + }, + { + "epoch": 1.2853394158373872, + "grad_norm": 0.7520087957382202, + "learning_rate": 0.0001, + "loss": 70.7122, + "step": 7960 + }, + { + "epoch": 1.2869542399224885, + "grad_norm": 0.6036237478256226, + "learning_rate": 0.0001, + "loss": 69.8718, + "step": 7970 + }, + { + "epoch": 1.2885690640075897, + "grad_norm": 0.6805875897407532, + "learning_rate": 0.0001, + "loss": 70.2391, + "step": 7980 + }, + { + "epoch": 1.2901838880926908, + "grad_norm": 0.774559497833252, + "learning_rate": 0.0001, + "loss": 71.1605, + "step": 7990 + }, + { + "epoch": 1.291798712177792, + "grad_norm": 0.6991063356399536, + "learning_rate": 0.0001, + "loss": 69.9779, + "step": 8000 + }, + { + "epoch": 1.2934135362628933, + "grad_norm": 0.5877949595451355, + "learning_rate": 0.0001, + "loss": 71.16, + "step": 8010 + }, + { + "epoch": 1.2950283603479946, + "grad_norm": 0.8840436339378357, + "learning_rate": 0.0001, + "loss": 70.9454, + "step": 8020 + }, + { + "epoch": 1.296643184433096, + "grad_norm": 0.7532126307487488, + "learning_rate": 0.0001, + "loss": 70.141, + "step": 8030 + }, + { + "epoch": 1.298258008518197, + "grad_norm": 0.6880044341087341, + "learning_rate": 0.0001, + "loss": 70.0746, + "step": 8040 + }, + { + "epoch": 1.2998728326032982, + "grad_norm": 0.6319366097450256, + "learning_rate": 0.0001, + "loss": 69.9396, + "step": 8050 + }, + { + "epoch": 1.3014876566883995, + "grad_norm": 0.6329445838928223, + "learning_rate": 0.0001, + "loss": 69.9043, + "step": 8060 + }, + { + "epoch": 1.3031024807735008, + "grad_norm": 0.8892332315444946, + "learning_rate": 0.0001, + "loss": 69.4187, + "step": 8070 + }, + { + "epoch": 1.304717304858602, + "grad_norm": 0.696500301361084, + "learning_rate": 0.0001, + "loss": 68.1885, + "step": 8080 + }, + { + "epoch": 1.306332128943703, + "grad_norm": 0.5933213233947754, + "learning_rate": 0.0001, + "loss": 68.8017, + "step": 8090 + }, + { + "epoch": 1.3079469530288044, + "grad_norm": 0.9274749159812927, + "learning_rate": 0.0001, + "loss": 71.5875, + "step": 8100 + }, + { + "epoch": 1.3095617771139056, + "grad_norm": 0.7540446519851685, + "learning_rate": 0.0001, + "loss": 71.1233, + "step": 8110 + }, + { + "epoch": 1.311176601199007, + "grad_norm": 0.6701658964157104, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8120 + }, + { + "epoch": 1.3127914252841082, + "grad_norm": 0.6096424460411072, + "learning_rate": 0.0001, + "loss": 69.9583, + "step": 8130 + }, + { + "epoch": 1.3144062493692092, + "grad_norm": 0.809524416923523, + "learning_rate": 0.0001, + "loss": 71.8149, + "step": 8140 + }, + { + "epoch": 1.3160210734543105, + "grad_norm": 0.7928242683410645, + "learning_rate": 0.0001, + "loss": 70.2084, + "step": 8150 + }, + { + "epoch": 1.3176358975394118, + "grad_norm": 0.7462850213050842, + "learning_rate": 0.0001, + "loss": 71.4789, + "step": 8160 + }, + { + "epoch": 1.319250721624513, + "grad_norm": 0.8166030049324036, + "learning_rate": 0.0001, + "loss": 69.3154, + "step": 8170 + }, + { + "epoch": 1.3208655457096143, + "grad_norm": 0.7002414464950562, + "learning_rate": 0.0001, + "loss": 70.8048, + "step": 8180 + }, + { + "epoch": 1.3224803697947154, + "grad_norm": 0.7653809189796448, + "learning_rate": 0.0001, + "loss": 70.0221, + "step": 8190 + }, + { + "epoch": 1.3240951938798167, + "grad_norm": 0.6777089238166809, + "learning_rate": 0.0001, + "loss": 70.614, + "step": 8200 + }, + { + "epoch": 1.325710017964918, + "grad_norm": 0.692739725112915, + "learning_rate": 0.0001, + "loss": 71.2311, + "step": 8210 + }, + { + "epoch": 1.3273248420500192, + "grad_norm": 0.7985519766807556, + "learning_rate": 0.0001, + "loss": 68.1982, + "step": 8220 + }, + { + "epoch": 1.3289396661351205, + "grad_norm": 0.7002689242362976, + "learning_rate": 0.0001, + "loss": 70.9181, + "step": 8230 + }, + { + "epoch": 1.3305544902202215, + "grad_norm": 0.7029191255569458, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 8240 + }, + { + "epoch": 1.3321693143053228, + "grad_norm": 0.7292912602424622, + "learning_rate": 0.0001, + "loss": 71.5386, + "step": 8250 + }, + { + "epoch": 1.333784138390424, + "grad_norm": 0.6141605973243713, + "learning_rate": 0.0001, + "loss": 72.3396, + "step": 8260 + }, + { + "epoch": 1.3353989624755254, + "grad_norm": 0.6486957669258118, + "learning_rate": 0.0001, + "loss": 69.7607, + "step": 8270 + }, + { + "epoch": 1.3370137865606266, + "grad_norm": 0.7813800573348999, + "learning_rate": 0.0001, + "loss": 68.8979, + "step": 8280 + }, + { + "epoch": 1.3386286106457277, + "grad_norm": 0.6805126070976257, + "learning_rate": 0.0001, + "loss": 70.2179, + "step": 8290 + }, + { + "epoch": 1.340243434730829, + "grad_norm": 0.7202772498130798, + "learning_rate": 0.0001, + "loss": 72.5008, + "step": 8300 + }, + { + "epoch": 1.3418582588159302, + "grad_norm": 0.6881810426712036, + "learning_rate": 0.0001, + "loss": 70.1987, + "step": 8310 + }, + { + "epoch": 1.3434730829010315, + "grad_norm": 0.6337536573410034, + "learning_rate": 0.0001, + "loss": 70.4073, + "step": 8320 + }, + { + "epoch": 1.3450879069861328, + "grad_norm": 0.7313241958618164, + "learning_rate": 0.0001, + "loss": 69.3598, + "step": 8330 + }, + { + "epoch": 1.3467027310712338, + "grad_norm": 0.635947585105896, + "learning_rate": 0.0001, + "loss": 69.5143, + "step": 8340 + }, + { + "epoch": 1.3483175551563351, + "grad_norm": 0.6219074726104736, + "learning_rate": 0.0001, + "loss": 69.9903, + "step": 8350 + }, + { + "epoch": 1.3499323792414364, + "grad_norm": 0.5928048491477966, + "learning_rate": 0.0001, + "loss": 70.4222, + "step": 8360 + }, + { + "epoch": 1.3515472033265377, + "grad_norm": 0.5927143096923828, + "learning_rate": 0.0001, + "loss": 69.8642, + "step": 8370 + }, + { + "epoch": 1.353162027411639, + "grad_norm": 0.7614993453025818, + "learning_rate": 0.0001, + "loss": 69.5787, + "step": 8380 + }, + { + "epoch": 1.35477685149674, + "grad_norm": 0.5488380193710327, + "learning_rate": 0.0001, + "loss": 69.0028, + "step": 8390 + }, + { + "epoch": 1.3563916755818413, + "grad_norm": 0.6219164133071899, + "learning_rate": 0.0001, + "loss": 69.4591, + "step": 8400 + }, + { + "epoch": 1.3580064996669425, + "grad_norm": 0.6940358877182007, + "learning_rate": 0.0001, + "loss": 71.3795, + "step": 8410 + }, + { + "epoch": 1.3596213237520438, + "grad_norm": 0.5919192433357239, + "learning_rate": 0.0001, + "loss": 71.2761, + "step": 8420 + }, + { + "epoch": 1.361236147837145, + "grad_norm": 1.0803378820419312, + "learning_rate": 0.0001, + "loss": 69.0046, + "step": 8430 + }, + { + "epoch": 1.3628509719222461, + "grad_norm": 0.7028629183769226, + "learning_rate": 0.0001, + "loss": 71.5996, + "step": 8440 + }, + { + "epoch": 1.3644657960073474, + "grad_norm": 0.6829854249954224, + "learning_rate": 0.0001, + "loss": 67.8831, + "step": 8450 + }, + { + "epoch": 1.3660806200924487, + "grad_norm": 0.6937337517738342, + "learning_rate": 0.0001, + "loss": 69.8321, + "step": 8460 + }, + { + "epoch": 1.36769544417755, + "grad_norm": 0.7725892066955566, + "learning_rate": 0.0001, + "loss": 71.0784, + "step": 8470 + }, + { + "epoch": 1.3693102682626512, + "grad_norm": 0.7171871066093445, + "learning_rate": 0.0001, + "loss": 69.0828, + "step": 8480 + }, + { + "epoch": 1.3709250923477523, + "grad_norm": 0.6605113744735718, + "learning_rate": 0.0001, + "loss": 70.9461, + "step": 8490 + }, + { + "epoch": 1.3725399164328536, + "grad_norm": 0.9005604982376099, + "learning_rate": 0.0001, + "loss": 71.474, + "step": 8500 + }, + { + "epoch": 1.3741547405179548, + "grad_norm": 0.7454419732093811, + "learning_rate": 0.0001, + "loss": 70.3113, + "step": 8510 + }, + { + "epoch": 1.375769564603056, + "grad_norm": 0.7574180960655212, + "learning_rate": 0.0001, + "loss": 70.9015, + "step": 8520 + }, + { + "epoch": 1.3773843886881574, + "grad_norm": 0.6525025367736816, + "learning_rate": 0.0001, + "loss": 72.3132, + "step": 8530 + }, + { + "epoch": 1.3789992127732584, + "grad_norm": 0.6548861861228943, + "learning_rate": 0.0001, + "loss": 68.6495, + "step": 8540 + }, + { + "epoch": 1.3806140368583597, + "grad_norm": 0.7881495952606201, + "learning_rate": 0.0001, + "loss": 70.1335, + "step": 8550 + }, + { + "epoch": 1.382228860943461, + "grad_norm": 0.619380533695221, + "learning_rate": 0.0001, + "loss": 71.7469, + "step": 8560 + }, + { + "epoch": 1.3838436850285623, + "grad_norm": 0.7637182474136353, + "learning_rate": 0.0001, + "loss": 69.4691, + "step": 8570 + }, + { + "epoch": 1.3854585091136635, + "grad_norm": 0.8711991906166077, + "learning_rate": 0.0001, + "loss": 70.8131, + "step": 8580 + }, + { + "epoch": 1.3870733331987646, + "grad_norm": 0.6550050973892212, + "learning_rate": 0.0001, + "loss": 70.0096, + "step": 8590 + }, + { + "epoch": 1.3886881572838659, + "grad_norm": 0.6310496926307678, + "learning_rate": 0.0001, + "loss": 70.4305, + "step": 8600 + }, + { + "epoch": 1.3903029813689671, + "grad_norm": 0.5780142545700073, + "learning_rate": 0.0001, + "loss": 68.9639, + "step": 8610 + }, + { + "epoch": 1.3919178054540684, + "grad_norm": 0.6526662111282349, + "learning_rate": 0.0001, + "loss": 69.1345, + "step": 8620 + }, + { + "epoch": 1.3935326295391697, + "grad_norm": 1.2579848766326904, + "learning_rate": 0.0001, + "loss": 71.6908, + "step": 8630 + }, + { + "epoch": 1.3951474536242707, + "grad_norm": 0.7702625393867493, + "learning_rate": 0.0001, + "loss": 69.1064, + "step": 8640 + }, + { + "epoch": 1.396762277709372, + "grad_norm": 0.7414085865020752, + "learning_rate": 0.0001, + "loss": 69.5106, + "step": 8650 + }, + { + "epoch": 1.3983771017944733, + "grad_norm": 0.666235089302063, + "learning_rate": 0.0001, + "loss": 67.938, + "step": 8660 + }, + { + "epoch": 1.3999919258795746, + "grad_norm": 0.7879519462585449, + "learning_rate": 0.0001, + "loss": 69.5422, + "step": 8670 + }, + { + "epoch": 1.4016067499646758, + "grad_norm": 0.6552250981330872, + "learning_rate": 0.0001, + "loss": 69.4161, + "step": 8680 + }, + { + "epoch": 1.4032215740497769, + "grad_norm": 0.7558363676071167, + "learning_rate": 0.0001, + "loss": 71.3052, + "step": 8690 + }, + { + "epoch": 1.4048363981348782, + "grad_norm": 0.677007794380188, + "learning_rate": 0.0001, + "loss": 70.0824, + "step": 8700 + }, + { + "epoch": 1.4064512222199794, + "grad_norm": 0.6416738629341125, + "learning_rate": 0.0001, + "loss": 68.3357, + "step": 8710 + }, + { + "epoch": 1.4080660463050807, + "grad_norm": 0.8067912459373474, + "learning_rate": 0.0001, + "loss": 69.8551, + "step": 8720 + }, + { + "epoch": 1.409680870390182, + "grad_norm": 0.6017255783081055, + "learning_rate": 0.0001, + "loss": 70.3936, + "step": 8730 + }, + { + "epoch": 1.411295694475283, + "grad_norm": 0.5473611354827881, + "learning_rate": 0.0001, + "loss": 69.1556, + "step": 8740 + }, + { + "epoch": 1.4129105185603843, + "grad_norm": 0.641984760761261, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 8750 + }, + { + "epoch": 1.4145253426454856, + "grad_norm": 0.7509932518005371, + "learning_rate": 0.0001, + "loss": 68.793, + "step": 8760 + }, + { + "epoch": 1.4161401667305868, + "grad_norm": 0.615459144115448, + "learning_rate": 0.0001, + "loss": 69.0787, + "step": 8770 + }, + { + "epoch": 1.4177549908156881, + "grad_norm": 0.625276505947113, + "learning_rate": 0.0001, + "loss": 68.5643, + "step": 8780 + }, + { + "epoch": 1.4193698149007892, + "grad_norm": 0.7294536828994751, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 8790 + }, + { + "epoch": 1.4209846389858904, + "grad_norm": 0.8188890814781189, + "learning_rate": 0.0001, + "loss": 70.7486, + "step": 8800 + }, + { + "epoch": 1.4225994630709917, + "grad_norm": 0.8662137389183044, + "learning_rate": 0.0001, + "loss": 69.0625, + "step": 8810 + }, + { + "epoch": 1.424214287156093, + "grad_norm": 0.7548071146011353, + "learning_rate": 0.0001, + "loss": 68.9828, + "step": 8820 + }, + { + "epoch": 1.4258291112411943, + "grad_norm": 0.7295185327529907, + "learning_rate": 0.0001, + "loss": 69.0606, + "step": 8830 + }, + { + "epoch": 1.4274439353262953, + "grad_norm": 0.7937467098236084, + "learning_rate": 0.0001, + "loss": 69.0135, + "step": 8840 + }, + { + "epoch": 1.4290587594113966, + "grad_norm": 0.5957842469215393, + "learning_rate": 0.0001, + "loss": 68.3992, + "step": 8850 + }, + { + "epoch": 1.4306735834964979, + "grad_norm": 0.786580502986908, + "learning_rate": 0.0001, + "loss": 69.4485, + "step": 8860 + }, + { + "epoch": 1.4322884075815991, + "grad_norm": 0.8539725542068481, + "learning_rate": 0.0001, + "loss": 67.1037, + "step": 8870 + }, + { + "epoch": 1.4339032316667004, + "grad_norm": 0.684476375579834, + "learning_rate": 0.0001, + "loss": 68.9814, + "step": 8880 + }, + { + "epoch": 1.4355180557518015, + "grad_norm": 0.6335768103599548, + "learning_rate": 0.0001, + "loss": 69.9716, + "step": 8890 + }, + { + "epoch": 1.4371328798369027, + "grad_norm": 0.9881808757781982, + "learning_rate": 0.0001, + "loss": 68.0338, + "step": 8900 + }, + { + "epoch": 1.438747703922004, + "grad_norm": 0.783983588218689, + "learning_rate": 0.0001, + "loss": 70.124, + "step": 8910 + }, + { + "epoch": 1.4403625280071053, + "grad_norm": 0.817820131778717, + "learning_rate": 0.0001, + "loss": 67.933, + "step": 8920 + }, + { + "epoch": 1.4419773520922066, + "grad_norm": 0.6900199055671692, + "learning_rate": 0.0001, + "loss": 69.8871, + "step": 8930 + }, + { + "epoch": 1.4435921761773076, + "grad_norm": 0.6702716946601868, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8940 + }, + { + "epoch": 1.445207000262409, + "grad_norm": 0.6699396967887878, + "learning_rate": 0.0001, + "loss": 67.8654, + "step": 8950 + }, + { + "epoch": 1.4468218243475102, + "grad_norm": 0.6388489603996277, + "learning_rate": 0.0001, + "loss": 69.5605, + "step": 8960 + }, + { + "epoch": 1.4484366484326114, + "grad_norm": 0.735910952091217, + "learning_rate": 0.0001, + "loss": 66.7865, + "step": 8970 + }, + { + "epoch": 1.4500514725177127, + "grad_norm": 0.6826426386833191, + "learning_rate": 0.0001, + "loss": 68.2896, + "step": 8980 + }, + { + "epoch": 1.4516662966028138, + "grad_norm": 0.8190213441848755, + "learning_rate": 0.0001, + "loss": 70.3524, + "step": 8990 + }, + { + "epoch": 1.453281120687915, + "grad_norm": 0.6774603724479675, + "learning_rate": 0.0001, + "loss": 68.9084, + "step": 9000 + }, + { + "epoch": 1.4548959447730163, + "grad_norm": 0.8953077793121338, + "learning_rate": 0.0001, + "loss": 69.228, + "step": 9010 + }, + { + "epoch": 1.4565107688581176, + "grad_norm": 1.8509501218795776, + "learning_rate": 0.0001, + "loss": 68.0847, + "step": 9020 + }, + { + "epoch": 1.4581255929432189, + "grad_norm": 0.671280026435852, + "learning_rate": 0.0001, + "loss": 70.0691, + "step": 9030 + }, + { + "epoch": 1.45974041702832, + "grad_norm": 0.7242400646209717, + "learning_rate": 0.0001, + "loss": 68.8001, + "step": 9040 + }, + { + "epoch": 1.4613552411134212, + "grad_norm": 0.6609033942222595, + "learning_rate": 0.0001, + "loss": 69.0398, + "step": 9050 + }, + { + "epoch": 1.4629700651985225, + "grad_norm": 0.6850711703300476, + "learning_rate": 0.0001, + "loss": 70.8849, + "step": 9060 + }, + { + "epoch": 1.4645848892836237, + "grad_norm": 0.7409449219703674, + "learning_rate": 0.0001, + "loss": 68.0411, + "step": 9070 + }, + { + "epoch": 1.466199713368725, + "grad_norm": 0.6832754611968994, + "learning_rate": 0.0001, + "loss": 69.1219, + "step": 9080 + }, + { + "epoch": 1.467814537453826, + "grad_norm": 0.6671826243400574, + "learning_rate": 0.0001, + "loss": 68.1225, + "step": 9090 + }, + { + "epoch": 1.4694293615389273, + "grad_norm": 0.6758706569671631, + "learning_rate": 0.0001, + "loss": 68.6981, + "step": 9100 + }, + { + "epoch": 1.4710441856240286, + "grad_norm": 0.7885150909423828, + "learning_rate": 0.0001, + "loss": 69.29, + "step": 9110 + }, + { + "epoch": 1.4726590097091299, + "grad_norm": 0.7636108994483948, + "learning_rate": 0.0001, + "loss": 67.3151, + "step": 9120 + }, + { + "epoch": 1.4742738337942312, + "grad_norm": 0.6518180966377258, + "learning_rate": 0.0001, + "loss": 68.6208, + "step": 9130 + }, + { + "epoch": 1.4758886578793322, + "grad_norm": 0.6715161204338074, + "learning_rate": 0.0001, + "loss": 70.5106, + "step": 9140 + }, + { + "epoch": 1.4775034819644335, + "grad_norm": 0.6342995762825012, + "learning_rate": 0.0001, + "loss": 70.3067, + "step": 9150 + }, + { + "epoch": 1.4791183060495348, + "grad_norm": 0.6773675084114075, + "learning_rate": 0.0001, + "loss": 69.2888, + "step": 9160 + }, + { + "epoch": 1.480733130134636, + "grad_norm": 0.6887578368186951, + "learning_rate": 0.0001, + "loss": 69.2788, + "step": 9170 + }, + { + "epoch": 1.4823479542197373, + "grad_norm": 0.6793885231018066, + "learning_rate": 0.0001, + "loss": 68.3523, + "step": 9180 + }, + { + "epoch": 1.4839627783048384, + "grad_norm": 0.7483563423156738, + "learning_rate": 0.0001, + "loss": 67.9991, + "step": 9190 + }, + { + "epoch": 1.4855776023899396, + "grad_norm": 0.7007381916046143, + "learning_rate": 0.0001, + "loss": 67.5335, + "step": 9200 + }, + { + "epoch": 1.487192426475041, + "grad_norm": 0.7565139532089233, + "learning_rate": 0.0001, + "loss": 69.1093, + "step": 9210 + }, + { + "epoch": 1.4888072505601422, + "grad_norm": 1.0212360620498657, + "learning_rate": 0.0001, + "loss": 68.5061, + "step": 9220 + }, + { + "epoch": 1.4904220746452435, + "grad_norm": 0.9198092818260193, + "learning_rate": 0.0001, + "loss": 67.9735, + "step": 9230 + }, + { + "epoch": 1.4920368987303445, + "grad_norm": 0.6228001713752747, + "learning_rate": 0.0001, + "loss": 69.2231, + "step": 9240 + }, + { + "epoch": 1.4936517228154458, + "grad_norm": 0.6618618965148926, + "learning_rate": 0.0001, + "loss": 68.0434, + "step": 9250 + }, + { + "epoch": 1.495266546900547, + "grad_norm": 0.6241064071655273, + "learning_rate": 0.0001, + "loss": 68.4004, + "step": 9260 + }, + { + "epoch": 1.4968813709856483, + "grad_norm": 0.7336916327476501, + "learning_rate": 0.0001, + "loss": 69.0389, + "step": 9270 + }, + { + "epoch": 1.4984961950707496, + "grad_norm": 0.718125581741333, + "learning_rate": 0.0001, + "loss": 68.4923, + "step": 9280 + }, + { + "epoch": 1.5001110191558507, + "grad_norm": 0.6694948673248291, + "learning_rate": 0.0001, + "loss": 68.1006, + "step": 9290 + }, + { + "epoch": 1.501725843240952, + "grad_norm": 0.5815121531486511, + "learning_rate": 0.0001, + "loss": 68.5943, + "step": 9300 + }, + { + "epoch": 1.5033406673260532, + "grad_norm": 1.4076049327850342, + "learning_rate": 0.0001, + "loss": 68.1251, + "step": 9310 + }, + { + "epoch": 1.5049554914111543, + "grad_norm": 0.7294307947158813, + "learning_rate": 0.0001, + "loss": 68.0539, + "step": 9320 + }, + { + "epoch": 1.5065703154962558, + "grad_norm": 0.627821683883667, + "learning_rate": 0.0001, + "loss": 68.6859, + "step": 9330 + }, + { + "epoch": 1.5081851395813568, + "grad_norm": 0.6683267951011658, + "learning_rate": 0.0001, + "loss": 68.2727, + "step": 9340 + }, + { + "epoch": 1.509799963666458, + "grad_norm": 0.7070158123970032, + "learning_rate": 0.0001, + "loss": 69.1171, + "step": 9350 + }, + { + "epoch": 1.5114147877515594, + "grad_norm": 0.6324070692062378, + "learning_rate": 0.0001, + "loss": 69.7498, + "step": 9360 + }, + { + "epoch": 1.5130296118366604, + "grad_norm": 0.9814477562904358, + "learning_rate": 0.0001, + "loss": 69.1426, + "step": 9370 + }, + { + "epoch": 1.514644435921762, + "grad_norm": 0.5883511304855347, + "learning_rate": 0.0001, + "loss": 68.5363, + "step": 9380 + }, + { + "epoch": 1.516259260006863, + "grad_norm": 0.8325281739234924, + "learning_rate": 0.0001, + "loss": 67.5951, + "step": 9390 + }, + { + "epoch": 1.5178740840919642, + "grad_norm": 0.7456872463226318, + "learning_rate": 0.0001, + "loss": 68.1876, + "step": 9400 + }, + { + "epoch": 1.5194889081770655, + "grad_norm": 0.7692755460739136, + "learning_rate": 0.0001, + "loss": 69.1677, + "step": 9410 + }, + { + "epoch": 1.5211037322621666, + "grad_norm": 0.7346193194389343, + "learning_rate": 0.0001, + "loss": 69.8608, + "step": 9420 + }, + { + "epoch": 1.522718556347268, + "grad_norm": 0.7658166289329529, + "learning_rate": 0.0001, + "loss": 68.8584, + "step": 9430 + }, + { + "epoch": 1.524333380432369, + "grad_norm": 0.7892371416091919, + "learning_rate": 0.0001, + "loss": 68.3882, + "step": 9440 + }, + { + "epoch": 1.5259482045174704, + "grad_norm": 0.5943968296051025, + "learning_rate": 0.0001, + "loss": 68.7935, + "step": 9450 + }, + { + "epoch": 1.5275630286025716, + "grad_norm": 0.6611922383308411, + "learning_rate": 0.0001, + "loss": 69.0865, + "step": 9460 + }, + { + "epoch": 1.5291778526876727, + "grad_norm": 0.6697778105735779, + "learning_rate": 0.0001, + "loss": 68.6131, + "step": 9470 + }, + { + "epoch": 1.5307926767727742, + "grad_norm": 0.862293541431427, + "learning_rate": 0.0001, + "loss": 71.0167, + "step": 9480 + }, + { + "epoch": 1.5324075008578752, + "grad_norm": 0.7098677754402161, + "learning_rate": 0.0001, + "loss": 67.7216, + "step": 9490 + }, + { + "epoch": 1.5340223249429765, + "grad_norm": 0.6578338742256165, + "learning_rate": 0.0001, + "loss": 69.872, + "step": 9500 + }, + { + "epoch": 1.5356371490280778, + "grad_norm": 0.7272375822067261, + "learning_rate": 0.0001, + "loss": 70.0971, + "step": 9510 + }, + { + "epoch": 1.5372519731131788, + "grad_norm": 0.6497064828872681, + "learning_rate": 0.0001, + "loss": 70.5921, + "step": 9520 + }, + { + "epoch": 1.5388667971982803, + "grad_norm": 0.6691288352012634, + "learning_rate": 0.0001, + "loss": 68.5213, + "step": 9530 + }, + { + "epoch": 1.5404816212833814, + "grad_norm": 0.9180259108543396, + "learning_rate": 0.0001, + "loss": 67.0699, + "step": 9540 + }, + { + "epoch": 1.5420964453684827, + "grad_norm": 0.7062082886695862, + "learning_rate": 0.0001, + "loss": 69.3965, + "step": 9550 + }, + { + "epoch": 1.543711269453584, + "grad_norm": 0.711613655090332, + "learning_rate": 0.0001, + "loss": 67.8276, + "step": 9560 + }, + { + "epoch": 1.545326093538685, + "grad_norm": 0.670032799243927, + "learning_rate": 0.0001, + "loss": 68.1687, + "step": 9570 + }, + { + "epoch": 1.5469409176237865, + "grad_norm": 0.7531955242156982, + "learning_rate": 0.0001, + "loss": 69.2144, + "step": 9580 + }, + { + "epoch": 1.5485557417088875, + "grad_norm": 0.6759967803955078, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 9590 + }, + { + "epoch": 1.5501705657939888, + "grad_norm": 0.6134868860244751, + "learning_rate": 0.0001, + "loss": 67.4348, + "step": 9600 + }, + { + "epoch": 1.55178538987909, + "grad_norm": 0.638785719871521, + "learning_rate": 0.0001, + "loss": 68.4635, + "step": 9610 + }, + { + "epoch": 1.5534002139641911, + "grad_norm": 0.7901169657707214, + "learning_rate": 0.0001, + "loss": 68.1079, + "step": 9620 + }, + { + "epoch": 1.5550150380492926, + "grad_norm": 0.6970444917678833, + "learning_rate": 0.0001, + "loss": 69.3883, + "step": 9630 + }, + { + "epoch": 1.5566298621343937, + "grad_norm": 0.6954355239868164, + "learning_rate": 0.0001, + "loss": 69.3841, + "step": 9640 + }, + { + "epoch": 1.558244686219495, + "grad_norm": 0.7097641825675964, + "learning_rate": 0.0001, + "loss": 68.3417, + "step": 9650 + }, + { + "epoch": 1.5598595103045962, + "grad_norm": 0.7082354426383972, + "learning_rate": 0.0001, + "loss": 69.9314, + "step": 9660 + }, + { + "epoch": 1.5614743343896973, + "grad_norm": 0.6570103764533997, + "learning_rate": 0.0001, + "loss": 69.7219, + "step": 9670 + }, + { + "epoch": 1.5630891584747988, + "grad_norm": 0.5705522894859314, + "learning_rate": 0.0001, + "loss": 67.5177, + "step": 9680 + }, + { + "epoch": 1.5647039825598998, + "grad_norm": 0.6992695331573486, + "learning_rate": 0.0001, + "loss": 67.6538, + "step": 9690 + }, + { + "epoch": 1.5663188066450011, + "grad_norm": 0.7523599863052368, + "learning_rate": 0.0001, + "loss": 66.869, + "step": 9700 + }, + { + "epoch": 1.5679336307301024, + "grad_norm": 0.6267555356025696, + "learning_rate": 0.0001, + "loss": 66.6805, + "step": 9710 + }, + { + "epoch": 1.5695484548152034, + "grad_norm": 0.6138116717338562, + "learning_rate": 0.0001, + "loss": 71.3217, + "step": 9720 + }, + { + "epoch": 1.571163278900305, + "grad_norm": 0.7408527731895447, + "learning_rate": 0.0001, + "loss": 69.2759, + "step": 9730 + }, + { + "epoch": 1.572778102985406, + "grad_norm": 0.813761830329895, + "learning_rate": 0.0001, + "loss": 70.4959, + "step": 9740 + }, + { + "epoch": 1.5743929270705073, + "grad_norm": 1.0363599061965942, + "learning_rate": 0.0001, + "loss": 68.1996, + "step": 9750 + }, + { + "epoch": 1.5760077511556085, + "grad_norm": 0.7773903012275696, + "learning_rate": 0.0001, + "loss": 67.9354, + "step": 9760 + }, + { + "epoch": 1.5776225752407096, + "grad_norm": 0.6465738415718079, + "learning_rate": 0.0001, + "loss": 69.5247, + "step": 9770 + }, + { + "epoch": 1.579237399325811, + "grad_norm": 0.8433385491371155, + "learning_rate": 0.0001, + "loss": 66.9448, + "step": 9780 + }, + { + "epoch": 1.5808522234109121, + "grad_norm": 0.8651545643806458, + "learning_rate": 0.0001, + "loss": 68.6994, + "step": 9790 + }, + { + "epoch": 1.5824670474960134, + "grad_norm": 0.6940448880195618, + "learning_rate": 0.0001, + "loss": 67.622, + "step": 9800 + }, + { + "epoch": 1.5840818715811147, + "grad_norm": 0.622756838798523, + "learning_rate": 0.0001, + "loss": 69.0994, + "step": 9810 + }, + { + "epoch": 1.5856966956662157, + "grad_norm": 0.6119604110717773, + "learning_rate": 0.0001, + "loss": 67.7682, + "step": 9820 + }, + { + "epoch": 1.5873115197513172, + "grad_norm": 0.5729681849479675, + "learning_rate": 0.0001, + "loss": 67.3113, + "step": 9830 + }, + { + "epoch": 1.5889263438364183, + "grad_norm": 0.7143844366073608, + "learning_rate": 0.0001, + "loss": 68.8128, + "step": 9840 + }, + { + "epoch": 1.5905411679215196, + "grad_norm": 0.9814873337745667, + "learning_rate": 0.0001, + "loss": 69.3099, + "step": 9850 + }, + { + "epoch": 1.5921559920066208, + "grad_norm": 0.8044945001602173, + "learning_rate": 0.0001, + "loss": 69.3939, + "step": 9860 + }, + { + "epoch": 1.5937708160917219, + "grad_norm": 0.8494839668273926, + "learning_rate": 0.0001, + "loss": 69.3889, + "step": 9870 + }, + { + "epoch": 1.5953856401768234, + "grad_norm": 0.6468870639801025, + "learning_rate": 0.0001, + "loss": 67.2047, + "step": 9880 + }, + { + "epoch": 1.5970004642619244, + "grad_norm": 0.6270474791526794, + "learning_rate": 0.0001, + "loss": 69.6813, + "step": 9890 + }, + { + "epoch": 1.5986152883470257, + "grad_norm": 0.7102645039558411, + "learning_rate": 0.0001, + "loss": 69.1546, + "step": 9900 + }, + { + "epoch": 1.600230112432127, + "grad_norm": 0.5929123759269714, + "learning_rate": 0.0001, + "loss": 67.2187, + "step": 9910 + }, + { + "epoch": 1.601844936517228, + "grad_norm": 0.6466200947761536, + "learning_rate": 0.0001, + "loss": 68.6876, + "step": 9920 + }, + { + "epoch": 1.6034597606023295, + "grad_norm": 0.6441522836685181, + "learning_rate": 0.0001, + "loss": 68.7955, + "step": 9930 + }, + { + "epoch": 1.6050745846874306, + "grad_norm": 0.7264062166213989, + "learning_rate": 0.0001, + "loss": 67.8876, + "step": 9940 + }, + { + "epoch": 1.6066894087725319, + "grad_norm": 0.6973199248313904, + "learning_rate": 0.0001, + "loss": 69.0904, + "step": 9950 + }, + { + "epoch": 1.6083042328576331, + "grad_norm": 0.582053005695343, + "learning_rate": 0.0001, + "loss": 69.3876, + "step": 9960 + }, + { + "epoch": 1.6099190569427342, + "grad_norm": 0.6431038975715637, + "learning_rate": 0.0001, + "loss": 67.4232, + "step": 9970 + }, + { + "epoch": 1.6115338810278357, + "grad_norm": 0.6313247084617615, + "learning_rate": 0.0001, + "loss": 68.3759, + "step": 9980 + }, + { + "epoch": 1.6131487051129367, + "grad_norm": 1.2003700733184814, + "learning_rate": 0.0001, + "loss": 68.3413, + "step": 9990 + }, + { + "epoch": 1.614763529198038, + "grad_norm": 1.0262068510055542, + "learning_rate": 0.0001, + "loss": 66.7308, + "step": 10000 + }, + { + "epoch": 1.6163783532831393, + "grad_norm": 0.5975973010063171, + "learning_rate": 0.0001, + "loss": 68.6893, + "step": 10010 + }, + { + "epoch": 1.6179931773682403, + "grad_norm": 0.6045264601707458, + "learning_rate": 0.0001, + "loss": 69.5241, + "step": 10020 + }, + { + "epoch": 1.6196080014533418, + "grad_norm": 0.654019832611084, + "learning_rate": 0.0001, + "loss": 66.1208, + "step": 10030 + }, + { + "epoch": 1.6212228255384429, + "grad_norm": 0.533981204032898, + "learning_rate": 0.0001, + "loss": 68.2129, + "step": 10040 + }, + { + "epoch": 1.6228376496235442, + "grad_norm": 1.184734582901001, + "learning_rate": 0.0001, + "loss": 70.0547, + "step": 10050 + }, + { + "epoch": 1.6244524737086454, + "grad_norm": 0.5836975574493408, + "learning_rate": 0.0001, + "loss": 68.1098, + "step": 10060 + }, + { + "epoch": 1.6260672977937465, + "grad_norm": 0.5856032967567444, + "learning_rate": 0.0001, + "loss": 67.483, + "step": 10070 + }, + { + "epoch": 1.627682121878848, + "grad_norm": 0.6735382080078125, + "learning_rate": 0.0001, + "loss": 68.8628, + "step": 10080 + }, + { + "epoch": 1.629296945963949, + "grad_norm": 0.5753204226493835, + "learning_rate": 0.0001, + "loss": 66.2777, + "step": 10090 + }, + { + "epoch": 1.6309117700490503, + "grad_norm": 0.8327603340148926, + "learning_rate": 0.0001, + "loss": 67.9309, + "step": 10100 + }, + { + "epoch": 1.6325265941341516, + "grad_norm": 0.6873740553855896, + "learning_rate": 0.0001, + "loss": 68.7627, + "step": 10110 + }, + { + "epoch": 1.6341414182192526, + "grad_norm": 0.7218474745750427, + "learning_rate": 0.0001, + "loss": 69.3637, + "step": 10120 + }, + { + "epoch": 1.6357562423043541, + "grad_norm": 0.8393228650093079, + "learning_rate": 0.0001, + "loss": 66.7686, + "step": 10130 + }, + { + "epoch": 1.6373710663894552, + "grad_norm": 0.6433869004249573, + "learning_rate": 0.0001, + "loss": 68.0896, + "step": 10140 + }, + { + "epoch": 1.6389858904745565, + "grad_norm": 0.7419982552528381, + "learning_rate": 0.0001, + "loss": 67.9707, + "step": 10150 + }, + { + "epoch": 1.6406007145596577, + "grad_norm": 0.5884562730789185, + "learning_rate": 0.0001, + "loss": 68.0287, + "step": 10160 + }, + { + "epoch": 1.6422155386447588, + "grad_norm": 0.729557454586029, + "learning_rate": 0.0001, + "loss": 69.942, + "step": 10170 + }, + { + "epoch": 1.6438303627298603, + "grad_norm": 0.6026669144630432, + "learning_rate": 0.0001, + "loss": 66.8192, + "step": 10180 + }, + { + "epoch": 1.6454451868149613, + "grad_norm": 0.6921732425689697, + "learning_rate": 0.0001, + "loss": 66.7087, + "step": 10190 + }, + { + "epoch": 1.6470600109000626, + "grad_norm": 0.755172848701477, + "learning_rate": 0.0001, + "loss": 70.799, + "step": 10200 + }, + { + "epoch": 1.6486748349851639, + "grad_norm": 0.7318310737609863, + "learning_rate": 0.0001, + "loss": 67.6957, + "step": 10210 + }, + { + "epoch": 1.650289659070265, + "grad_norm": 0.6915041208267212, + "learning_rate": 0.0001, + "loss": 69.3388, + "step": 10220 + }, + { + "epoch": 1.6519044831553664, + "grad_norm": 0.7173444628715515, + "learning_rate": 0.0001, + "loss": 68.3612, + "step": 10230 + }, + { + "epoch": 1.6535193072404675, + "grad_norm": 0.7485487461090088, + "learning_rate": 0.0001, + "loss": 68.5366, + "step": 10240 + }, + { + "epoch": 1.6551341313255687, + "grad_norm": 0.5838190913200378, + "learning_rate": 0.0001, + "loss": 68.3841, + "step": 10250 + }, + { + "epoch": 1.65674895541067, + "grad_norm": 0.5908819437026978, + "learning_rate": 0.0001, + "loss": 67.8741, + "step": 10260 + }, + { + "epoch": 1.658363779495771, + "grad_norm": 0.7475612163543701, + "learning_rate": 0.0001, + "loss": 69.0636, + "step": 10270 + }, + { + "epoch": 1.6599786035808726, + "grad_norm": 0.5451382994651794, + "learning_rate": 0.0001, + "loss": 68.6695, + "step": 10280 + }, + { + "epoch": 1.6615934276659736, + "grad_norm": 0.6277157068252563, + "learning_rate": 0.0001, + "loss": 67.5432, + "step": 10290 + }, + { + "epoch": 1.663208251751075, + "grad_norm": 0.8144198060035706, + "learning_rate": 0.0001, + "loss": 66.7892, + "step": 10300 + }, + { + "epoch": 1.6648230758361762, + "grad_norm": 0.7400529980659485, + "learning_rate": 0.0001, + "loss": 66.2653, + "step": 10310 + }, + { + "epoch": 1.6664378999212772, + "grad_norm": 0.74524986743927, + "learning_rate": 0.0001, + "loss": 67.8252, + "step": 10320 + }, + { + "epoch": 1.6680527240063787, + "grad_norm": 0.5554162263870239, + "learning_rate": 0.0001, + "loss": 69.7704, + "step": 10330 + }, + { + "epoch": 1.6696675480914798, + "grad_norm": 0.8194338083267212, + "learning_rate": 0.0001, + "loss": 69.4668, + "step": 10340 + }, + { + "epoch": 1.671282372176581, + "grad_norm": 0.7947783470153809, + "learning_rate": 0.0001, + "loss": 64.9276, + "step": 10350 + }, + { + "epoch": 1.6728971962616823, + "grad_norm": 0.6145230531692505, + "learning_rate": 0.0001, + "loss": 68.1815, + "step": 10360 + }, + { + "epoch": 1.6745120203467834, + "grad_norm": 0.6293516755104065, + "learning_rate": 0.0001, + "loss": 67.8994, + "step": 10370 + }, + { + "epoch": 1.6761268444318849, + "grad_norm": 0.6160871386528015, + "learning_rate": 0.0001, + "loss": 66.7463, + "step": 10380 + }, + { + "epoch": 1.677741668516986, + "grad_norm": 0.6925204992294312, + "learning_rate": 0.0001, + "loss": 68.5261, + "step": 10390 + }, + { + "epoch": 1.6793564926020872, + "grad_norm": 0.6892788410186768, + "learning_rate": 0.0001, + "loss": 66.8928, + "step": 10400 + }, + { + "epoch": 1.6809713166871885, + "grad_norm": 0.9066945910453796, + "learning_rate": 0.0001, + "loss": 66.2556, + "step": 10410 + }, + { + "epoch": 1.6825861407722895, + "grad_norm": 0.5857477784156799, + "learning_rate": 0.0001, + "loss": 67.1866, + "step": 10420 + }, + { + "epoch": 1.684200964857391, + "grad_norm": 0.7989841103553772, + "learning_rate": 0.0001, + "loss": 67.7762, + "step": 10430 + }, + { + "epoch": 1.685815788942492, + "grad_norm": 0.559428870677948, + "learning_rate": 0.0001, + "loss": 67.5628, + "step": 10440 + }, + { + "epoch": 1.6874306130275933, + "grad_norm": 0.6718077659606934, + "learning_rate": 0.0001, + "loss": 69.2877, + "step": 10450 + }, + { + "epoch": 1.6890454371126946, + "grad_norm": 0.6952136754989624, + "learning_rate": 0.0001, + "loss": 67.2734, + "step": 10460 + }, + { + "epoch": 1.6906602611977957, + "grad_norm": 0.6079897880554199, + "learning_rate": 0.0001, + "loss": 67.4677, + "step": 10470 + }, + { + "epoch": 1.692275085282897, + "grad_norm": 0.6139684915542603, + "learning_rate": 0.0001, + "loss": 69.2464, + "step": 10480 + }, + { + "epoch": 1.6938899093679982, + "grad_norm": 0.7009351253509521, + "learning_rate": 0.0001, + "loss": 67.5424, + "step": 10490 + }, + { + "epoch": 1.6955047334530995, + "grad_norm": 0.8567383885383606, + "learning_rate": 0.0001, + "loss": 65.6573, + "step": 10500 + }, + { + "epoch": 1.6971195575382008, + "grad_norm": 0.6903256177902222, + "learning_rate": 0.0001, + "loss": 69.2218, + "step": 10510 + }, + { + "epoch": 1.6987343816233018, + "grad_norm": 0.5958806872367859, + "learning_rate": 0.0001, + "loss": 67.3718, + "step": 10520 + }, + { + "epoch": 1.700349205708403, + "grad_norm": 0.5983722805976868, + "learning_rate": 0.0001, + "loss": 68.7944, + "step": 10530 + }, + { + "epoch": 1.7019640297935044, + "grad_norm": 0.8119397759437561, + "learning_rate": 0.0001, + "loss": 66.7936, + "step": 10540 + }, + { + "epoch": 1.7035788538786056, + "grad_norm": 0.6383484601974487, + "learning_rate": 0.0001, + "loss": 67.8636, + "step": 10550 + }, + { + "epoch": 1.705193677963707, + "grad_norm": 0.6574084162712097, + "learning_rate": 0.0001, + "loss": 67.26, + "step": 10560 + }, + { + "epoch": 1.706808502048808, + "grad_norm": 0.6716108918190002, + "learning_rate": 0.0001, + "loss": 66.485, + "step": 10570 + }, + { + "epoch": 1.7084233261339092, + "grad_norm": 0.6998292803764343, + "learning_rate": 0.0001, + "loss": 66.4201, + "step": 10580 + }, + { + "epoch": 1.7100381502190105, + "grad_norm": 0.7300788760185242, + "learning_rate": 0.0001, + "loss": 67.1446, + "step": 10590 + }, + { + "epoch": 1.7116529743041118, + "grad_norm": 0.7066853642463684, + "learning_rate": 0.0001, + "loss": 70.8954, + "step": 10600 + }, + { + "epoch": 1.713267798389213, + "grad_norm": 0.6933701038360596, + "learning_rate": 0.0001, + "loss": 67.8473, + "step": 10610 + }, + { + "epoch": 1.7148826224743141, + "grad_norm": 0.6885779500007629, + "learning_rate": 0.0001, + "loss": 65.3276, + "step": 10620 + }, + { + "epoch": 1.7164974465594154, + "grad_norm": 0.6542202830314636, + "learning_rate": 0.0001, + "loss": 69.2596, + "step": 10630 + }, + { + "epoch": 1.7181122706445167, + "grad_norm": 0.5719691514968872, + "learning_rate": 0.0001, + "loss": 69.1907, + "step": 10640 + }, + { + "epoch": 1.719727094729618, + "grad_norm": 0.6596891283988953, + "learning_rate": 0.0001, + "loss": 69.0778, + "step": 10650 + }, + { + "epoch": 1.7213419188147192, + "grad_norm": 0.6860799193382263, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 10660 + }, + { + "epoch": 1.7229567428998203, + "grad_norm": 0.7912564277648926, + "learning_rate": 0.0001, + "loss": 68.2091, + "step": 10670 + }, + { + "epoch": 1.7245715669849215, + "grad_norm": 0.6673903465270996, + "learning_rate": 0.0001, + "loss": 68.24, + "step": 10680 + }, + { + "epoch": 1.7261863910700228, + "grad_norm": 0.6243769526481628, + "learning_rate": 0.0001, + "loss": 68.6396, + "step": 10690 + }, + { + "epoch": 1.727801215155124, + "grad_norm": 0.6622587442398071, + "learning_rate": 0.0001, + "loss": 67.9406, + "step": 10700 + }, + { + "epoch": 1.7294160392402254, + "grad_norm": 0.6808138489723206, + "learning_rate": 0.0001, + "loss": 69.1403, + "step": 10710 + }, + { + "epoch": 1.7310308633253264, + "grad_norm": 0.8498415350914001, + "learning_rate": 0.0001, + "loss": 70.1438, + "step": 10720 + }, + { + "epoch": 1.7326456874104277, + "grad_norm": 1.0091742277145386, + "learning_rate": 0.0001, + "loss": 67.0196, + "step": 10730 + }, + { + "epoch": 1.734260511495529, + "grad_norm": 0.6997767686843872, + "learning_rate": 0.0001, + "loss": 67.0307, + "step": 10740 + }, + { + "epoch": 1.7358753355806302, + "grad_norm": 0.6618793606758118, + "learning_rate": 0.0001, + "loss": 67.906, + "step": 10750 + }, + { + "epoch": 1.7374901596657315, + "grad_norm": 0.7082014679908752, + "learning_rate": 0.0001, + "loss": 68.0665, + "step": 10760 + }, + { + "epoch": 1.7391049837508326, + "grad_norm": 0.8494391441345215, + "learning_rate": 0.0001, + "loss": 68.4202, + "step": 10770 + }, + { + "epoch": 1.7407198078359338, + "grad_norm": 0.6247998476028442, + "learning_rate": 0.0001, + "loss": 66.4399, + "step": 10780 + }, + { + "epoch": 1.742334631921035, + "grad_norm": 0.8239936828613281, + "learning_rate": 0.0001, + "loss": 68.4374, + "step": 10790 + }, + { + "epoch": 1.7439494560061364, + "grad_norm": 0.6734293103218079, + "learning_rate": 0.0001, + "loss": 66.3411, + "step": 10800 + }, + { + "epoch": 1.7455642800912377, + "grad_norm": 0.7066450119018555, + "learning_rate": 0.0001, + "loss": 67.7102, + "step": 10810 + }, + { + "epoch": 1.7471791041763387, + "grad_norm": 0.7440647482872009, + "learning_rate": 0.0001, + "loss": 67.7221, + "step": 10820 + }, + { + "epoch": 1.74879392826144, + "grad_norm": 0.7771329283714294, + "learning_rate": 0.0001, + "loss": 67.5907, + "step": 10830 + }, + { + "epoch": 1.7504087523465413, + "grad_norm": 1.2484230995178223, + "learning_rate": 0.0001, + "loss": 66.3242, + "step": 10840 + }, + { + "epoch": 1.7520235764316425, + "grad_norm": 0.6946772933006287, + "learning_rate": 0.0001, + "loss": 67.8741, + "step": 10850 + }, + { + "epoch": 1.7536384005167438, + "grad_norm": 0.7676035761833191, + "learning_rate": 0.0001, + "loss": 67.3776, + "step": 10860 + }, + { + "epoch": 1.7552532246018449, + "grad_norm": 0.5534589290618896, + "learning_rate": 0.0001, + "loss": 67.6656, + "step": 10870 + }, + { + "epoch": 1.7568680486869461, + "grad_norm": 1.2745810747146606, + "learning_rate": 0.0001, + "loss": 68.5959, + "step": 10880 + }, + { + "epoch": 1.7584828727720474, + "grad_norm": 0.6241582036018372, + "learning_rate": 0.0001, + "loss": 67.7142, + "step": 10890 + }, + { + "epoch": 1.7600976968571487, + "grad_norm": 0.9308313131332397, + "learning_rate": 0.0001, + "loss": 67.0322, + "step": 10900 + }, + { + "epoch": 1.76171252094225, + "grad_norm": 0.5639570951461792, + "learning_rate": 0.0001, + "loss": 67.928, + "step": 10910 + }, + { + "epoch": 1.763327345027351, + "grad_norm": 0.6692782044410706, + "learning_rate": 0.0001, + "loss": 67.625, + "step": 10920 + }, + { + "epoch": 1.7649421691124523, + "grad_norm": 0.7322304248809814, + "learning_rate": 0.0001, + "loss": 66.2372, + "step": 10930 + }, + { + "epoch": 1.7665569931975535, + "grad_norm": 0.6275238394737244, + "learning_rate": 0.0001, + "loss": 67.7628, + "step": 10940 + }, + { + "epoch": 1.7681718172826548, + "grad_norm": 0.6832380890846252, + "learning_rate": 0.0001, + "loss": 67.6587, + "step": 10950 + }, + { + "epoch": 1.769786641367756, + "grad_norm": 0.6934792399406433, + "learning_rate": 0.0001, + "loss": 67.1411, + "step": 10960 + }, + { + "epoch": 1.7714014654528571, + "grad_norm": 0.7031943202018738, + "learning_rate": 0.0001, + "loss": 69.2734, + "step": 10970 + }, + { + "epoch": 1.7730162895379584, + "grad_norm": 0.91269451379776, + "learning_rate": 0.0001, + "loss": 68.1353, + "step": 10980 + }, + { + "epoch": 1.7746311136230597, + "grad_norm": 0.8927266001701355, + "learning_rate": 0.0001, + "loss": 68.003, + "step": 10990 + }, + { + "epoch": 1.776245937708161, + "grad_norm": 0.9115116596221924, + "learning_rate": 0.0001, + "loss": 69.1506, + "step": 11000 + }, + { + "epoch": 1.7778607617932622, + "grad_norm": 0.6055391430854797, + "learning_rate": 0.0001, + "loss": 68.9785, + "step": 11010 + }, + { + "epoch": 1.7794755858783633, + "grad_norm": 0.872809112071991, + "learning_rate": 0.0001, + "loss": 66.373, + "step": 11020 + }, + { + "epoch": 1.7810904099634646, + "grad_norm": 0.5838716626167297, + "learning_rate": 0.0001, + "loss": 69.507, + "step": 11030 + }, + { + "epoch": 1.7827052340485658, + "grad_norm": 0.6108883619308472, + "learning_rate": 0.0001, + "loss": 67.3743, + "step": 11040 + }, + { + "epoch": 1.784320058133667, + "grad_norm": 0.5457114577293396, + "learning_rate": 0.0001, + "loss": 66.1342, + "step": 11050 + }, + { + "epoch": 1.7859348822187684, + "grad_norm": 0.7696211338043213, + "learning_rate": 0.0001, + "loss": 67.9779, + "step": 11060 + }, + { + "epoch": 1.7875497063038694, + "grad_norm": 0.5795412659645081, + "learning_rate": 0.0001, + "loss": 67.1763, + "step": 11070 + }, + { + "epoch": 1.7891645303889707, + "grad_norm": 0.7401878833770752, + "learning_rate": 0.0001, + "loss": 67.4972, + "step": 11080 + }, + { + "epoch": 1.790779354474072, + "grad_norm": 0.666051983833313, + "learning_rate": 0.0001, + "loss": 68.7621, + "step": 11090 + }, + { + "epoch": 1.792394178559173, + "grad_norm": 0.9650185108184814, + "learning_rate": 0.0001, + "loss": 66.7721, + "step": 11100 + }, + { + "epoch": 1.7940090026442745, + "grad_norm": 0.6026908159255981, + "learning_rate": 0.0001, + "loss": 66.2993, + "step": 11110 + }, + { + "epoch": 1.7956238267293756, + "grad_norm": 0.6560576558113098, + "learning_rate": 0.0001, + "loss": 67.8543, + "step": 11120 + }, + { + "epoch": 1.7972386508144769, + "grad_norm": 0.9523794054985046, + "learning_rate": 0.0001, + "loss": 66.7881, + "step": 11130 + }, + { + "epoch": 1.7988534748995781, + "grad_norm": 0.7241806983947754, + "learning_rate": 0.0001, + "loss": 68.1071, + "step": 11140 + }, + { + "epoch": 1.8004682989846792, + "grad_norm": 0.639866292476654, + "learning_rate": 0.0001, + "loss": 68.992, + "step": 11150 + }, + { + "epoch": 1.8020831230697807, + "grad_norm": 0.6050712466239929, + "learning_rate": 0.0001, + "loss": 67.7993, + "step": 11160 + }, + { + "epoch": 1.8036979471548817, + "grad_norm": 0.5777723789215088, + "learning_rate": 0.0001, + "loss": 66.9515, + "step": 11170 + }, + { + "epoch": 1.805312771239983, + "grad_norm": 0.6153426766395569, + "learning_rate": 0.0001, + "loss": 68.3413, + "step": 11180 + }, + { + "epoch": 1.8069275953250843, + "grad_norm": 0.6118552088737488, + "learning_rate": 0.0001, + "loss": 67.7853, + "step": 11190 + }, + { + "epoch": 1.8085424194101853, + "grad_norm": 0.6520341038703918, + "learning_rate": 0.0001, + "loss": 67.3511, + "step": 11200 + }, + { + "epoch": 1.8101572434952868, + "grad_norm": 0.6189568638801575, + "learning_rate": 0.0001, + "loss": 69.28, + "step": 11210 + }, + { + "epoch": 1.811772067580388, + "grad_norm": 0.602411150932312, + "learning_rate": 0.0001, + "loss": 67.5133, + "step": 11220 + }, + { + "epoch": 1.8133868916654892, + "grad_norm": 0.7359414100646973, + "learning_rate": 0.0001, + "loss": 68.631, + "step": 11230 + }, + { + "epoch": 1.8150017157505904, + "grad_norm": 0.6216995120048523, + "learning_rate": 0.0001, + "loss": 68.3588, + "step": 11240 + }, + { + "epoch": 1.8166165398356915, + "grad_norm": 0.7683486938476562, + "learning_rate": 0.0001, + "loss": 67.1434, + "step": 11250 + }, + { + "epoch": 1.818231363920793, + "grad_norm": 0.5565283894538879, + "learning_rate": 0.0001, + "loss": 66.9995, + "step": 11260 + }, + { + "epoch": 1.819846188005894, + "grad_norm": 0.5623223781585693, + "learning_rate": 0.0001, + "loss": 68.1316, + "step": 11270 + }, + { + "epoch": 1.8214610120909953, + "grad_norm": 0.532821774482727, + "learning_rate": 0.0001, + "loss": 69.1903, + "step": 11280 + }, + { + "epoch": 1.8230758361760966, + "grad_norm": 0.6297196745872498, + "learning_rate": 0.0001, + "loss": 68.4965, + "step": 11290 + }, + { + "epoch": 1.8246906602611976, + "grad_norm": 0.839862048625946, + "learning_rate": 0.0001, + "loss": 67.1782, + "step": 11300 + }, + { + "epoch": 1.8263054843462991, + "grad_norm": 0.7021651864051819, + "learning_rate": 0.0001, + "loss": 67.4355, + "step": 11310 + }, + { + "epoch": 1.8279203084314002, + "grad_norm": 0.676034152507782, + "learning_rate": 0.0001, + "loss": 68.3721, + "step": 11320 + }, + { + "epoch": 1.8295351325165015, + "grad_norm": 0.6621078252792358, + "learning_rate": 0.0001, + "loss": 67.3255, + "step": 11330 + }, + { + "epoch": 1.8311499566016027, + "grad_norm": 0.5902897119522095, + "learning_rate": 0.0001, + "loss": 68.0641, + "step": 11340 + }, + { + "epoch": 1.8327647806867038, + "grad_norm": 0.6334371566772461, + "learning_rate": 0.0001, + "loss": 68.2682, + "step": 11350 + }, + { + "epoch": 1.8343796047718053, + "grad_norm": 0.6753348708152771, + "learning_rate": 0.0001, + "loss": 66.6749, + "step": 11360 + }, + { + "epoch": 1.8359944288569063, + "grad_norm": 0.7221750617027283, + "learning_rate": 0.0001, + "loss": 68.4142, + "step": 11370 + }, + { + "epoch": 1.8376092529420076, + "grad_norm": 0.7476344108581543, + "learning_rate": 0.0001, + "loss": 67.1416, + "step": 11380 + }, + { + "epoch": 1.8392240770271089, + "grad_norm": 0.5927479863166809, + "learning_rate": 0.0001, + "loss": 67.6368, + "step": 11390 + }, + { + "epoch": 1.84083890111221, + "grad_norm": 0.6753717660903931, + "learning_rate": 0.0001, + "loss": 67.1026, + "step": 11400 + }, + { + "epoch": 1.8424537251973114, + "grad_norm": 0.6500268578529358, + "learning_rate": 0.0001, + "loss": 68.7537, + "step": 11410 + }, + { + "epoch": 1.8440685492824125, + "grad_norm": 0.5930709838867188, + "learning_rate": 0.0001, + "loss": 65.7429, + "step": 11420 + }, + { + "epoch": 1.8456833733675138, + "grad_norm": 0.6467777490615845, + "learning_rate": 0.0001, + "loss": 68.1333, + "step": 11430 + }, + { + "epoch": 1.847298197452615, + "grad_norm": 0.6935669183731079, + "learning_rate": 0.0001, + "loss": 66.7728, + "step": 11440 + }, + { + "epoch": 1.848913021537716, + "grad_norm": 0.5838116407394409, + "learning_rate": 0.0001, + "loss": 66.6005, + "step": 11450 + }, + { + "epoch": 1.8505278456228176, + "grad_norm": 0.8621205687522888, + "learning_rate": 0.0001, + "loss": 66.0859, + "step": 11460 + }, + { + "epoch": 1.8521426697079186, + "grad_norm": 0.7119213342666626, + "learning_rate": 0.0001, + "loss": 66.6198, + "step": 11470 + }, + { + "epoch": 1.85375749379302, + "grad_norm": 0.6861911416053772, + "learning_rate": 0.0001, + "loss": 67.2275, + "step": 11480 + }, + { + "epoch": 1.8553723178781212, + "grad_norm": 0.5653937458992004, + "learning_rate": 0.0001, + "loss": 67.827, + "step": 11490 + }, + { + "epoch": 1.8569871419632222, + "grad_norm": 0.6458482146263123, + "learning_rate": 0.0001, + "loss": 66.3906, + "step": 11500 + }, + { + "epoch": 1.8586019660483237, + "grad_norm": 0.6701959371566772, + "learning_rate": 0.0001, + "loss": 67.88, + "step": 11510 + }, + { + "epoch": 1.8602167901334248, + "grad_norm": 0.6598711609840393, + "learning_rate": 0.0001, + "loss": 66.8373, + "step": 11520 + }, + { + "epoch": 1.861831614218526, + "grad_norm": 0.6095236539840698, + "learning_rate": 0.0001, + "loss": 67.6478, + "step": 11530 + }, + { + "epoch": 1.8634464383036273, + "grad_norm": 0.8192022442817688, + "learning_rate": 0.0001, + "loss": 67.7343, + "step": 11540 + }, + { + "epoch": 1.8650612623887284, + "grad_norm": 0.5788753628730774, + "learning_rate": 0.0001, + "loss": 66.7909, + "step": 11550 + }, + { + "epoch": 1.8666760864738299, + "grad_norm": 0.5383989214897156, + "learning_rate": 0.0001, + "loss": 68.5349, + "step": 11560 + }, + { + "epoch": 1.868290910558931, + "grad_norm": 0.6224480271339417, + "learning_rate": 0.0001, + "loss": 68.1938, + "step": 11570 + }, + { + "epoch": 1.8699057346440322, + "grad_norm": 0.5852788686752319, + "learning_rate": 0.0001, + "loss": 66.4678, + "step": 11580 + }, + { + "epoch": 1.8715205587291335, + "grad_norm": 0.730378270149231, + "learning_rate": 0.0001, + "loss": 67.57, + "step": 11590 + }, + { + "epoch": 1.8731353828142345, + "grad_norm": 0.6197090744972229, + "learning_rate": 0.0001, + "loss": 65.7801, + "step": 11600 + }, + { + "epoch": 1.874750206899336, + "grad_norm": 0.518974781036377, + "learning_rate": 0.0001, + "loss": 68.9308, + "step": 11610 + }, + { + "epoch": 1.876365030984437, + "grad_norm": 0.6601742506027222, + "learning_rate": 0.0001, + "loss": 66.821, + "step": 11620 + }, + { + "epoch": 1.8779798550695384, + "grad_norm": 0.7409974932670593, + "learning_rate": 0.0001, + "loss": 66.6883, + "step": 11630 + }, + { + "epoch": 1.8795946791546396, + "grad_norm": 0.6372189521789551, + "learning_rate": 0.0001, + "loss": 67.2109, + "step": 11640 + }, + { + "epoch": 1.8812095032397407, + "grad_norm": 0.6107574701309204, + "learning_rate": 0.0001, + "loss": 67.029, + "step": 11650 + }, + { + "epoch": 1.8828243273248422, + "grad_norm": 0.6543708443641663, + "learning_rate": 0.0001, + "loss": 64.2133, + "step": 11660 + }, + { + "epoch": 1.8844391514099432, + "grad_norm": 0.5161535739898682, + "learning_rate": 0.0001, + "loss": 66.7091, + "step": 11670 + }, + { + "epoch": 1.8860539754950445, + "grad_norm": 0.6089748740196228, + "learning_rate": 0.0001, + "loss": 69.2073, + "step": 11680 + }, + { + "epoch": 1.8876687995801458, + "grad_norm": 0.6930866837501526, + "learning_rate": 0.0001, + "loss": 68.7227, + "step": 11690 + }, + { + "epoch": 1.8892836236652468, + "grad_norm": 0.6563196182250977, + "learning_rate": 0.0001, + "loss": 68.27, + "step": 11700 + }, + { + "epoch": 1.8908984477503483, + "grad_norm": 0.6173767447471619, + "learning_rate": 0.0001, + "loss": 68.5535, + "step": 11710 + }, + { + "epoch": 1.8925132718354494, + "grad_norm": 0.5498561263084412, + "learning_rate": 0.0001, + "loss": 67.7681, + "step": 11720 + }, + { + "epoch": 1.8941280959205506, + "grad_norm": 0.584891676902771, + "learning_rate": 0.0001, + "loss": 68.1709, + "step": 11730 + }, + { + "epoch": 1.895742920005652, + "grad_norm": 0.7491183280944824, + "learning_rate": 0.0001, + "loss": 65.7435, + "step": 11740 + }, + { + "epoch": 1.897357744090753, + "grad_norm": 0.6519315242767334, + "learning_rate": 0.0001, + "loss": 67.5436, + "step": 11750 + }, + { + "epoch": 1.8989725681758545, + "grad_norm": 0.5740950107574463, + "learning_rate": 0.0001, + "loss": 66.3092, + "step": 11760 + }, + { + "epoch": 1.9005873922609555, + "grad_norm": 0.7978914380073547, + "learning_rate": 0.0001, + "loss": 66.8598, + "step": 11770 + }, + { + "epoch": 1.9022022163460568, + "grad_norm": 0.5536732077598572, + "learning_rate": 0.0001, + "loss": 68.1609, + "step": 11780 + }, + { + "epoch": 1.903817040431158, + "grad_norm": 0.6358754634857178, + "learning_rate": 0.0001, + "loss": 68.2948, + "step": 11790 + }, + { + "epoch": 1.9054318645162591, + "grad_norm": 0.8400535583496094, + "learning_rate": 0.0001, + "loss": 67.7722, + "step": 11800 + }, + { + "epoch": 1.9070466886013606, + "grad_norm": 0.8256928324699402, + "learning_rate": 0.0001, + "loss": 65.8444, + "step": 11810 + }, + { + "epoch": 1.9086615126864617, + "grad_norm": 0.9160122871398926, + "learning_rate": 0.0001, + "loss": 67.4363, + "step": 11820 + }, + { + "epoch": 1.910276336771563, + "grad_norm": 0.6507587432861328, + "learning_rate": 0.0001, + "loss": 65.9515, + "step": 11830 + }, + { + "epoch": 1.9118911608566642, + "grad_norm": 0.5940394401550293, + "learning_rate": 0.0001, + "loss": 65.8083, + "step": 11840 + }, + { + "epoch": 1.9135059849417653, + "grad_norm": 0.7403995990753174, + "learning_rate": 0.0001, + "loss": 66.7136, + "step": 11850 + }, + { + "epoch": 1.9151208090268668, + "grad_norm": 0.6039106845855713, + "learning_rate": 0.0001, + "loss": 67.6913, + "step": 11860 + }, + { + "epoch": 1.9167356331119678, + "grad_norm": 0.9714781641960144, + "learning_rate": 0.0001, + "loss": 67.9658, + "step": 11870 + }, + { + "epoch": 1.918350457197069, + "grad_norm": 0.6880549192428589, + "learning_rate": 0.0001, + "loss": 68.4463, + "step": 11880 + }, + { + "epoch": 1.9199652812821704, + "grad_norm": 0.5718733668327332, + "learning_rate": 0.0001, + "loss": 66.9625, + "step": 11890 + }, + { + "epoch": 1.9215801053672714, + "grad_norm": 0.6208164691925049, + "learning_rate": 0.0001, + "loss": 65.9462, + "step": 11900 + }, + { + "epoch": 1.923194929452373, + "grad_norm": 0.5957098007202148, + "learning_rate": 0.0001, + "loss": 66.9398, + "step": 11910 + }, + { + "epoch": 1.924809753537474, + "grad_norm": 0.6295427680015564, + "learning_rate": 0.0001, + "loss": 66.2246, + "step": 11920 + }, + { + "epoch": 1.9264245776225752, + "grad_norm": 0.8903796672821045, + "learning_rate": 0.0001, + "loss": 66.2999, + "step": 11930 + }, + { + "epoch": 1.9280394017076765, + "grad_norm": 0.6410874128341675, + "learning_rate": 0.0001, + "loss": 67.6857, + "step": 11940 + }, + { + "epoch": 1.9296542257927776, + "grad_norm": 0.5800307393074036, + "learning_rate": 0.0001, + "loss": 66.0943, + "step": 11950 + }, + { + "epoch": 1.931269049877879, + "grad_norm": 0.6834051609039307, + "learning_rate": 0.0001, + "loss": 67.6365, + "step": 11960 + }, + { + "epoch": 1.9328838739629801, + "grad_norm": 0.6112056970596313, + "learning_rate": 0.0001, + "loss": 66.8131, + "step": 11970 + }, + { + "epoch": 1.9344986980480814, + "grad_norm": 0.6657601594924927, + "learning_rate": 0.0001, + "loss": 65.7584, + "step": 11980 + }, + { + "epoch": 1.9361135221331827, + "grad_norm": 0.7570573091506958, + "learning_rate": 0.0001, + "loss": 68.6683, + "step": 11990 + }, + { + "epoch": 1.9377283462182837, + "grad_norm": 0.6231463551521301, + "learning_rate": 0.0001, + "loss": 67.71, + "step": 12000 + }, + { + "epoch": 1.9393431703033852, + "grad_norm": 0.5517798662185669, + "learning_rate": 0.0001, + "loss": 65.5666, + "step": 12010 + }, + { + "epoch": 1.9409579943884863, + "grad_norm": 0.6614659428596497, + "learning_rate": 0.0001, + "loss": 65.4867, + "step": 12020 + }, + { + "epoch": 1.9425728184735875, + "grad_norm": 1.027906060218811, + "learning_rate": 0.0001, + "loss": 65.2863, + "step": 12030 + }, + { + "epoch": 1.9441876425586888, + "grad_norm": 0.5714905261993408, + "learning_rate": 0.0001, + "loss": 67.52, + "step": 12040 + }, + { + "epoch": 1.9458024666437899, + "grad_norm": 0.6580681204795837, + "learning_rate": 0.0001, + "loss": 68.3794, + "step": 12050 + }, + { + "epoch": 1.9474172907288914, + "grad_norm": 0.5728104710578918, + "learning_rate": 0.0001, + "loss": 65.6871, + "step": 12060 + }, + { + "epoch": 1.9490321148139924, + "grad_norm": 0.6828856468200684, + "learning_rate": 0.0001, + "loss": 66.6735, + "step": 12070 + }, + { + "epoch": 1.9506469388990937, + "grad_norm": 0.7448258996009827, + "learning_rate": 0.0001, + "loss": 67.4387, + "step": 12080 + }, + { + "epoch": 1.952261762984195, + "grad_norm": 0.5058867931365967, + "learning_rate": 0.0001, + "loss": 67.3999, + "step": 12090 + }, + { + "epoch": 1.953876587069296, + "grad_norm": 0.5884380340576172, + "learning_rate": 0.0001, + "loss": 66.6189, + "step": 12100 + }, + { + "epoch": 1.9554914111543975, + "grad_norm": 0.7010582685470581, + "learning_rate": 0.0001, + "loss": 66.4048, + "step": 12110 + }, + { + "epoch": 1.9571062352394986, + "grad_norm": 0.556446373462677, + "learning_rate": 0.0001, + "loss": 67.7398, + "step": 12120 + }, + { + "epoch": 1.9587210593245998, + "grad_norm": 0.7856518626213074, + "learning_rate": 0.0001, + "loss": 64.4189, + "step": 12130 + }, + { + "epoch": 1.960335883409701, + "grad_norm": 0.573811948299408, + "learning_rate": 0.0001, + "loss": 66.081, + "step": 12140 + }, + { + "epoch": 1.9619507074948022, + "grad_norm": 0.6113094687461853, + "learning_rate": 0.0001, + "loss": 65.6262, + "step": 12150 + }, + { + "epoch": 1.9635655315799037, + "grad_norm": 0.7158921360969543, + "learning_rate": 0.0001, + "loss": 66.646, + "step": 12160 + }, + { + "epoch": 1.9651803556650047, + "grad_norm": 0.6789208054542542, + "learning_rate": 0.0001, + "loss": 66.9576, + "step": 12170 + }, + { + "epoch": 1.966795179750106, + "grad_norm": 0.6550324559211731, + "learning_rate": 0.0001, + "loss": 67.2544, + "step": 12180 + }, + { + "epoch": 1.9684100038352073, + "grad_norm": 0.6580352187156677, + "learning_rate": 0.0001, + "loss": 67.6755, + "step": 12190 + }, + { + "epoch": 1.9700248279203083, + "grad_norm": 0.7927120327949524, + "learning_rate": 0.0001, + "loss": 66.9819, + "step": 12200 + }, + { + "epoch": 1.9716396520054098, + "grad_norm": 0.5637071132659912, + "learning_rate": 0.0001, + "loss": 67.9308, + "step": 12210 + }, + { + "epoch": 1.9732544760905109, + "grad_norm": 0.5903081297874451, + "learning_rate": 0.0001, + "loss": 67.0279, + "step": 12220 + }, + { + "epoch": 1.9748693001756121, + "grad_norm": 0.7173757553100586, + "learning_rate": 0.0001, + "loss": 66.1328, + "step": 12230 + }, + { + "epoch": 1.9764841242607134, + "grad_norm": 0.7677658796310425, + "learning_rate": 0.0001, + "loss": 66.8729, + "step": 12240 + }, + { + "epoch": 1.9780989483458145, + "grad_norm": 0.5733856558799744, + "learning_rate": 0.0001, + "loss": 65.7204, + "step": 12250 + }, + { + "epoch": 1.979713772430916, + "grad_norm": 0.575779378414154, + "learning_rate": 0.0001, + "loss": 65.082, + "step": 12260 + }, + { + "epoch": 1.981328596516017, + "grad_norm": 0.7983589768409729, + "learning_rate": 0.0001, + "loss": 66.7954, + "step": 12270 + }, + { + "epoch": 1.9829434206011183, + "grad_norm": 0.6657546758651733, + "learning_rate": 0.0001, + "loss": 66.1977, + "step": 12280 + }, + { + "epoch": 1.9845582446862196, + "grad_norm": 0.723057746887207, + "learning_rate": 0.0001, + "loss": 65.2823, + "step": 12290 + }, + { + "epoch": 1.9861730687713206, + "grad_norm": 0.7241561412811279, + "learning_rate": 0.0001, + "loss": 66.5179, + "step": 12300 + }, + { + "epoch": 1.987787892856422, + "grad_norm": 0.7907382249832153, + "learning_rate": 0.0001, + "loss": 67.1835, + "step": 12310 + }, + { + "epoch": 1.9894027169415232, + "grad_norm": 1.0993635654449463, + "learning_rate": 0.0001, + "loss": 66.2844, + "step": 12320 + }, + { + "epoch": 1.9910175410266244, + "grad_norm": 0.6327719688415527, + "learning_rate": 0.0001, + "loss": 64.3111, + "step": 12330 + }, + { + "epoch": 1.9926323651117257, + "grad_norm": 0.5955413579940796, + "learning_rate": 0.0001, + "loss": 67.3991, + "step": 12340 + }, + { + "epoch": 1.9942471891968268, + "grad_norm": 0.7130835056304932, + "learning_rate": 0.0001, + "loss": 67.2216, + "step": 12350 + }, + { + "epoch": 1.9958620132819282, + "grad_norm": 0.5579924583435059, + "learning_rate": 0.0001, + "loss": 65.8894, + "step": 12360 + }, + { + "epoch": 1.9974768373670293, + "grad_norm": 0.7395238876342773, + "learning_rate": 0.0001, + "loss": 66.5182, + "step": 12370 + }, + { + "epoch": 1.9990916614521306, + "grad_norm": 0.6070617437362671, + "learning_rate": 0.0001, + "loss": 67.0429, + "step": 12380 + }, + { + "epoch": 2.0006459296340404, + "grad_norm": 0.6658774018287659, + "learning_rate": 0.0001, + "loss": 65.1403, + "step": 12390 + }, + { + "epoch": 2.002260753719142, + "grad_norm": 0.7703005075454712, + "learning_rate": 0.0001, + "loss": 66.0792, + "step": 12400 + }, + { + "epoch": 2.003875577804243, + "grad_norm": 0.6887365579605103, + "learning_rate": 0.0001, + "loss": 67.5743, + "step": 12410 + }, + { + "epoch": 2.005490401889344, + "grad_norm": 0.6181573271751404, + "learning_rate": 0.0001, + "loss": 65.7725, + "step": 12420 + }, + { + "epoch": 2.0071052259744455, + "grad_norm": 0.6926079988479614, + "learning_rate": 0.0001, + "loss": 66.3818, + "step": 12430 + }, + { + "epoch": 2.0087200500595466, + "grad_norm": 0.9466767311096191, + "learning_rate": 0.0001, + "loss": 66.8674, + "step": 12440 + }, + { + "epoch": 2.010334874144648, + "grad_norm": 0.693327784538269, + "learning_rate": 0.0001, + "loss": 67.1573, + "step": 12450 + }, + { + "epoch": 2.011949698229749, + "grad_norm": 0.6354495286941528, + "learning_rate": 0.0001, + "loss": 67.6646, + "step": 12460 + }, + { + "epoch": 2.01356452231485, + "grad_norm": 0.6671343445777893, + "learning_rate": 0.0001, + "loss": 66.9442, + "step": 12470 + }, + { + "epoch": 2.0151793463999517, + "grad_norm": 0.5987095236778259, + "learning_rate": 0.0001, + "loss": 68.265, + "step": 12480 + }, + { + "epoch": 2.0167941704850527, + "grad_norm": 0.8267726302146912, + "learning_rate": 0.0001, + "loss": 67.7709, + "step": 12490 + }, + { + "epoch": 2.018408994570154, + "grad_norm": 0.5237421989440918, + "learning_rate": 0.0001, + "loss": 64.8426, + "step": 12500 + }, + { + "epoch": 2.0200238186552553, + "grad_norm": 0.5602227449417114, + "learning_rate": 0.0001, + "loss": 65.4287, + "step": 12510 + }, + { + "epoch": 2.0216386427403563, + "grad_norm": 0.6129289269447327, + "learning_rate": 0.0001, + "loss": 65.9583, + "step": 12520 + }, + { + "epoch": 2.023253466825458, + "grad_norm": 0.6047223210334778, + "learning_rate": 0.0001, + "loss": 66.9257, + "step": 12530 + }, + { + "epoch": 2.024868290910559, + "grad_norm": 0.6117433905601501, + "learning_rate": 0.0001, + "loss": 63.0344, + "step": 12540 + }, + { + "epoch": 2.0264831149956604, + "grad_norm": 0.6688846945762634, + "learning_rate": 0.0001, + "loss": 66.2295, + "step": 12550 + }, + { + "epoch": 2.0280979390807614, + "grad_norm": 0.779574453830719, + "learning_rate": 0.0001, + "loss": 67.066, + "step": 12560 + }, + { + "epoch": 2.0297127631658625, + "grad_norm": 0.624839186668396, + "learning_rate": 0.0001, + "loss": 66.7905, + "step": 12570 + }, + { + "epoch": 2.031327587250964, + "grad_norm": 0.5970683693885803, + "learning_rate": 0.0001, + "loss": 67.6997, + "step": 12580 + }, + { + "epoch": 2.032942411336065, + "grad_norm": 0.6005929708480835, + "learning_rate": 0.0001, + "loss": 66.717, + "step": 12590 + }, + { + "epoch": 2.0345572354211665, + "grad_norm": 0.6358321905136108, + "learning_rate": 0.0001, + "loss": 67.8494, + "step": 12600 + }, + { + "epoch": 2.0361720595062676, + "grad_norm": 0.8642273545265198, + "learning_rate": 0.0001, + "loss": 67.1791, + "step": 12610 + }, + { + "epoch": 2.0377868835913686, + "grad_norm": 0.587282121181488, + "learning_rate": 0.0001, + "loss": 66.8169, + "step": 12620 + }, + { + "epoch": 2.03940170767647, + "grad_norm": 0.5977926850318909, + "learning_rate": 0.0001, + "loss": 66.8393, + "step": 12630 + }, + { + "epoch": 2.041016531761571, + "grad_norm": 0.5654817223548889, + "learning_rate": 0.0001, + "loss": 68.8243, + "step": 12640 + }, + { + "epoch": 2.0426313558466727, + "grad_norm": 0.5778902173042297, + "learning_rate": 0.0001, + "loss": 66.8893, + "step": 12650 + }, + { + "epoch": 2.0442461799317737, + "grad_norm": 0.6533665657043457, + "learning_rate": 0.0001, + "loss": 66.6174, + "step": 12660 + }, + { + "epoch": 2.0458610040168748, + "grad_norm": 0.5945654511451721, + "learning_rate": 0.0001, + "loss": 67.8193, + "step": 12670 + }, + { + "epoch": 2.0474758281019763, + "grad_norm": 0.5898523926734924, + "learning_rate": 0.0001, + "loss": 65.6194, + "step": 12680 + }, + { + "epoch": 2.0490906521870773, + "grad_norm": 0.5497958660125732, + "learning_rate": 0.0001, + "loss": 66.4571, + "step": 12690 + }, + { + "epoch": 2.050705476272179, + "grad_norm": 0.6195752024650574, + "learning_rate": 0.0001, + "loss": 66.0555, + "step": 12700 + }, + { + "epoch": 2.05232030035728, + "grad_norm": 0.6400334239006042, + "learning_rate": 0.0001, + "loss": 66.7153, + "step": 12710 + }, + { + "epoch": 2.053935124442381, + "grad_norm": 0.7953216433525085, + "learning_rate": 0.0001, + "loss": 66.5822, + "step": 12720 + }, + { + "epoch": 2.0555499485274824, + "grad_norm": 0.8052420616149902, + "learning_rate": 0.0001, + "loss": 66.4395, + "step": 12730 + }, + { + "epoch": 2.0571647726125835, + "grad_norm": 0.567957878112793, + "learning_rate": 0.0001, + "loss": 65.7762, + "step": 12740 + }, + { + "epoch": 2.058779596697685, + "grad_norm": 0.6517317891120911, + "learning_rate": 0.0001, + "loss": 66.7526, + "step": 12750 + }, + { + "epoch": 2.060394420782786, + "grad_norm": 0.6032442450523376, + "learning_rate": 0.0001, + "loss": 64.2725, + "step": 12760 + }, + { + "epoch": 2.062009244867887, + "grad_norm": 0.6162487268447876, + "learning_rate": 0.0001, + "loss": 65.4797, + "step": 12770 + }, + { + "epoch": 2.0636240689529886, + "grad_norm": 0.7512190937995911, + "learning_rate": 0.0001, + "loss": 65.9602, + "step": 12780 + }, + { + "epoch": 2.0652388930380896, + "grad_norm": 0.7601141929626465, + "learning_rate": 0.0001, + "loss": 66.0806, + "step": 12790 + }, + { + "epoch": 2.066853717123191, + "grad_norm": 0.5737994909286499, + "learning_rate": 0.0001, + "loss": 66.8124, + "step": 12800 + }, + { + "epoch": 2.068468541208292, + "grad_norm": 0.6629387736320496, + "learning_rate": 0.0001, + "loss": 65.8944, + "step": 12810 + }, + { + "epoch": 2.070083365293393, + "grad_norm": 0.6374203562736511, + "learning_rate": 0.0001, + "loss": 67.1589, + "step": 12820 + }, + { + "epoch": 2.0716981893784947, + "grad_norm": 0.6732942461967468, + "learning_rate": 0.0001, + "loss": 67.1901, + "step": 12830 + }, + { + "epoch": 2.0733130134635958, + "grad_norm": 0.6219559907913208, + "learning_rate": 0.0001, + "loss": 66.613, + "step": 12840 + }, + { + "epoch": 2.0749278375486973, + "grad_norm": 0.591238796710968, + "learning_rate": 0.0001, + "loss": 67.6111, + "step": 12850 + }, + { + "epoch": 2.0765426616337983, + "grad_norm": 0.6472163200378418, + "learning_rate": 0.0001, + "loss": 66.305, + "step": 12860 + }, + { + "epoch": 2.0781574857188994, + "grad_norm": 0.6349777579307556, + "learning_rate": 0.0001, + "loss": 65.2865, + "step": 12870 + }, + { + "epoch": 2.079772309804001, + "grad_norm": 0.9257296323776245, + "learning_rate": 0.0001, + "loss": 67.6326, + "step": 12880 + }, + { + "epoch": 2.081387133889102, + "grad_norm": 0.5744728446006775, + "learning_rate": 0.0001, + "loss": 65.7023, + "step": 12890 + }, + { + "epoch": 2.0830019579742034, + "grad_norm": 0.6725465059280396, + "learning_rate": 0.0001, + "loss": 65.1233, + "step": 12900 + }, + { + "epoch": 2.0846167820593045, + "grad_norm": 0.6141199469566345, + "learning_rate": 0.0001, + "loss": 65.5593, + "step": 12910 + }, + { + "epoch": 2.0862316061444055, + "grad_norm": 0.6364370584487915, + "learning_rate": 0.0001, + "loss": 65.3899, + "step": 12920 + }, + { + "epoch": 2.087846430229507, + "grad_norm": 0.5948928594589233, + "learning_rate": 0.0001, + "loss": 63.5976, + "step": 12930 + }, + { + "epoch": 2.089461254314608, + "grad_norm": 0.630434513092041, + "learning_rate": 0.0001, + "loss": 66.764, + "step": 12940 + }, + { + "epoch": 2.091076078399709, + "grad_norm": 0.6178410649299622, + "learning_rate": 0.0001, + "loss": 66.3948, + "step": 12950 + }, + { + "epoch": 2.0926909024848106, + "grad_norm": 0.7695903778076172, + "learning_rate": 0.0001, + "loss": 68.2218, + "step": 12960 + }, + { + "epoch": 2.0943057265699117, + "grad_norm": 0.6712848544120789, + "learning_rate": 0.0001, + "loss": 66.8801, + "step": 12970 + }, + { + "epoch": 2.095920550655013, + "grad_norm": 0.7133443355560303, + "learning_rate": 0.0001, + "loss": 65.343, + "step": 12980 + }, + { + "epoch": 2.097535374740114, + "grad_norm": 0.6836514472961426, + "learning_rate": 0.0001, + "loss": 64.4993, + "step": 12990 + }, + { + "epoch": 2.0991501988252157, + "grad_norm": 0.6734977960586548, + "learning_rate": 0.0001, + "loss": 67.6078, + "step": 13000 + }, + { + "epoch": 2.1007650229103167, + "grad_norm": 0.590630054473877, + "learning_rate": 0.0001, + "loss": 65.4567, + "step": 13010 + }, + { + "epoch": 2.102379846995418, + "grad_norm": 0.7174671292304993, + "learning_rate": 0.0001, + "loss": 66.2743, + "step": 13020 + }, + { + "epoch": 2.1039946710805193, + "grad_norm": 0.5266520977020264, + "learning_rate": 0.0001, + "loss": 65.0063, + "step": 13030 + }, + { + "epoch": 2.1056094951656203, + "grad_norm": 0.550865113735199, + "learning_rate": 0.0001, + "loss": 67.0641, + "step": 13040 + }, + { + "epoch": 2.1072243192507214, + "grad_norm": 0.7976025938987732, + "learning_rate": 0.0001, + "loss": 67.0135, + "step": 13050 + }, + { + "epoch": 2.108839143335823, + "grad_norm": 0.7592310905456543, + "learning_rate": 0.0001, + "loss": 65.9875, + "step": 13060 + }, + { + "epoch": 2.110453967420924, + "grad_norm": 0.7613953948020935, + "learning_rate": 0.0001, + "loss": 66.0958, + "step": 13070 + }, + { + "epoch": 2.1120687915060254, + "grad_norm": 0.6951309442520142, + "learning_rate": 0.0001, + "loss": 65.949, + "step": 13080 + }, + { + "epoch": 2.1136836155911265, + "grad_norm": 0.6439896821975708, + "learning_rate": 0.0001, + "loss": 66.4056, + "step": 13090 + }, + { + "epoch": 2.115298439676228, + "grad_norm": 0.6922518610954285, + "learning_rate": 0.0001, + "loss": 67.537, + "step": 13100 + }, + { + "epoch": 2.116913263761329, + "grad_norm": 0.5330061316490173, + "learning_rate": 0.0001, + "loss": 67.0438, + "step": 13110 + }, + { + "epoch": 2.11852808784643, + "grad_norm": 0.6507022380828857, + "learning_rate": 0.0001, + "loss": 66.9908, + "step": 13120 + }, + { + "epoch": 2.1201429119315316, + "grad_norm": 0.994196891784668, + "learning_rate": 0.0001, + "loss": 63.9818, + "step": 13130 + }, + { + "epoch": 2.1217577360166326, + "grad_norm": 0.7431663870811462, + "learning_rate": 0.0001, + "loss": 65.8259, + "step": 13140 + }, + { + "epoch": 2.1233725601017337, + "grad_norm": 0.5852991938591003, + "learning_rate": 0.0001, + "loss": 65.918, + "step": 13150 + }, + { + "epoch": 2.124987384186835, + "grad_norm": 0.551194429397583, + "learning_rate": 0.0001, + "loss": 64.9907, + "step": 13160 + }, + { + "epoch": 2.1266022082719362, + "grad_norm": 0.6882205605506897, + "learning_rate": 0.0001, + "loss": 66.7266, + "step": 13170 + }, + { + "epoch": 2.1282170323570377, + "grad_norm": 0.8906047940254211, + "learning_rate": 0.0001, + "loss": 65.3441, + "step": 13180 + }, + { + "epoch": 2.129831856442139, + "grad_norm": 0.5340781211853027, + "learning_rate": 0.0001, + "loss": 65.6772, + "step": 13190 + }, + { + "epoch": 2.1314466805272403, + "grad_norm": 0.7657933831214905, + "learning_rate": 0.0001, + "loss": 65.3888, + "step": 13200 + }, + { + "epoch": 2.1330615046123413, + "grad_norm": 1.2086169719696045, + "learning_rate": 0.0001, + "loss": 65.7214, + "step": 13210 + }, + { + "epoch": 2.1346763286974424, + "grad_norm": 0.6794751286506653, + "learning_rate": 0.0001, + "loss": 66.6761, + "step": 13220 + }, + { + "epoch": 2.136291152782544, + "grad_norm": 0.5939100384712219, + "learning_rate": 0.0001, + "loss": 64.1858, + "step": 13230 + }, + { + "epoch": 2.137905976867645, + "grad_norm": 0.581163763999939, + "learning_rate": 0.0001, + "loss": 65.9456, + "step": 13240 + }, + { + "epoch": 2.139520800952746, + "grad_norm": 0.676103949546814, + "learning_rate": 0.0001, + "loss": 65.1919, + "step": 13250 + }, + { + "epoch": 2.1411356250378475, + "grad_norm": 0.6052015423774719, + "learning_rate": 0.0001, + "loss": 67.9872, + "step": 13260 + }, + { + "epoch": 2.1427504491229485, + "grad_norm": 0.5938939452171326, + "learning_rate": 0.0001, + "loss": 64.7395, + "step": 13270 + }, + { + "epoch": 2.14436527320805, + "grad_norm": 0.563115656375885, + "learning_rate": 0.0001, + "loss": 63.9757, + "step": 13280 + }, + { + "epoch": 2.145980097293151, + "grad_norm": 0.5397975444793701, + "learning_rate": 0.0001, + "loss": 65.6596, + "step": 13290 + }, + { + "epoch": 2.1475949213782526, + "grad_norm": 0.6397805213928223, + "learning_rate": 0.0001, + "loss": 67.0618, + "step": 13300 + }, + { + "epoch": 2.1492097454633536, + "grad_norm": 0.678053081035614, + "learning_rate": 0.0001, + "loss": 65.8194, + "step": 13310 + }, + { + "epoch": 2.1508245695484547, + "grad_norm": 0.6067008972167969, + "learning_rate": 0.0001, + "loss": 65.9035, + "step": 13320 + }, + { + "epoch": 2.152439393633556, + "grad_norm": 0.6560337543487549, + "learning_rate": 0.0001, + "loss": 64.9907, + "step": 13330 + }, + { + "epoch": 2.1540542177186572, + "grad_norm": 0.5570183396339417, + "learning_rate": 0.0001, + "loss": 64.5903, + "step": 13340 + }, + { + "epoch": 2.1556690418037583, + "grad_norm": 0.6721981167793274, + "learning_rate": 0.0001, + "loss": 66.5618, + "step": 13350 + }, + { + "epoch": 2.15728386588886, + "grad_norm": 0.7251304388046265, + "learning_rate": 0.0001, + "loss": 66.724, + "step": 13360 + }, + { + "epoch": 2.158898689973961, + "grad_norm": 0.5928503274917603, + "learning_rate": 0.0001, + "loss": 66.6991, + "step": 13370 + }, + { + "epoch": 2.1605135140590623, + "grad_norm": 0.7684288620948792, + "learning_rate": 0.0001, + "loss": 66.4064, + "step": 13380 + }, + { + "epoch": 2.1621283381441634, + "grad_norm": 0.5499702095985413, + "learning_rate": 0.0001, + "loss": 66.7671, + "step": 13390 + }, + { + "epoch": 2.163743162229265, + "grad_norm": 0.5267577767372131, + "learning_rate": 0.0001, + "loss": 65.8112, + "step": 13400 + }, + { + "epoch": 2.165357986314366, + "grad_norm": 0.6214171648025513, + "learning_rate": 0.0001, + "loss": 65.851, + "step": 13410 + }, + { + "epoch": 2.166972810399467, + "grad_norm": 0.5952336192131042, + "learning_rate": 0.0001, + "loss": 65.4166, + "step": 13420 + }, + { + "epoch": 2.1685876344845685, + "grad_norm": 1.3454428911209106, + "learning_rate": 0.0001, + "loss": 64.7546, + "step": 13430 + }, + { + "epoch": 2.1702024585696695, + "grad_norm": 0.6586198210716248, + "learning_rate": 0.0001, + "loss": 65.6818, + "step": 13440 + }, + { + "epoch": 2.1718172826547706, + "grad_norm": 0.8028526306152344, + "learning_rate": 0.0001, + "loss": 67.2677, + "step": 13450 + }, + { + "epoch": 2.173432106739872, + "grad_norm": 0.5949039459228516, + "learning_rate": 0.0001, + "loss": 66.994, + "step": 13460 + }, + { + "epoch": 2.175046930824973, + "grad_norm": 0.6666626930236816, + "learning_rate": 0.0001, + "loss": 66.6509, + "step": 13470 + }, + { + "epoch": 2.1766617549100746, + "grad_norm": 0.5985739827156067, + "learning_rate": 0.0001, + "loss": 65.9211, + "step": 13480 + }, + { + "epoch": 2.1782765789951757, + "grad_norm": 0.6771738529205322, + "learning_rate": 0.0001, + "loss": 65.3477, + "step": 13490 + }, + { + "epoch": 2.1798914030802767, + "grad_norm": 0.6147419214248657, + "learning_rate": 0.0001, + "loss": 64.4142, + "step": 13500 + }, + { + "epoch": 2.1815062271653782, + "grad_norm": 0.6774442791938782, + "learning_rate": 0.0001, + "loss": 64.5831, + "step": 13510 + }, + { + "epoch": 2.1831210512504793, + "grad_norm": 0.5778859853744507, + "learning_rate": 0.0001, + "loss": 64.4449, + "step": 13520 + }, + { + "epoch": 2.184735875335581, + "grad_norm": 0.5697338581085205, + "learning_rate": 0.0001, + "loss": 67.1627, + "step": 13530 + }, + { + "epoch": 2.186350699420682, + "grad_norm": 1.2444179058074951, + "learning_rate": 0.0001, + "loss": 64.7959, + "step": 13540 + }, + { + "epoch": 2.187965523505783, + "grad_norm": 0.6489098072052002, + "learning_rate": 0.0001, + "loss": 65.9926, + "step": 13550 + }, + { + "epoch": 2.1895803475908844, + "grad_norm": 0.6871664524078369, + "learning_rate": 0.0001, + "loss": 67.681, + "step": 13560 + }, + { + "epoch": 2.1911951716759854, + "grad_norm": 0.5581620931625366, + "learning_rate": 0.0001, + "loss": 65.6707, + "step": 13570 + }, + { + "epoch": 2.192809995761087, + "grad_norm": 0.522402286529541, + "learning_rate": 0.0001, + "loss": 65.9014, + "step": 13580 + }, + { + "epoch": 2.194424819846188, + "grad_norm": 0.6149793267250061, + "learning_rate": 0.0001, + "loss": 64.6348, + "step": 13590 + }, + { + "epoch": 2.196039643931289, + "grad_norm": 0.5616180896759033, + "learning_rate": 0.0001, + "loss": 66.3635, + "step": 13600 + }, + { + "epoch": 2.1976544680163905, + "grad_norm": 1.024154543876648, + "learning_rate": 0.0001, + "loss": 66.7156, + "step": 13610 + }, + { + "epoch": 2.1992692921014916, + "grad_norm": 1.035205602645874, + "learning_rate": 0.0001, + "loss": 65.996, + "step": 13620 + }, + { + "epoch": 2.200884116186593, + "grad_norm": 0.5354496240615845, + "learning_rate": 0.0001, + "loss": 66.6107, + "step": 13630 + }, + { + "epoch": 2.202498940271694, + "grad_norm": 0.5648348331451416, + "learning_rate": 0.0001, + "loss": 66.473, + "step": 13640 + }, + { + "epoch": 2.204113764356795, + "grad_norm": 0.6247662901878357, + "learning_rate": 0.0001, + "loss": 66.576, + "step": 13650 + }, + { + "epoch": 2.2057285884418967, + "grad_norm": 0.5871408581733704, + "learning_rate": 0.0001, + "loss": 66.6254, + "step": 13660 + }, + { + "epoch": 2.2073434125269977, + "grad_norm": 0.6394441723823547, + "learning_rate": 0.0001, + "loss": 66.2924, + "step": 13670 + }, + { + "epoch": 2.2089582366120992, + "grad_norm": 0.6017318367958069, + "learning_rate": 0.0001, + "loss": 66.6313, + "step": 13680 + }, + { + "epoch": 2.2105730606972003, + "grad_norm": 0.5696585774421692, + "learning_rate": 0.0001, + "loss": 65.9198, + "step": 13690 + }, + { + "epoch": 2.2121878847823013, + "grad_norm": 0.5966867804527283, + "learning_rate": 0.0001, + "loss": 67.507, + "step": 13700 + }, + { + "epoch": 2.213802708867403, + "grad_norm": 0.6163555979728699, + "learning_rate": 0.0001, + "loss": 67.2444, + "step": 13710 + }, + { + "epoch": 2.215417532952504, + "grad_norm": 0.5670737624168396, + "learning_rate": 0.0001, + "loss": 66.5747, + "step": 13720 + }, + { + "epoch": 2.2170323570376054, + "grad_norm": 0.6289104223251343, + "learning_rate": 0.0001, + "loss": 65.6289, + "step": 13730 + }, + { + "epoch": 2.2186471811227064, + "grad_norm": 0.5649837255477905, + "learning_rate": 0.0001, + "loss": 66.0953, + "step": 13740 + }, + { + "epoch": 2.2202620052078075, + "grad_norm": 0.6066703200340271, + "learning_rate": 0.0001, + "loss": 65.9425, + "step": 13750 + }, + { + "epoch": 2.221876829292909, + "grad_norm": 0.6202117204666138, + "learning_rate": 0.0001, + "loss": 65.3268, + "step": 13760 + }, + { + "epoch": 2.22349165337801, + "grad_norm": 0.6883425116539001, + "learning_rate": 0.0001, + "loss": 64.1026, + "step": 13770 + }, + { + "epoch": 2.2251064774631115, + "grad_norm": 0.5564731955528259, + "learning_rate": 0.0001, + "loss": 65.1203, + "step": 13780 + }, + { + "epoch": 2.2267213015482126, + "grad_norm": 0.48229852318763733, + "learning_rate": 0.0001, + "loss": 65.2423, + "step": 13790 + }, + { + "epoch": 2.2283361256333136, + "grad_norm": 0.549240231513977, + "learning_rate": 0.0001, + "loss": 68.3811, + "step": 13800 + }, + { + "epoch": 2.229950949718415, + "grad_norm": 0.7217127084732056, + "learning_rate": 0.0001, + "loss": 64.9966, + "step": 13810 + }, + { + "epoch": 2.231565773803516, + "grad_norm": 0.8491377830505371, + "learning_rate": 0.0001, + "loss": 65.6468, + "step": 13820 + }, + { + "epoch": 2.2331805978886177, + "grad_norm": 0.5460603833198547, + "learning_rate": 0.0001, + "loss": 65.192, + "step": 13830 + }, + { + "epoch": 2.2347954219737187, + "grad_norm": 0.5784180164337158, + "learning_rate": 0.0001, + "loss": 67.3488, + "step": 13840 + }, + { + "epoch": 2.2364102460588198, + "grad_norm": 0.7575398087501526, + "learning_rate": 0.0001, + "loss": 65.6969, + "step": 13850 + }, + { + "epoch": 2.2380250701439213, + "grad_norm": 0.5111517310142517, + "learning_rate": 0.0001, + "loss": 66.482, + "step": 13860 + }, + { + "epoch": 2.2396398942290223, + "grad_norm": 0.6383832693099976, + "learning_rate": 0.0001, + "loss": 67.4894, + "step": 13870 + }, + { + "epoch": 2.241254718314124, + "grad_norm": 0.5922625064849854, + "learning_rate": 0.0001, + "loss": 66.1718, + "step": 13880 + }, + { + "epoch": 2.242869542399225, + "grad_norm": 5.395338535308838, + "learning_rate": 0.0001, + "loss": 64.7385, + "step": 13890 + }, + { + "epoch": 2.244484366484326, + "grad_norm": 1.4203481674194336, + "learning_rate": 0.0001, + "loss": 64.7676, + "step": 13900 + }, + { + "epoch": 2.2460991905694274, + "grad_norm": 0.6692256331443787, + "learning_rate": 0.0001, + "loss": 65.856, + "step": 13910 + }, + { + "epoch": 2.2477140146545285, + "grad_norm": 0.5868324041366577, + "learning_rate": 0.0001, + "loss": 66.1823, + "step": 13920 + }, + { + "epoch": 2.24932883873963, + "grad_norm": 0.6828433275222778, + "learning_rate": 0.0001, + "loss": 65.7601, + "step": 13930 + }, + { + "epoch": 2.250943662824731, + "grad_norm": 0.5108211040496826, + "learning_rate": 0.0001, + "loss": 63.413, + "step": 13940 + }, + { + "epoch": 2.252558486909832, + "grad_norm": 0.5538185238838196, + "learning_rate": 0.0001, + "loss": 64.9088, + "step": 13950 + }, + { + "epoch": 2.2541733109949336, + "grad_norm": 0.7211924195289612, + "learning_rate": 0.0001, + "loss": 66.0293, + "step": 13960 + }, + { + "epoch": 2.2557881350800346, + "grad_norm": 0.5632925033569336, + "learning_rate": 0.0001, + "loss": 64.8261, + "step": 13970 + }, + { + "epoch": 2.257402959165136, + "grad_norm": 0.574968159198761, + "learning_rate": 0.0001, + "loss": 65.2667, + "step": 13980 + }, + { + "epoch": 2.259017783250237, + "grad_norm": 0.5919519066810608, + "learning_rate": 0.0001, + "loss": 66.2643, + "step": 13990 + }, + { + "epoch": 2.260632607335338, + "grad_norm": 0.6732460260391235, + "learning_rate": 0.0001, + "loss": 64.7524, + "step": 14000 + }, + { + "epoch": 2.2622474314204397, + "grad_norm": 0.5927128791809082, + "learning_rate": 0.0001, + "loss": 65.653, + "step": 14010 + }, + { + "epoch": 2.2638622555055408, + "grad_norm": 0.5900641083717346, + "learning_rate": 0.0001, + "loss": 65.3171, + "step": 14020 + }, + { + "epoch": 2.2654770795906423, + "grad_norm": 0.5498414635658264, + "learning_rate": 0.0001, + "loss": 66.8886, + "step": 14030 + }, + { + "epoch": 2.2670919036757433, + "grad_norm": 0.7032901048660278, + "learning_rate": 0.0001, + "loss": 65.9949, + "step": 14040 + }, + { + "epoch": 2.2687067277608444, + "grad_norm": 0.6699029207229614, + "learning_rate": 0.0001, + "loss": 67.2374, + "step": 14050 + }, + { + "epoch": 2.270321551845946, + "grad_norm": 0.5847286581993103, + "learning_rate": 0.0001, + "loss": 63.996, + "step": 14060 + }, + { + "epoch": 2.271936375931047, + "grad_norm": 0.6173244714736938, + "learning_rate": 0.0001, + "loss": 66.927, + "step": 14070 + }, + { + "epoch": 2.2735512000161484, + "grad_norm": 0.6063929200172424, + "learning_rate": 0.0001, + "loss": 64.1213, + "step": 14080 + }, + { + "epoch": 2.2751660241012495, + "grad_norm": 0.5927698016166687, + "learning_rate": 0.0001, + "loss": 65.5379, + "step": 14090 + }, + { + "epoch": 2.2767808481863505, + "grad_norm": 0.6632462739944458, + "learning_rate": 0.0001, + "loss": 66.4461, + "step": 14100 + }, + { + "epoch": 2.278395672271452, + "grad_norm": 0.6032512187957764, + "learning_rate": 0.0001, + "loss": 65.3191, + "step": 14110 + }, + { + "epoch": 2.280010496356553, + "grad_norm": 0.5799509882926941, + "learning_rate": 0.0001, + "loss": 64.7406, + "step": 14120 + }, + { + "epoch": 2.2816253204416546, + "grad_norm": 0.6965101361274719, + "learning_rate": 0.0001, + "loss": 65.2813, + "step": 14130 + }, + { + "epoch": 2.2832401445267556, + "grad_norm": 0.6447696089744568, + "learning_rate": 0.0001, + "loss": 65.6274, + "step": 14140 + }, + { + "epoch": 2.2848549686118567, + "grad_norm": 0.7639263272285461, + "learning_rate": 0.0001, + "loss": 64.9972, + "step": 14150 + }, + { + "epoch": 2.286469792696958, + "grad_norm": 0.6040478944778442, + "learning_rate": 0.0001, + "loss": 66.1419, + "step": 14160 + }, + { + "epoch": 2.288084616782059, + "grad_norm": 0.6466631889343262, + "learning_rate": 0.0001, + "loss": 66.0022, + "step": 14170 + }, + { + "epoch": 2.2896994408671607, + "grad_norm": 0.6795923709869385, + "learning_rate": 0.0001, + "loss": 65.7562, + "step": 14180 + }, + { + "epoch": 2.2913142649522618, + "grad_norm": 0.5632458329200745, + "learning_rate": 0.0001, + "loss": 64.7196, + "step": 14190 + }, + { + "epoch": 2.292929089037363, + "grad_norm": 1.2942395210266113, + "learning_rate": 0.0001, + "loss": 65.3416, + "step": 14200 + }, + { + "epoch": 2.2945439131224643, + "grad_norm": 0.728187084197998, + "learning_rate": 0.0001, + "loss": 65.3743, + "step": 14210 + }, + { + "epoch": 2.2961587372075654, + "grad_norm": 0.5735196471214294, + "learning_rate": 0.0001, + "loss": 67.1841, + "step": 14220 + }, + { + "epoch": 2.297773561292667, + "grad_norm": 0.5804634690284729, + "learning_rate": 0.0001, + "loss": 65.856, + "step": 14230 + }, + { + "epoch": 2.299388385377768, + "grad_norm": 0.5995981693267822, + "learning_rate": 0.0001, + "loss": 65.7268, + "step": 14240 + }, + { + "epoch": 2.301003209462869, + "grad_norm": 0.4901711940765381, + "learning_rate": 0.0001, + "loss": 66.5355, + "step": 14250 + }, + { + "epoch": 2.3026180335479705, + "grad_norm": 0.6467465162277222, + "learning_rate": 0.0001, + "loss": 66.0457, + "step": 14260 + }, + { + "epoch": 2.3042328576330715, + "grad_norm": 0.6831316351890564, + "learning_rate": 0.0001, + "loss": 65.7363, + "step": 14270 + }, + { + "epoch": 2.305847681718173, + "grad_norm": 0.5308393836021423, + "learning_rate": 0.0001, + "loss": 64.5911, + "step": 14280 + }, + { + "epoch": 2.307462505803274, + "grad_norm": 0.6398120522499084, + "learning_rate": 0.0001, + "loss": 63.9705, + "step": 14290 + }, + { + "epoch": 2.309077329888375, + "grad_norm": 0.5657719969749451, + "learning_rate": 0.0001, + "loss": 65.9251, + "step": 14300 + }, + { + "epoch": 2.3106921539734766, + "grad_norm": 0.5824252367019653, + "learning_rate": 0.0001, + "loss": 64.7748, + "step": 14310 + }, + { + "epoch": 2.3123069780585777, + "grad_norm": 0.6470819115638733, + "learning_rate": 0.0001, + "loss": 65.2014, + "step": 14320 + }, + { + "epoch": 2.313921802143679, + "grad_norm": 0.643173336982727, + "learning_rate": 0.0001, + "loss": 65.9344, + "step": 14330 + }, + { + "epoch": 2.31553662622878, + "grad_norm": 0.6090084314346313, + "learning_rate": 0.0001, + "loss": 66.0823, + "step": 14340 + }, + { + "epoch": 2.3171514503138813, + "grad_norm": 0.627765953540802, + "learning_rate": 0.0001, + "loss": 66.2703, + "step": 14350 + }, + { + "epoch": 2.3187662743989828, + "grad_norm": 0.6484197974205017, + "learning_rate": 0.0001, + "loss": 65.928, + "step": 14360 + }, + { + "epoch": 2.320381098484084, + "grad_norm": 0.653978168964386, + "learning_rate": 0.0001, + "loss": 67.0193, + "step": 14370 + }, + { + "epoch": 2.3219959225691853, + "grad_norm": 0.6991199851036072, + "learning_rate": 0.0001, + "loss": 66.1559, + "step": 14380 + }, + { + "epoch": 2.3236107466542864, + "grad_norm": 0.938056468963623, + "learning_rate": 0.0001, + "loss": 66.6212, + "step": 14390 + }, + { + "epoch": 2.3252255707393874, + "grad_norm": 0.6154502630233765, + "learning_rate": 0.0001, + "loss": 64.5868, + "step": 14400 + }, + { + "epoch": 2.326840394824489, + "grad_norm": 0.6453447341918945, + "learning_rate": 0.0001, + "loss": 67.5199, + "step": 14410 + }, + { + "epoch": 2.32845521890959, + "grad_norm": 0.6678239107131958, + "learning_rate": 0.0001, + "loss": 65.3518, + "step": 14420 + }, + { + "epoch": 2.3300700429946914, + "grad_norm": 0.6608372330665588, + "learning_rate": 0.0001, + "loss": 65.1985, + "step": 14430 + }, + { + "epoch": 2.3316848670797925, + "grad_norm": 0.625511646270752, + "learning_rate": 0.0001, + "loss": 66.9224, + "step": 14440 + }, + { + "epoch": 2.3332996911648936, + "grad_norm": 0.6678557991981506, + "learning_rate": 0.0001, + "loss": 65.8618, + "step": 14450 + }, + { + "epoch": 2.334914515249995, + "grad_norm": 0.6184708476066589, + "learning_rate": 0.0001, + "loss": 66.5419, + "step": 14460 + }, + { + "epoch": 2.336529339335096, + "grad_norm": 0.6668255925178528, + "learning_rate": 0.0001, + "loss": 64.0567, + "step": 14470 + }, + { + "epoch": 2.3381441634201976, + "grad_norm": 0.5756039023399353, + "learning_rate": 0.0001, + "loss": 63.043, + "step": 14480 + }, + { + "epoch": 2.3397589875052986, + "grad_norm": 0.6090635657310486, + "learning_rate": 0.0001, + "loss": 65.8793, + "step": 14490 + }, + { + "epoch": 2.3413738115903997, + "grad_norm": 0.8366266489028931, + "learning_rate": 0.0001, + "loss": 65.1995, + "step": 14500 + }, + { + "epoch": 2.342988635675501, + "grad_norm": 0.6535093188285828, + "learning_rate": 0.0001, + "loss": 63.3831, + "step": 14510 + }, + { + "epoch": 2.3446034597606022, + "grad_norm": 0.9697316884994507, + "learning_rate": 0.0001, + "loss": 65.1754, + "step": 14520 + }, + { + "epoch": 2.3462182838457037, + "grad_norm": 0.7946458458900452, + "learning_rate": 0.0001, + "loss": 65.5404, + "step": 14530 + }, + { + "epoch": 2.347833107930805, + "grad_norm": 0.5621834397315979, + "learning_rate": 0.0001, + "loss": 65.8532, + "step": 14540 + }, + { + "epoch": 2.349447932015906, + "grad_norm": 0.6668163537979126, + "learning_rate": 0.0001, + "loss": 65.0884, + "step": 14550 + }, + { + "epoch": 2.3510627561010073, + "grad_norm": 0.5969575047492981, + "learning_rate": 0.0001, + "loss": 66.5821, + "step": 14560 + }, + { + "epoch": 2.3526775801861084, + "grad_norm": 0.6905517578125, + "learning_rate": 0.0001, + "loss": 65.0928, + "step": 14570 + }, + { + "epoch": 2.35429240427121, + "grad_norm": 0.5888230204582214, + "learning_rate": 0.0001, + "loss": 67.4596, + "step": 14580 + }, + { + "epoch": 2.355907228356311, + "grad_norm": 0.5115624666213989, + "learning_rate": 0.0001, + "loss": 65.771, + "step": 14590 + }, + { + "epoch": 2.357522052441412, + "grad_norm": 0.7888309359550476, + "learning_rate": 0.0001, + "loss": 63.5326, + "step": 14600 + }, + { + "epoch": 2.3591368765265135, + "grad_norm": 0.6653981804847717, + "learning_rate": 0.0001, + "loss": 64.8015, + "step": 14610 + }, + { + "epoch": 2.3607517006116145, + "grad_norm": 0.7898159027099609, + "learning_rate": 0.0001, + "loss": 66.398, + "step": 14620 + }, + { + "epoch": 2.362366524696716, + "grad_norm": 1.02628755569458, + "learning_rate": 0.0001, + "loss": 67.6004, + "step": 14630 + }, + { + "epoch": 2.363981348781817, + "grad_norm": 0.5777941942214966, + "learning_rate": 0.0001, + "loss": 64.9402, + "step": 14640 + }, + { + "epoch": 2.365596172866918, + "grad_norm": 0.7004761695861816, + "learning_rate": 0.0001, + "loss": 64.9344, + "step": 14650 + }, + { + "epoch": 2.3672109969520196, + "grad_norm": 0.6230147480964661, + "learning_rate": 0.0001, + "loss": 64.8412, + "step": 14660 + }, + { + "epoch": 2.3688258210371207, + "grad_norm": 0.7234131097793579, + "learning_rate": 0.0001, + "loss": 65.4454, + "step": 14670 + }, + { + "epoch": 2.370440645122222, + "grad_norm": 0.7054212093353271, + "learning_rate": 0.0001, + "loss": 62.2294, + "step": 14680 + }, + { + "epoch": 2.3720554692073232, + "grad_norm": 0.5377123355865479, + "learning_rate": 0.0001, + "loss": 63.1186, + "step": 14690 + }, + { + "epoch": 2.3736702932924243, + "grad_norm": 0.5648056864738464, + "learning_rate": 0.0001, + "loss": 66.1893, + "step": 14700 + }, + { + "epoch": 2.375285117377526, + "grad_norm": 0.6017388105392456, + "learning_rate": 0.0001, + "loss": 64.8003, + "step": 14710 + }, + { + "epoch": 2.376899941462627, + "grad_norm": 0.592069149017334, + "learning_rate": 0.0001, + "loss": 64.7885, + "step": 14720 + }, + { + "epoch": 2.3785147655477283, + "grad_norm": 0.5448320508003235, + "learning_rate": 0.0001, + "loss": 64.3022, + "step": 14730 + }, + { + "epoch": 2.3801295896328294, + "grad_norm": 0.7200132012367249, + "learning_rate": 0.0001, + "loss": 65.3915, + "step": 14740 + }, + { + "epoch": 2.3817444137179304, + "grad_norm": 0.6497626304626465, + "learning_rate": 0.0001, + "loss": 63.9914, + "step": 14750 + }, + { + "epoch": 2.383359237803032, + "grad_norm": 0.7847583293914795, + "learning_rate": 0.0001, + "loss": 65.7746, + "step": 14760 + }, + { + "epoch": 2.384974061888133, + "grad_norm": 0.6991321444511414, + "learning_rate": 0.0001, + "loss": 65.0386, + "step": 14770 + }, + { + "epoch": 2.386588885973234, + "grad_norm": 0.5715296268463135, + "learning_rate": 0.0001, + "loss": 64.2633, + "step": 14780 + }, + { + "epoch": 2.3882037100583355, + "grad_norm": 0.5266057848930359, + "learning_rate": 0.0001, + "loss": 63.0002, + "step": 14790 + }, + { + "epoch": 2.3898185341434366, + "grad_norm": 0.7040116786956787, + "learning_rate": 0.0001, + "loss": 66.2895, + "step": 14800 + }, + { + "epoch": 2.391433358228538, + "grad_norm": 0.6963793635368347, + "learning_rate": 0.0001, + "loss": 64.7487, + "step": 14810 + }, + { + "epoch": 2.393048182313639, + "grad_norm": 0.5361505746841431, + "learning_rate": 0.0001, + "loss": 66.6834, + "step": 14820 + }, + { + "epoch": 2.3946630063987406, + "grad_norm": 0.8016838431358337, + "learning_rate": 0.0001, + "loss": 64.3359, + "step": 14830 + }, + { + "epoch": 2.3962778304838417, + "grad_norm": 0.5636866092681885, + "learning_rate": 0.0001, + "loss": 65.2468, + "step": 14840 + }, + { + "epoch": 2.3978926545689427, + "grad_norm": 0.6926373839378357, + "learning_rate": 0.0001, + "loss": 66.3353, + "step": 14850 + }, + { + "epoch": 2.3995074786540442, + "grad_norm": 0.6361585855484009, + "learning_rate": 0.0001, + "loss": 64.8466, + "step": 14860 + }, + { + "epoch": 2.4011223027391453, + "grad_norm": 0.6232298612594604, + "learning_rate": 0.0001, + "loss": 65.1253, + "step": 14870 + }, + { + "epoch": 2.4027371268242463, + "grad_norm": 0.5644697546958923, + "learning_rate": 0.0001, + "loss": 67.1739, + "step": 14880 + }, + { + "epoch": 2.404351950909348, + "grad_norm": 0.5655877590179443, + "learning_rate": 0.0001, + "loss": 65.1823, + "step": 14890 + }, + { + "epoch": 2.405966774994449, + "grad_norm": 0.8255041837692261, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 14900 + }, + { + "epoch": 2.4075815990795504, + "grad_norm": 0.856689989566803, + "learning_rate": 0.0001, + "loss": 64.4978, + "step": 14910 + }, + { + "epoch": 2.4091964231646514, + "grad_norm": 0.6280230283737183, + "learning_rate": 0.0001, + "loss": 66.3989, + "step": 14920 + }, + { + "epoch": 2.410811247249753, + "grad_norm": 0.5739511251449585, + "learning_rate": 0.0001, + "loss": 64.6365, + "step": 14930 + }, + { + "epoch": 2.412426071334854, + "grad_norm": 0.6244319677352905, + "learning_rate": 0.0001, + "loss": 65.5078, + "step": 14940 + }, + { + "epoch": 2.414040895419955, + "grad_norm": 0.6932142376899719, + "learning_rate": 0.0001, + "loss": 65.1869, + "step": 14950 + }, + { + "epoch": 2.4156557195050565, + "grad_norm": 0.5393000245094299, + "learning_rate": 0.0001, + "loss": 65.553, + "step": 14960 + }, + { + "epoch": 2.4172705435901576, + "grad_norm": 0.6555871963500977, + "learning_rate": 0.0001, + "loss": 66.625, + "step": 14970 + }, + { + "epoch": 2.4188853676752586, + "grad_norm": 0.6515706181526184, + "learning_rate": 0.0001, + "loss": 64.876, + "step": 14980 + }, + { + "epoch": 2.42050019176036, + "grad_norm": 0.6380309462547302, + "learning_rate": 0.0001, + "loss": 63.1801, + "step": 14990 + }, + { + "epoch": 2.422115015845461, + "grad_norm": 0.8313520550727844, + "learning_rate": 0.0001, + "loss": 66.2692, + "step": 15000 + }, + { + "epoch": 2.4237298399305627, + "grad_norm": 0.6654364466667175, + "learning_rate": 0.0001, + "loss": 65.4955, + "step": 15010 + }, + { + "epoch": 2.4253446640156637, + "grad_norm": 0.6382327675819397, + "learning_rate": 0.0001, + "loss": 65.0912, + "step": 15020 + }, + { + "epoch": 2.4269594881007652, + "grad_norm": 0.5685209035873413, + "learning_rate": 0.0001, + "loss": 64.8058, + "step": 15030 + }, + { + "epoch": 2.4285743121858663, + "grad_norm": 0.6129642724990845, + "learning_rate": 0.0001, + "loss": 65.8475, + "step": 15040 + }, + { + "epoch": 2.4301891362709673, + "grad_norm": 0.6892847418785095, + "learning_rate": 0.0001, + "loss": 64.7247, + "step": 15050 + }, + { + "epoch": 2.431803960356069, + "grad_norm": 0.6307751536369324, + "learning_rate": 0.0001, + "loss": 64.0042, + "step": 15060 + }, + { + "epoch": 2.43341878444117, + "grad_norm": 0.5506501197814941, + "learning_rate": 0.0001, + "loss": 64.5555, + "step": 15070 + }, + { + "epoch": 2.435033608526271, + "grad_norm": 0.5580377578735352, + "learning_rate": 0.0001, + "loss": 65.0344, + "step": 15080 + }, + { + "epoch": 2.4366484326113724, + "grad_norm": 0.5712753534317017, + "learning_rate": 0.0001, + "loss": 66.6778, + "step": 15090 + }, + { + "epoch": 2.4382632566964735, + "grad_norm": 0.7153586745262146, + "learning_rate": 0.0001, + "loss": 65.893, + "step": 15100 + }, + { + "epoch": 2.439878080781575, + "grad_norm": 0.6946969628334045, + "learning_rate": 0.0001, + "loss": 64.1815, + "step": 15110 + }, + { + "epoch": 2.441492904866676, + "grad_norm": 0.608729898929596, + "learning_rate": 0.0001, + "loss": 64.8054, + "step": 15120 + }, + { + "epoch": 2.4431077289517775, + "grad_norm": 0.6703302264213562, + "learning_rate": 0.0001, + "loss": 65.3736, + "step": 15130 + }, + { + "epoch": 2.4447225530368786, + "grad_norm": 0.736193835735321, + "learning_rate": 0.0001, + "loss": 66.7051, + "step": 15140 + }, + { + "epoch": 2.4463373771219796, + "grad_norm": 0.48381003737449646, + "learning_rate": 0.0001, + "loss": 64.3723, + "step": 15150 + }, + { + "epoch": 2.447952201207081, + "grad_norm": 0.7487664222717285, + "learning_rate": 0.0001, + "loss": 64.4967, + "step": 15160 + }, + { + "epoch": 2.449567025292182, + "grad_norm": 0.7033581733703613, + "learning_rate": 0.0001, + "loss": 66.2185, + "step": 15170 + }, + { + "epoch": 2.4511818493772832, + "grad_norm": 0.5489983558654785, + "learning_rate": 0.0001, + "loss": 64.9698, + "step": 15180 + }, + { + "epoch": 2.4527966734623847, + "grad_norm": 0.6467112898826599, + "learning_rate": 0.0001, + "loss": 64.5987, + "step": 15190 + }, + { + "epoch": 2.4544114975474858, + "grad_norm": 0.6499149799346924, + "learning_rate": 0.0001, + "loss": 65.757, + "step": 15200 + }, + { + "epoch": 2.4560263216325873, + "grad_norm": 0.7622562050819397, + "learning_rate": 0.0001, + "loss": 66.4642, + "step": 15210 + }, + { + "epoch": 2.4576411457176883, + "grad_norm": 0.8985865712165833, + "learning_rate": 0.0001, + "loss": 64.5804, + "step": 15220 + }, + { + "epoch": 2.45925596980279, + "grad_norm": 0.6116635203361511, + "learning_rate": 0.0001, + "loss": 64.8022, + "step": 15230 + }, + { + "epoch": 2.460870793887891, + "grad_norm": 0.6313971281051636, + "learning_rate": 0.0001, + "loss": 63.4837, + "step": 15240 + }, + { + "epoch": 2.462485617972992, + "grad_norm": 0.7002763152122498, + "learning_rate": 0.0001, + "loss": 64.7201, + "step": 15250 + }, + { + "epoch": 2.4641004420580934, + "grad_norm": 0.6030148863792419, + "learning_rate": 0.0001, + "loss": 65.0987, + "step": 15260 + }, + { + "epoch": 2.4657152661431945, + "grad_norm": 0.5838733911514282, + "learning_rate": 0.0001, + "loss": 64.9182, + "step": 15270 + }, + { + "epoch": 2.4673300902282955, + "grad_norm": 1.1749393939971924, + "learning_rate": 0.0001, + "loss": 63.4387, + "step": 15280 + }, + { + "epoch": 2.468944914313397, + "grad_norm": 0.5215787291526794, + "learning_rate": 0.0001, + "loss": 64.7365, + "step": 15290 + }, + { + "epoch": 2.470559738398498, + "grad_norm": 0.5576916933059692, + "learning_rate": 0.0001, + "loss": 65.644, + "step": 15300 + }, + { + "epoch": 2.4721745624835996, + "grad_norm": 0.6760591864585876, + "learning_rate": 0.0001, + "loss": 64.729, + "step": 15310 + }, + { + "epoch": 2.4737893865687006, + "grad_norm": 0.7162330150604248, + "learning_rate": 0.0001, + "loss": 66.8289, + "step": 15320 + }, + { + "epoch": 2.475404210653802, + "grad_norm": 0.5687427520751953, + "learning_rate": 0.0001, + "loss": 63.9522, + "step": 15330 + }, + { + "epoch": 2.477019034738903, + "grad_norm": 0.5663572549819946, + "learning_rate": 0.0001, + "loss": 65.7151, + "step": 15340 + }, + { + "epoch": 2.478633858824004, + "grad_norm": 0.6018327474594116, + "learning_rate": 0.0001, + "loss": 64.6599, + "step": 15350 + }, + { + "epoch": 2.4802486829091057, + "grad_norm": 0.6908984184265137, + "learning_rate": 0.0001, + "loss": 65.3083, + "step": 15360 + }, + { + "epoch": 2.4818635069942068, + "grad_norm": 0.6582157015800476, + "learning_rate": 0.0001, + "loss": 64.6219, + "step": 15370 + }, + { + "epoch": 2.483478331079308, + "grad_norm": 0.7800308465957642, + "learning_rate": 0.0001, + "loss": 66.6917, + "step": 15380 + }, + { + "epoch": 2.4850931551644093, + "grad_norm": 0.5621150732040405, + "learning_rate": 0.0001, + "loss": 66.6579, + "step": 15390 + }, + { + "epoch": 2.4867079792495104, + "grad_norm": 0.6318013668060303, + "learning_rate": 0.0001, + "loss": 64.8888, + "step": 15400 + }, + { + "epoch": 2.488322803334612, + "grad_norm": 0.522668719291687, + "learning_rate": 0.0001, + "loss": 64.6468, + "step": 15410 + }, + { + "epoch": 2.489937627419713, + "grad_norm": 0.5940443873405457, + "learning_rate": 0.0001, + "loss": 65.0119, + "step": 15420 + }, + { + "epoch": 2.4915524515048144, + "grad_norm": 0.5932700037956238, + "learning_rate": 0.0001, + "loss": 66.2562, + "step": 15430 + }, + { + "epoch": 2.4931672755899155, + "grad_norm": 0.5753180980682373, + "learning_rate": 0.0001, + "loss": 65.6223, + "step": 15440 + }, + { + "epoch": 2.4947820996750165, + "grad_norm": 1.0136449337005615, + "learning_rate": 0.0001, + "loss": 65.4779, + "step": 15450 + }, + { + "epoch": 2.496396923760118, + "grad_norm": 0.6616164445877075, + "learning_rate": 0.0001, + "loss": 65.2368, + "step": 15460 + }, + { + "epoch": 2.498011747845219, + "grad_norm": 0.584385871887207, + "learning_rate": 0.0001, + "loss": 63.5195, + "step": 15470 + }, + { + "epoch": 2.49962657193032, + "grad_norm": 0.6892259120941162, + "learning_rate": 0.0001, + "loss": 63.3364, + "step": 15480 + }, + { + "epoch": 2.5012413960154216, + "grad_norm": 0.594386637210846, + "learning_rate": 0.0001, + "loss": 65.062, + "step": 15490 + }, + { + "epoch": 2.5028562201005227, + "grad_norm": 0.6604944467544556, + "learning_rate": 0.0001, + "loss": 65.3744, + "step": 15500 + }, + { + "epoch": 2.504471044185624, + "grad_norm": 0.5402690768241882, + "learning_rate": 0.0001, + "loss": 65.0583, + "step": 15510 + }, + { + "epoch": 2.506085868270725, + "grad_norm": 0.4574640989303589, + "learning_rate": 0.0001, + "loss": 64.3828, + "step": 15520 + }, + { + "epoch": 2.5077006923558267, + "grad_norm": 0.62662672996521, + "learning_rate": 0.0001, + "loss": 62.774, + "step": 15530 + }, + { + "epoch": 2.5093155164409278, + "grad_norm": 1.1336137056350708, + "learning_rate": 0.0001, + "loss": 63.9481, + "step": 15540 + }, + { + "epoch": 2.510930340526029, + "grad_norm": 0.7171993255615234, + "learning_rate": 0.0001, + "loss": 63.2949, + "step": 15550 + }, + { + "epoch": 2.5125451646111303, + "grad_norm": 0.6186808943748474, + "learning_rate": 0.0001, + "loss": 65.3805, + "step": 15560 + }, + { + "epoch": 2.5141599886962314, + "grad_norm": 0.5631883144378662, + "learning_rate": 0.0001, + "loss": 65.9464, + "step": 15570 + }, + { + "epoch": 2.5157748127813324, + "grad_norm": 0.6015043258666992, + "learning_rate": 0.0001, + "loss": 66.5938, + "step": 15580 + }, + { + "epoch": 2.517389636866434, + "grad_norm": 0.63025963306427, + "learning_rate": 0.0001, + "loss": 65.6977, + "step": 15590 + }, + { + "epoch": 2.519004460951535, + "grad_norm": 0.6715611815452576, + "learning_rate": 0.0001, + "loss": 66.1286, + "step": 15600 + }, + { + "epoch": 2.5206192850366365, + "grad_norm": 0.6191041469573975, + "learning_rate": 0.0001, + "loss": 64.257, + "step": 15610 + }, + { + "epoch": 2.5222341091217375, + "grad_norm": 0.5771703720092773, + "learning_rate": 0.0001, + "loss": 64.1293, + "step": 15620 + }, + { + "epoch": 2.523848933206839, + "grad_norm": 0.6421854496002197, + "learning_rate": 0.0001, + "loss": 64.27, + "step": 15630 + }, + { + "epoch": 2.52546375729194, + "grad_norm": 0.5963718295097351, + "learning_rate": 0.0001, + "loss": 66.5889, + "step": 15640 + }, + { + "epoch": 2.527078581377041, + "grad_norm": 0.5750302672386169, + "learning_rate": 0.0001, + "loss": 64.4224, + "step": 15650 + }, + { + "epoch": 2.5286934054621426, + "grad_norm": 0.558082640171051, + "learning_rate": 0.0001, + "loss": 64.3853, + "step": 15660 + }, + { + "epoch": 2.5303082295472437, + "grad_norm": 0.6801859736442566, + "learning_rate": 0.0001, + "loss": 65.5297, + "step": 15670 + }, + { + "epoch": 2.5319230536323447, + "grad_norm": 0.7200396656990051, + "learning_rate": 0.0001, + "loss": 66.7467, + "step": 15680 + }, + { + "epoch": 2.533537877717446, + "grad_norm": 0.5736697316169739, + "learning_rate": 0.0001, + "loss": 62.2177, + "step": 15690 + }, + { + "epoch": 2.5351527018025473, + "grad_norm": 1.6533524990081787, + "learning_rate": 0.0001, + "loss": 65.245, + "step": 15700 + }, + { + "epoch": 2.5367675258876488, + "grad_norm": 0.578233003616333, + "learning_rate": 0.0001, + "loss": 67.0796, + "step": 15710 + }, + { + "epoch": 2.53838234997275, + "grad_norm": 0.5405007600784302, + "learning_rate": 0.0001, + "loss": 65.4651, + "step": 15720 + }, + { + "epoch": 2.5399971740578513, + "grad_norm": 0.65571528673172, + "learning_rate": 0.0001, + "loss": 65.8385, + "step": 15730 + }, + { + "epoch": 2.5416119981429524, + "grad_norm": 0.5576966404914856, + "learning_rate": 0.0001, + "loss": 64.3478, + "step": 15740 + }, + { + "epoch": 2.5432268222280534, + "grad_norm": 0.7222445011138916, + "learning_rate": 0.0001, + "loss": 65.8258, + "step": 15750 + }, + { + "epoch": 2.544841646313155, + "grad_norm": 0.6464501619338989, + "learning_rate": 0.0001, + "loss": 63.535, + "step": 15760 + }, + { + "epoch": 2.546456470398256, + "grad_norm": 0.5374952554702759, + "learning_rate": 0.0001, + "loss": 63.7492, + "step": 15770 + }, + { + "epoch": 2.548071294483357, + "grad_norm": 0.6572895646095276, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 15780 + }, + { + "epoch": 2.5496861185684585, + "grad_norm": 0.578153133392334, + "learning_rate": 0.0001, + "loss": 64.311, + "step": 15790 + }, + { + "epoch": 2.5513009426535596, + "grad_norm": 0.6136942505836487, + "learning_rate": 0.0001, + "loss": 64.6734, + "step": 15800 + }, + { + "epoch": 2.552915766738661, + "grad_norm": 0.8686643242835999, + "learning_rate": 0.0001, + "loss": 64.5421, + "step": 15810 + }, + { + "epoch": 2.554530590823762, + "grad_norm": 0.8858036994934082, + "learning_rate": 0.0001, + "loss": 66.358, + "step": 15820 + }, + { + "epoch": 2.5561454149088636, + "grad_norm": 0.6373804807662964, + "learning_rate": 0.0001, + "loss": 63.7582, + "step": 15830 + }, + { + "epoch": 2.5577602389939647, + "grad_norm": 1.5556457042694092, + "learning_rate": 0.0001, + "loss": 63.8109, + "step": 15840 + }, + { + "epoch": 2.5593750630790657, + "grad_norm": 0.5945340394973755, + "learning_rate": 0.0001, + "loss": 64.7864, + "step": 15850 + }, + { + "epoch": 2.560989887164167, + "grad_norm": 0.6172908544540405, + "learning_rate": 0.0001, + "loss": 62.7738, + "step": 15860 + }, + { + "epoch": 2.5626047112492683, + "grad_norm": 0.6360666155815125, + "learning_rate": 0.0001, + "loss": 63.4814, + "step": 15870 + }, + { + "epoch": 2.5642195353343693, + "grad_norm": 0.7986658811569214, + "learning_rate": 0.0001, + "loss": 64.3178, + "step": 15880 + }, + { + "epoch": 2.565834359419471, + "grad_norm": 0.5716906785964966, + "learning_rate": 0.0001, + "loss": 64.371, + "step": 15890 + }, + { + "epoch": 2.567449183504572, + "grad_norm": 0.5859067440032959, + "learning_rate": 0.0001, + "loss": 64.0359, + "step": 15900 + }, + { + "epoch": 2.5690640075896733, + "grad_norm": 0.6288742423057556, + "learning_rate": 0.0001, + "loss": 64.4613, + "step": 15910 + }, + { + "epoch": 2.5706788316747744, + "grad_norm": 0.9052578210830688, + "learning_rate": 0.0001, + "loss": 66.3599, + "step": 15920 + }, + { + "epoch": 2.572293655759876, + "grad_norm": 0.5532401204109192, + "learning_rate": 0.0001, + "loss": 64.1753, + "step": 15930 + }, + { + "epoch": 2.573908479844977, + "grad_norm": 0.5749264359474182, + "learning_rate": 0.0001, + "loss": 63.4118, + "step": 15940 + }, + { + "epoch": 2.575523303930078, + "grad_norm": 2.064448356628418, + "learning_rate": 0.0001, + "loss": 63.2922, + "step": 15950 + }, + { + "epoch": 2.5771381280151795, + "grad_norm": 0.6260086297988892, + "learning_rate": 0.0001, + "loss": 65.2183, + "step": 15960 + }, + { + "epoch": 2.5787529521002805, + "grad_norm": 0.5910785794258118, + "learning_rate": 0.0001, + "loss": 63.3134, + "step": 15970 + }, + { + "epoch": 2.5803677761853816, + "grad_norm": 0.5972382426261902, + "learning_rate": 0.0001, + "loss": 64.3669, + "step": 15980 + }, + { + "epoch": 2.581982600270483, + "grad_norm": 0.5171772837638855, + "learning_rate": 0.0001, + "loss": 64.3819, + "step": 15990 + }, + { + "epoch": 2.583597424355584, + "grad_norm": 0.5792955160140991, + "learning_rate": 0.0001, + "loss": 65.8632, + "step": 16000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9591100649023996e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6a151f546555c78ebc502dd7b703b665f0672d6f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35235df8408202e91954a585851d27187bafaf4e1d75c4e63c51fbf721136ff5 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-16000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/latest b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/latest new file mode 100644 index 0000000000000000000000000000000000000000..b73f60d7a0f5e4a77d445820eafbd0d5e13129a2 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/latest @@ -0,0 +1 @@ +global_step18000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..14ee08869f0c94df416941c1493b24c47250a459 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50e1afafe1062cea493b06c0c852549e1d509dc45728ab9a8afb8e5ba05c7b6b +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..f84b161a336a4392a29cb821baf29644a062fb89 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6c52f3900e03602809fd641e0a3ff189b9017023f611acf66567d95d7ec57cf +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..80e724081795c72a499c2c49135bade1e4d72f32 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:700680853ff94bd9110cd89541efcc3c85cd57f03542cb23a2b21ca343750471 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..4183773eedf44142bcf09b05c17f405346cd9607 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee285c11683720027da3cb2204aa10217bb4ce6c367ba03b05d0987417a0da8b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..f4a5dd2fa47191423721c15837eeee07b6c39f24 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ba42f0421aa4153327998b86d81f1108b4eebbdd0836d4416b62380d35cc5aa +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..25e57c128e88ca3192c346707e0954f76f85ab94 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96721606ee7045acd811cc417531278d21a8c69b71d92bc54bc4a809365a2d6d +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..cdbf9421b1f7db8235f62d3c1a09d153de058fb2 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86b80c9ebae65e15510f7c5be989254fb81ba37825f3b709334d07073ace19d9 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..38c5d05dfd3c16370780102f628a52f3fc6c2370 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59945708863816e76e34f0caad06b215df445846f0adc0c813ce3a4a457c4185 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..153eed094c64a0b0724d1ee10c00a0d85ebca330 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84929720191b864c8c7b7d0fac1a692a285980f2d789e75775db1e7b102924f3 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..21565de482b40f006d37963ebafcdfe26c25d297 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6811f4eff35f13019fe7a0f3e11af3e201050fd538642f614a8e73a804630d2 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f316974114f68985c76cb2d1ecafa62ef239fb56 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/trainer_state.json @@ -0,0 +1,12634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.90656224137583, + "eval_steps": 2000, + "global_step": 18000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0016148240851012292, + "grad_norm": 43.90101623535156, + "learning_rate": 0.0001, + "loss": 307.2084, + "step": 10 + }, + { + "epoch": 0.0032296481702024584, + "grad_norm": 19.106313705444336, + "learning_rate": 0.0001, + "loss": 320.2794, + "step": 20 + }, + { + "epoch": 0.004844472255303688, + "grad_norm": 5.771847248077393, + "learning_rate": 0.0001, + "loss": 247.4977, + "step": 30 + }, + { + "epoch": 0.006459296340404917, + "grad_norm": 2.630659580230713, + "learning_rate": 0.0001, + "loss": 229.9593, + "step": 40 + }, + { + "epoch": 0.008074120425506147, + "grad_norm": 2.487499475479126, + "learning_rate": 0.0001, + "loss": 228.65, + "step": 50 + }, + { + "epoch": 0.009688944510607376, + "grad_norm": 2.078263521194458, + "learning_rate": 0.0001, + "loss": 227.648, + "step": 60 + }, + { + "epoch": 0.011303768595708604, + "grad_norm": 3.105417490005493, + "learning_rate": 0.0001, + "loss": 227.9345, + "step": 70 + }, + { + "epoch": 0.012918592680809833, + "grad_norm": 2.2145848274230957, + "learning_rate": 0.0001, + "loss": 227.3662, + "step": 80 + }, + { + "epoch": 0.014533416765911064, + "grad_norm": 1.9644615650177002, + "learning_rate": 0.0001, + "loss": 226.9145, + "step": 90 + }, + { + "epoch": 0.016148240851012293, + "grad_norm": 1.867382287979126, + "learning_rate": 0.0001, + "loss": 227.5703, + "step": 100 + }, + { + "epoch": 0.017763064936113522, + "grad_norm": 1.4613710641860962, + "learning_rate": 0.0001, + "loss": 227.1107, + "step": 110 + }, + { + "epoch": 0.01937788902121475, + "grad_norm": 2.8953869342803955, + "learning_rate": 0.0001, + "loss": 226.8185, + "step": 120 + }, + { + "epoch": 0.02099271310631598, + "grad_norm": 1.8446482419967651, + "learning_rate": 0.0001, + "loss": 226.9624, + "step": 130 + }, + { + "epoch": 0.02260753719141721, + "grad_norm": 1.3048362731933594, + "learning_rate": 0.0001, + "loss": 226.8404, + "step": 140 + }, + { + "epoch": 0.024222361276518438, + "grad_norm": 1.7100074291229248, + "learning_rate": 0.0001, + "loss": 226.1071, + "step": 150 + }, + { + "epoch": 0.025837185361619667, + "grad_norm": 1.7875330448150635, + "learning_rate": 0.0001, + "loss": 226.0764, + "step": 160 + }, + { + "epoch": 0.0274520094467209, + "grad_norm": 1.7503769397735596, + "learning_rate": 0.0001, + "loss": 226.0772, + "step": 170 + }, + { + "epoch": 0.029066833531822128, + "grad_norm": 2.7083983421325684, + "learning_rate": 0.0001, + "loss": 224.7667, + "step": 180 + }, + { + "epoch": 0.030681657616923357, + "grad_norm": 1.8192371129989624, + "learning_rate": 0.0001, + "loss": 223.6494, + "step": 190 + }, + { + "epoch": 0.032296481702024586, + "grad_norm": 3.210358142852783, + "learning_rate": 0.0001, + "loss": 223.3545, + "step": 200 + }, + { + "epoch": 0.03391130578712581, + "grad_norm": 5.819669246673584, + "learning_rate": 0.0001, + "loss": 222.1732, + "step": 210 + }, + { + "epoch": 0.035526129872227044, + "grad_norm": 4.185793399810791, + "learning_rate": 0.0001, + "loss": 221.6866, + "step": 220 + }, + { + "epoch": 0.03714095395732828, + "grad_norm": 2.7514216899871826, + "learning_rate": 0.0001, + "loss": 221.0769, + "step": 230 + }, + { + "epoch": 0.0387557780424295, + "grad_norm": 1.5478988885879517, + "learning_rate": 0.0001, + "loss": 219.7232, + "step": 240 + }, + { + "epoch": 0.040370602127530734, + "grad_norm": 1.4958223104476929, + "learning_rate": 0.0001, + "loss": 218.5454, + "step": 250 + }, + { + "epoch": 0.04198542621263196, + "grad_norm": 6.36635684967041, + "learning_rate": 0.0001, + "loss": 218.5061, + "step": 260 + }, + { + "epoch": 0.04360025029773319, + "grad_norm": 3.029601812362671, + "learning_rate": 0.0001, + "loss": 216.3466, + "step": 270 + }, + { + "epoch": 0.04521507438283442, + "grad_norm": 2.6436100006103516, + "learning_rate": 0.0001, + "loss": 214.0003, + "step": 280 + }, + { + "epoch": 0.04682989846793565, + "grad_norm": 5.329655647277832, + "learning_rate": 0.0001, + "loss": 212.9169, + "step": 290 + }, + { + "epoch": 0.048444722553036876, + "grad_norm": 4.930875301361084, + "learning_rate": 0.0001, + "loss": 209.7737, + "step": 300 + }, + { + "epoch": 0.05005954663813811, + "grad_norm": 18.902585983276367, + "learning_rate": 0.0001, + "loss": 209.4267, + "step": 310 + }, + { + "epoch": 0.051674370723239334, + "grad_norm": 5.364975929260254, + "learning_rate": 0.0001, + "loss": 206.49, + "step": 320 + }, + { + "epoch": 0.053289194808340566, + "grad_norm": 3.1421146392822266, + "learning_rate": 0.0001, + "loss": 204.8132, + "step": 330 + }, + { + "epoch": 0.0549040188934418, + "grad_norm": 3.399320602416992, + "learning_rate": 0.0001, + "loss": 202.4785, + "step": 340 + }, + { + "epoch": 0.056518842978543024, + "grad_norm": 5.137377738952637, + "learning_rate": 0.0001, + "loss": 199.5143, + "step": 350 + }, + { + "epoch": 0.058133667063644257, + "grad_norm": 5.4967522621154785, + "learning_rate": 0.0001, + "loss": 196.2809, + "step": 360 + }, + { + "epoch": 0.05974849114874548, + "grad_norm": 6.435070037841797, + "learning_rate": 0.0001, + "loss": 196.8831, + "step": 370 + }, + { + "epoch": 0.061363315233846714, + "grad_norm": 5.7334370613098145, + "learning_rate": 0.0001, + "loss": 193.141, + "step": 380 + }, + { + "epoch": 0.06297813931894794, + "grad_norm": 3.6522979736328125, + "learning_rate": 0.0001, + "loss": 191.5819, + "step": 390 + }, + { + "epoch": 0.06459296340404917, + "grad_norm": 2.614051103591919, + "learning_rate": 0.0001, + "loss": 190.046, + "step": 400 + }, + { + "epoch": 0.0662077874891504, + "grad_norm": 4.845924377441406, + "learning_rate": 0.0001, + "loss": 188.7121, + "step": 410 + }, + { + "epoch": 0.06782261157425162, + "grad_norm": 4.319681644439697, + "learning_rate": 0.0001, + "loss": 185.3697, + "step": 420 + }, + { + "epoch": 0.06943743565935286, + "grad_norm": 2.7018792629241943, + "learning_rate": 0.0001, + "loss": 183.5171, + "step": 430 + }, + { + "epoch": 0.07105225974445409, + "grad_norm": 2.6407113075256348, + "learning_rate": 0.0001, + "loss": 181.245, + "step": 440 + }, + { + "epoch": 0.07266708382955532, + "grad_norm": 3.314251184463501, + "learning_rate": 0.0001, + "loss": 179.3874, + "step": 450 + }, + { + "epoch": 0.07428190791465655, + "grad_norm": 2.635136365890503, + "learning_rate": 0.0001, + "loss": 177.8257, + "step": 460 + }, + { + "epoch": 0.07589673199975777, + "grad_norm": 5.227738857269287, + "learning_rate": 0.0001, + "loss": 176.4699, + "step": 470 + }, + { + "epoch": 0.077511556084859, + "grad_norm": 4.998415470123291, + "learning_rate": 0.0001, + "loss": 175.7601, + "step": 480 + }, + { + "epoch": 0.07912638016996024, + "grad_norm": 2.946892261505127, + "learning_rate": 0.0001, + "loss": 173.936, + "step": 490 + }, + { + "epoch": 0.08074120425506147, + "grad_norm": 4.20358943939209, + "learning_rate": 0.0001, + "loss": 172.065, + "step": 500 + }, + { + "epoch": 0.08235602834016269, + "grad_norm": 3.125765562057495, + "learning_rate": 0.0001, + "loss": 171.3727, + "step": 510 + }, + { + "epoch": 0.08397085242526392, + "grad_norm": 2.4870612621307373, + "learning_rate": 0.0001, + "loss": 169.9, + "step": 520 + }, + { + "epoch": 0.08558567651036515, + "grad_norm": 3.917384147644043, + "learning_rate": 0.0001, + "loss": 168.5049, + "step": 530 + }, + { + "epoch": 0.08720050059546638, + "grad_norm": 2.302142858505249, + "learning_rate": 0.0001, + "loss": 167.2027, + "step": 540 + }, + { + "epoch": 0.08881532468056762, + "grad_norm": 4.794841289520264, + "learning_rate": 0.0001, + "loss": 166.0887, + "step": 550 + }, + { + "epoch": 0.09043014876566884, + "grad_norm": 2.200291395187378, + "learning_rate": 0.0001, + "loss": 164.5882, + "step": 560 + }, + { + "epoch": 0.09204497285077007, + "grad_norm": 1.9572436809539795, + "learning_rate": 0.0001, + "loss": 163.3007, + "step": 570 + }, + { + "epoch": 0.0936597969358713, + "grad_norm": 3.443633556365967, + "learning_rate": 0.0001, + "loss": 160.6805, + "step": 580 + }, + { + "epoch": 0.09527462102097253, + "grad_norm": 2.6079020500183105, + "learning_rate": 0.0001, + "loss": 160.3372, + "step": 590 + }, + { + "epoch": 0.09688944510607375, + "grad_norm": 2.4401192665100098, + "learning_rate": 0.0001, + "loss": 160.0956, + "step": 600 + }, + { + "epoch": 0.09850426919117498, + "grad_norm": 2.686101198196411, + "learning_rate": 0.0001, + "loss": 157.6138, + "step": 610 + }, + { + "epoch": 0.10011909327627622, + "grad_norm": 2.594151496887207, + "learning_rate": 0.0001, + "loss": 156.0726, + "step": 620 + }, + { + "epoch": 0.10173391736137745, + "grad_norm": 2.509141683578491, + "learning_rate": 0.0001, + "loss": 155.9923, + "step": 630 + }, + { + "epoch": 0.10334874144647867, + "grad_norm": 2.4077377319335938, + "learning_rate": 0.0001, + "loss": 155.214, + "step": 640 + }, + { + "epoch": 0.1049635655315799, + "grad_norm": 4.083552837371826, + "learning_rate": 0.0001, + "loss": 152.8257, + "step": 650 + }, + { + "epoch": 0.10657838961668113, + "grad_norm": 2.7188713550567627, + "learning_rate": 0.0001, + "loss": 151.875, + "step": 660 + }, + { + "epoch": 0.10819321370178236, + "grad_norm": 2.41287899017334, + "learning_rate": 0.0001, + "loss": 151.935, + "step": 670 + }, + { + "epoch": 0.1098080377868836, + "grad_norm": 3.995708465576172, + "learning_rate": 0.0001, + "loss": 151.0069, + "step": 680 + }, + { + "epoch": 0.11142286187198482, + "grad_norm": 3.217705488204956, + "learning_rate": 0.0001, + "loss": 149.5922, + "step": 690 + }, + { + "epoch": 0.11303768595708605, + "grad_norm": 3.4349842071533203, + "learning_rate": 0.0001, + "loss": 148.0044, + "step": 700 + }, + { + "epoch": 0.11465251004218728, + "grad_norm": 2.106865167617798, + "learning_rate": 0.0001, + "loss": 147.8404, + "step": 710 + }, + { + "epoch": 0.11626733412728851, + "grad_norm": 1.8729510307312012, + "learning_rate": 0.0001, + "loss": 145.1299, + "step": 720 + }, + { + "epoch": 0.11788215821238973, + "grad_norm": 4.14620304107666, + "learning_rate": 0.0001, + "loss": 144.3928, + "step": 730 + }, + { + "epoch": 0.11949698229749096, + "grad_norm": 2.202505350112915, + "learning_rate": 0.0001, + "loss": 144.8357, + "step": 740 + }, + { + "epoch": 0.1211118063825922, + "grad_norm": 2.201470375061035, + "learning_rate": 0.0001, + "loss": 143.2202, + "step": 750 + }, + { + "epoch": 0.12272663046769343, + "grad_norm": 1.5977853536605835, + "learning_rate": 0.0001, + "loss": 143.3537, + "step": 760 + }, + { + "epoch": 0.12434145455279466, + "grad_norm": 2.4403927326202393, + "learning_rate": 0.0001, + "loss": 141.2914, + "step": 770 + }, + { + "epoch": 0.12595627863789588, + "grad_norm": 1.6895846128463745, + "learning_rate": 0.0001, + "loss": 139.9639, + "step": 780 + }, + { + "epoch": 0.12757110272299713, + "grad_norm": 3.4370665550231934, + "learning_rate": 0.0001, + "loss": 140.715, + "step": 790 + }, + { + "epoch": 0.12918592680809834, + "grad_norm": 2.1449077129364014, + "learning_rate": 0.0001, + "loss": 139.2094, + "step": 800 + }, + { + "epoch": 0.13080075089319956, + "grad_norm": 2.4578912258148193, + "learning_rate": 0.0001, + "loss": 137.6724, + "step": 810 + }, + { + "epoch": 0.1324155749783008, + "grad_norm": 1.8994814157485962, + "learning_rate": 0.0001, + "loss": 136.9551, + "step": 820 + }, + { + "epoch": 0.13403039906340203, + "grad_norm": 2.6531031131744385, + "learning_rate": 0.0001, + "loss": 136.5812, + "step": 830 + }, + { + "epoch": 0.13564522314850325, + "grad_norm": 3.8680083751678467, + "learning_rate": 0.0001, + "loss": 134.3655, + "step": 840 + }, + { + "epoch": 0.1372600472336045, + "grad_norm": 1.9594208002090454, + "learning_rate": 0.0001, + "loss": 135.4958, + "step": 850 + }, + { + "epoch": 0.1388748713187057, + "grad_norm": 1.5475000143051147, + "learning_rate": 0.0001, + "loss": 135.2142, + "step": 860 + }, + { + "epoch": 0.14048969540380696, + "grad_norm": 1.6569288969039917, + "learning_rate": 0.0001, + "loss": 133.4162, + "step": 870 + }, + { + "epoch": 0.14210451948890818, + "grad_norm": 1.6724224090576172, + "learning_rate": 0.0001, + "loss": 133.7519, + "step": 880 + }, + { + "epoch": 0.1437193435740094, + "grad_norm": 1.7299500703811646, + "learning_rate": 0.0001, + "loss": 131.471, + "step": 890 + }, + { + "epoch": 0.14533416765911064, + "grad_norm": 1.9444470405578613, + "learning_rate": 0.0001, + "loss": 128.5966, + "step": 900 + }, + { + "epoch": 0.14694899174421186, + "grad_norm": 2.404723644256592, + "learning_rate": 0.0001, + "loss": 129.9824, + "step": 910 + }, + { + "epoch": 0.1485638158293131, + "grad_norm": 2.1493897438049316, + "learning_rate": 0.0001, + "loss": 128.0219, + "step": 920 + }, + { + "epoch": 0.15017863991441432, + "grad_norm": 2.8861117362976074, + "learning_rate": 0.0001, + "loss": 128.5297, + "step": 930 + }, + { + "epoch": 0.15179346399951554, + "grad_norm": 2.546415090560913, + "learning_rate": 0.0001, + "loss": 126.6817, + "step": 940 + }, + { + "epoch": 0.1534082880846168, + "grad_norm": 1.727663278579712, + "learning_rate": 0.0001, + "loss": 125.0413, + "step": 950 + }, + { + "epoch": 0.155023112169718, + "grad_norm": 3.7156920433044434, + "learning_rate": 0.0001, + "loss": 124.8085, + "step": 960 + }, + { + "epoch": 0.15663793625481925, + "grad_norm": 3.0576517581939697, + "learning_rate": 0.0001, + "loss": 125.9107, + "step": 970 + }, + { + "epoch": 0.15825276033992047, + "grad_norm": 1.9951574802398682, + "learning_rate": 0.0001, + "loss": 123.5359, + "step": 980 + }, + { + "epoch": 0.1598675844250217, + "grad_norm": 1.4620885848999023, + "learning_rate": 0.0001, + "loss": 121.9363, + "step": 990 + }, + { + "epoch": 0.16148240851012294, + "grad_norm": 1.7640169858932495, + "learning_rate": 0.0001, + "loss": 121.1764, + "step": 1000 + }, + { + "epoch": 0.16309723259522416, + "grad_norm": 2.921750545501709, + "learning_rate": 0.0001, + "loss": 119.6931, + "step": 1010 + }, + { + "epoch": 0.16471205668032537, + "grad_norm": 1.5623588562011719, + "learning_rate": 0.0001, + "loss": 119.097, + "step": 1020 + }, + { + "epoch": 0.16632688076542662, + "grad_norm": 2.2386627197265625, + "learning_rate": 0.0001, + "loss": 120.9224, + "step": 1030 + }, + { + "epoch": 0.16794170485052784, + "grad_norm": 1.3213791847229004, + "learning_rate": 0.0001, + "loss": 119.5649, + "step": 1040 + }, + { + "epoch": 0.16955652893562909, + "grad_norm": 1.5255440473556519, + "learning_rate": 0.0001, + "loss": 118.1524, + "step": 1050 + }, + { + "epoch": 0.1711713530207303, + "grad_norm": 1.6038514375686646, + "learning_rate": 0.0001, + "loss": 118.6307, + "step": 1060 + }, + { + "epoch": 0.17278617710583152, + "grad_norm": 2.356739044189453, + "learning_rate": 0.0001, + "loss": 116.8472, + "step": 1070 + }, + { + "epoch": 0.17440100119093277, + "grad_norm": 2.5326743125915527, + "learning_rate": 0.0001, + "loss": 115.6625, + "step": 1080 + }, + { + "epoch": 0.176015825276034, + "grad_norm": 1.2823396921157837, + "learning_rate": 0.0001, + "loss": 115.3312, + "step": 1090 + }, + { + "epoch": 0.17763064936113523, + "grad_norm": 1.981788158416748, + "learning_rate": 0.0001, + "loss": 116.2257, + "step": 1100 + }, + { + "epoch": 0.17924547344623645, + "grad_norm": 1.540440559387207, + "learning_rate": 0.0001, + "loss": 114.5031, + "step": 1110 + }, + { + "epoch": 0.18086029753133767, + "grad_norm": 1.7667800188064575, + "learning_rate": 0.0001, + "loss": 113.7663, + "step": 1120 + }, + { + "epoch": 0.18247512161643892, + "grad_norm": 2.2793102264404297, + "learning_rate": 0.0001, + "loss": 115.219, + "step": 1130 + }, + { + "epoch": 0.18408994570154014, + "grad_norm": 2.132046699523926, + "learning_rate": 0.0001, + "loss": 112.8482, + "step": 1140 + }, + { + "epoch": 0.18570476978664135, + "grad_norm": 1.3153988122940063, + "learning_rate": 0.0001, + "loss": 111.4546, + "step": 1150 + }, + { + "epoch": 0.1873195938717426, + "grad_norm": 1.5246292352676392, + "learning_rate": 0.0001, + "loss": 112.4348, + "step": 1160 + }, + { + "epoch": 0.18893441795684382, + "grad_norm": 1.365096092224121, + "learning_rate": 0.0001, + "loss": 109.755, + "step": 1170 + }, + { + "epoch": 0.19054924204194507, + "grad_norm": 1.2777953147888184, + "learning_rate": 0.0001, + "loss": 111.0318, + "step": 1180 + }, + { + "epoch": 0.19216406612704628, + "grad_norm": 1.8672829866409302, + "learning_rate": 0.0001, + "loss": 109.1648, + "step": 1190 + }, + { + "epoch": 0.1937788902121475, + "grad_norm": 1.709987759590149, + "learning_rate": 0.0001, + "loss": 108.1616, + "step": 1200 + }, + { + "epoch": 0.19539371429724875, + "grad_norm": 1.176277995109558, + "learning_rate": 0.0001, + "loss": 109.3591, + "step": 1210 + }, + { + "epoch": 0.19700853838234997, + "grad_norm": 1.4181180000305176, + "learning_rate": 0.0001, + "loss": 110.0216, + "step": 1220 + }, + { + "epoch": 0.19862336246745121, + "grad_norm": 1.3246116638183594, + "learning_rate": 0.0001, + "loss": 107.9769, + "step": 1230 + }, + { + "epoch": 0.20023818655255243, + "grad_norm": 1.2501498460769653, + "learning_rate": 0.0001, + "loss": 108.2601, + "step": 1240 + }, + { + "epoch": 0.20185301063765365, + "grad_norm": 1.2277404069900513, + "learning_rate": 0.0001, + "loss": 107.1258, + "step": 1250 + }, + { + "epoch": 0.2034678347227549, + "grad_norm": 1.8064894676208496, + "learning_rate": 0.0001, + "loss": 105.6278, + "step": 1260 + }, + { + "epoch": 0.20508265880785612, + "grad_norm": 1.708897352218628, + "learning_rate": 0.0001, + "loss": 106.9269, + "step": 1270 + }, + { + "epoch": 0.20669748289295733, + "grad_norm": 1.4507478475570679, + "learning_rate": 0.0001, + "loss": 106.9236, + "step": 1280 + }, + { + "epoch": 0.20831230697805858, + "grad_norm": 1.4904310703277588, + "learning_rate": 0.0001, + "loss": 106.0653, + "step": 1290 + }, + { + "epoch": 0.2099271310631598, + "grad_norm": 1.2637165784835815, + "learning_rate": 0.0001, + "loss": 104.8665, + "step": 1300 + }, + { + "epoch": 0.21154195514826105, + "grad_norm": 1.4085133075714111, + "learning_rate": 0.0001, + "loss": 104.8891, + "step": 1310 + }, + { + "epoch": 0.21315677923336226, + "grad_norm": 1.5710629224777222, + "learning_rate": 0.0001, + "loss": 104.3009, + "step": 1320 + }, + { + "epoch": 0.21477160331846348, + "grad_norm": 1.3634663820266724, + "learning_rate": 0.0001, + "loss": 104.4545, + "step": 1330 + }, + { + "epoch": 0.21638642740356473, + "grad_norm": 1.5565859079360962, + "learning_rate": 0.0001, + "loss": 103.6205, + "step": 1340 + }, + { + "epoch": 0.21800125148866595, + "grad_norm": 1.4491888284683228, + "learning_rate": 0.0001, + "loss": 104.0892, + "step": 1350 + }, + { + "epoch": 0.2196160755737672, + "grad_norm": 1.7609063386917114, + "learning_rate": 0.0001, + "loss": 104.5177, + "step": 1360 + }, + { + "epoch": 0.2212308996588684, + "grad_norm": 1.6723753213882446, + "learning_rate": 0.0001, + "loss": 104.5552, + "step": 1370 + }, + { + "epoch": 0.22284572374396963, + "grad_norm": 1.3781665563583374, + "learning_rate": 0.0001, + "loss": 102.1022, + "step": 1380 + }, + { + "epoch": 0.22446054782907088, + "grad_norm": 1.5397366285324097, + "learning_rate": 0.0001, + "loss": 102.5882, + "step": 1390 + }, + { + "epoch": 0.2260753719141721, + "grad_norm": 1.4567879438400269, + "learning_rate": 0.0001, + "loss": 101.3781, + "step": 1400 + }, + { + "epoch": 0.22769019599927334, + "grad_norm": 1.2882301807403564, + "learning_rate": 0.0001, + "loss": 100.9378, + "step": 1410 + }, + { + "epoch": 0.22930502008437456, + "grad_norm": 1.4510424137115479, + "learning_rate": 0.0001, + "loss": 101.5383, + "step": 1420 + }, + { + "epoch": 0.23091984416947578, + "grad_norm": 2.8331449031829834, + "learning_rate": 0.0001, + "loss": 100.2773, + "step": 1430 + }, + { + "epoch": 0.23253466825457703, + "grad_norm": 1.363616943359375, + "learning_rate": 0.0001, + "loss": 101.646, + "step": 1440 + }, + { + "epoch": 0.23414949233967824, + "grad_norm": 1.2184548377990723, + "learning_rate": 0.0001, + "loss": 101.1133, + "step": 1450 + }, + { + "epoch": 0.23576431642477946, + "grad_norm": 2.235656976699829, + "learning_rate": 0.0001, + "loss": 100.2044, + "step": 1460 + }, + { + "epoch": 0.2373791405098807, + "grad_norm": 1.2334418296813965, + "learning_rate": 0.0001, + "loss": 99.4031, + "step": 1470 + }, + { + "epoch": 0.23899396459498193, + "grad_norm": 1.2394684553146362, + "learning_rate": 0.0001, + "loss": 101.1247, + "step": 1480 + }, + { + "epoch": 0.24060878868008317, + "grad_norm": 1.5218186378479004, + "learning_rate": 0.0001, + "loss": 100.4413, + "step": 1490 + }, + { + "epoch": 0.2422236127651844, + "grad_norm": 1.2112364768981934, + "learning_rate": 0.0001, + "loss": 99.8722, + "step": 1500 + }, + { + "epoch": 0.2438384368502856, + "grad_norm": 1.2277610301971436, + "learning_rate": 0.0001, + "loss": 99.0276, + "step": 1510 + }, + { + "epoch": 0.24545326093538686, + "grad_norm": 1.1005053520202637, + "learning_rate": 0.0001, + "loss": 98.6764, + "step": 1520 + }, + { + "epoch": 0.24706808502048808, + "grad_norm": 1.0754456520080566, + "learning_rate": 0.0001, + "loss": 97.8968, + "step": 1530 + }, + { + "epoch": 0.24868290910558932, + "grad_norm": 1.0695950984954834, + "learning_rate": 0.0001, + "loss": 98.4032, + "step": 1540 + }, + { + "epoch": 0.25029773319069054, + "grad_norm": 1.5098627805709839, + "learning_rate": 0.0001, + "loss": 97.9391, + "step": 1550 + }, + { + "epoch": 0.25191255727579176, + "grad_norm": 1.006681203842163, + "learning_rate": 0.0001, + "loss": 96.5525, + "step": 1560 + }, + { + "epoch": 0.253527381360893, + "grad_norm": 1.3987332582473755, + "learning_rate": 0.0001, + "loss": 98.383, + "step": 1570 + }, + { + "epoch": 0.25514220544599425, + "grad_norm": 1.290887713432312, + "learning_rate": 0.0001, + "loss": 99.0901, + "step": 1580 + }, + { + "epoch": 0.25675702953109547, + "grad_norm": 0.991177499294281, + "learning_rate": 0.0001, + "loss": 97.7093, + "step": 1590 + }, + { + "epoch": 0.2583718536161967, + "grad_norm": 1.147647738456726, + "learning_rate": 0.0001, + "loss": 96.9181, + "step": 1600 + }, + { + "epoch": 0.2599866777012979, + "grad_norm": 1.1756514310836792, + "learning_rate": 0.0001, + "loss": 98.3935, + "step": 1610 + }, + { + "epoch": 0.2616015017863991, + "grad_norm": 1.1708602905273438, + "learning_rate": 0.0001, + "loss": 96.715, + "step": 1620 + }, + { + "epoch": 0.2632163258715004, + "grad_norm": 1.4577715396881104, + "learning_rate": 0.0001, + "loss": 97.2971, + "step": 1630 + }, + { + "epoch": 0.2648311499566016, + "grad_norm": 1.1172800064086914, + "learning_rate": 0.0001, + "loss": 97.6494, + "step": 1640 + }, + { + "epoch": 0.26644597404170284, + "grad_norm": 1.1491258144378662, + "learning_rate": 0.0001, + "loss": 97.5503, + "step": 1650 + }, + { + "epoch": 0.26806079812680406, + "grad_norm": 1.2188555002212524, + "learning_rate": 0.0001, + "loss": 95.3423, + "step": 1660 + }, + { + "epoch": 0.2696756222119053, + "grad_norm": 1.8595614433288574, + "learning_rate": 0.0001, + "loss": 96.0112, + "step": 1670 + }, + { + "epoch": 0.2712904462970065, + "grad_norm": 1.1887950897216797, + "learning_rate": 0.0001, + "loss": 95.4939, + "step": 1680 + }, + { + "epoch": 0.27290527038210777, + "grad_norm": 1.1777839660644531, + "learning_rate": 0.0001, + "loss": 95.9619, + "step": 1690 + }, + { + "epoch": 0.274520094467209, + "grad_norm": 1.1241233348846436, + "learning_rate": 0.0001, + "loss": 96.2834, + "step": 1700 + }, + { + "epoch": 0.2761349185523102, + "grad_norm": 1.0900121927261353, + "learning_rate": 0.0001, + "loss": 94.7518, + "step": 1710 + }, + { + "epoch": 0.2777497426374114, + "grad_norm": 1.0615205764770508, + "learning_rate": 0.0001, + "loss": 95.8032, + "step": 1720 + }, + { + "epoch": 0.27936456672251264, + "grad_norm": 1.2626168727874756, + "learning_rate": 0.0001, + "loss": 93.9454, + "step": 1730 + }, + { + "epoch": 0.2809793908076139, + "grad_norm": 0.8869941830635071, + "learning_rate": 0.0001, + "loss": 95.4085, + "step": 1740 + }, + { + "epoch": 0.28259421489271513, + "grad_norm": 1.0350773334503174, + "learning_rate": 0.0001, + "loss": 95.7954, + "step": 1750 + }, + { + "epoch": 0.28420903897781635, + "grad_norm": 1.364768147468567, + "learning_rate": 0.0001, + "loss": 93.6447, + "step": 1760 + }, + { + "epoch": 0.28582386306291757, + "grad_norm": 1.0115883350372314, + "learning_rate": 0.0001, + "loss": 93.7905, + "step": 1770 + }, + { + "epoch": 0.2874386871480188, + "grad_norm": 0.9393435120582581, + "learning_rate": 0.0001, + "loss": 94.3108, + "step": 1780 + }, + { + "epoch": 0.28905351123312006, + "grad_norm": 1.0769627094268799, + "learning_rate": 0.0001, + "loss": 92.566, + "step": 1790 + }, + { + "epoch": 0.2906683353182213, + "grad_norm": 2.526201009750366, + "learning_rate": 0.0001, + "loss": 92.8161, + "step": 1800 + }, + { + "epoch": 0.2922831594033225, + "grad_norm": 1.3066468238830566, + "learning_rate": 0.0001, + "loss": 92.0459, + "step": 1810 + }, + { + "epoch": 0.2938979834884237, + "grad_norm": 1.121713638305664, + "learning_rate": 0.0001, + "loss": 93.5032, + "step": 1820 + }, + { + "epoch": 0.29551280757352494, + "grad_norm": 1.3744316101074219, + "learning_rate": 0.0001, + "loss": 94.2623, + "step": 1830 + }, + { + "epoch": 0.2971276316586262, + "grad_norm": 1.1561540365219116, + "learning_rate": 0.0001, + "loss": 94.1611, + "step": 1840 + }, + { + "epoch": 0.29874245574372743, + "grad_norm": 1.3947464227676392, + "learning_rate": 0.0001, + "loss": 93.5962, + "step": 1850 + }, + { + "epoch": 0.30035727982882865, + "grad_norm": 1.0011566877365112, + "learning_rate": 0.0001, + "loss": 92.0199, + "step": 1860 + }, + { + "epoch": 0.30197210391392987, + "grad_norm": 0.9978747963905334, + "learning_rate": 0.0001, + "loss": 91.0945, + "step": 1870 + }, + { + "epoch": 0.3035869279990311, + "grad_norm": 0.92522132396698, + "learning_rate": 0.0001, + "loss": 93.274, + "step": 1880 + }, + { + "epoch": 0.30520175208413236, + "grad_norm": 1.0089120864868164, + "learning_rate": 0.0001, + "loss": 91.9308, + "step": 1890 + }, + { + "epoch": 0.3068165761692336, + "grad_norm": 1.0535134077072144, + "learning_rate": 0.0001, + "loss": 92.563, + "step": 1900 + }, + { + "epoch": 0.3084314002543348, + "grad_norm": 0.9782275557518005, + "learning_rate": 0.0001, + "loss": 90.3486, + "step": 1910 + }, + { + "epoch": 0.310046224339436, + "grad_norm": 1.1433162689208984, + "learning_rate": 0.0001, + "loss": 90.4977, + "step": 1920 + }, + { + "epoch": 0.31166104842453723, + "grad_norm": 0.98392653465271, + "learning_rate": 0.0001, + "loss": 92.6529, + "step": 1930 + }, + { + "epoch": 0.3132758725096385, + "grad_norm": 0.8538148403167725, + "learning_rate": 0.0001, + "loss": 90.6859, + "step": 1940 + }, + { + "epoch": 0.3148906965947397, + "grad_norm": 1.019405484199524, + "learning_rate": 0.0001, + "loss": 91.0028, + "step": 1950 + }, + { + "epoch": 0.31650552067984095, + "grad_norm": 0.9828067421913147, + "learning_rate": 0.0001, + "loss": 89.9826, + "step": 1960 + }, + { + "epoch": 0.31812034476494216, + "grad_norm": 0.9725450277328491, + "learning_rate": 0.0001, + "loss": 91.7342, + "step": 1970 + }, + { + "epoch": 0.3197351688500434, + "grad_norm": 1.0139408111572266, + "learning_rate": 0.0001, + "loss": 90.5362, + "step": 1980 + }, + { + "epoch": 0.3213499929351446, + "grad_norm": 1.2493853569030762, + "learning_rate": 0.0001, + "loss": 92.7222, + "step": 1990 + }, + { + "epoch": 0.3229648170202459, + "grad_norm": 1.0152915716171265, + "learning_rate": 0.0001, + "loss": 92.2271, + "step": 2000 + }, + { + "epoch": 0.3245796411053471, + "grad_norm": 0.9745819568634033, + "learning_rate": 0.0001, + "loss": 88.7034, + "step": 2010 + }, + { + "epoch": 0.3261944651904483, + "grad_norm": 1.009624719619751, + "learning_rate": 0.0001, + "loss": 90.7428, + "step": 2020 + }, + { + "epoch": 0.32780928927554953, + "grad_norm": 0.910858690738678, + "learning_rate": 0.0001, + "loss": 89.8199, + "step": 2030 + }, + { + "epoch": 0.32942411336065075, + "grad_norm": 1.1745878458023071, + "learning_rate": 0.0001, + "loss": 90.8267, + "step": 2040 + }, + { + "epoch": 0.331038937445752, + "grad_norm": 0.9409273266792297, + "learning_rate": 0.0001, + "loss": 90.2937, + "step": 2050 + }, + { + "epoch": 0.33265376153085324, + "grad_norm": 1.4152942895889282, + "learning_rate": 0.0001, + "loss": 90.3965, + "step": 2060 + }, + { + "epoch": 0.33426858561595446, + "grad_norm": 0.9529693722724915, + "learning_rate": 0.0001, + "loss": 90.4739, + "step": 2070 + }, + { + "epoch": 0.3358834097010557, + "grad_norm": 0.8531096577644348, + "learning_rate": 0.0001, + "loss": 90.9301, + "step": 2080 + }, + { + "epoch": 0.3374982337861569, + "grad_norm": 0.9314167499542236, + "learning_rate": 0.0001, + "loss": 90.4235, + "step": 2090 + }, + { + "epoch": 0.33911305787125817, + "grad_norm": 0.9253250360488892, + "learning_rate": 0.0001, + "loss": 88.1893, + "step": 2100 + }, + { + "epoch": 0.3407278819563594, + "grad_norm": 1.773053526878357, + "learning_rate": 0.0001, + "loss": 88.3417, + "step": 2110 + }, + { + "epoch": 0.3423427060414606, + "grad_norm": 0.9432951211929321, + "learning_rate": 0.0001, + "loss": 89.7141, + "step": 2120 + }, + { + "epoch": 0.34395753012656183, + "grad_norm": 3.8847527503967285, + "learning_rate": 0.0001, + "loss": 89.1009, + "step": 2130 + }, + { + "epoch": 0.34557235421166305, + "grad_norm": 1.0309821367263794, + "learning_rate": 0.0001, + "loss": 89.2965, + "step": 2140 + }, + { + "epoch": 0.3471871782967643, + "grad_norm": 0.9590495824813843, + "learning_rate": 0.0001, + "loss": 89.1734, + "step": 2150 + }, + { + "epoch": 0.34880200238186554, + "grad_norm": 0.9943560361862183, + "learning_rate": 0.0001, + "loss": 85.425, + "step": 2160 + }, + { + "epoch": 0.35041682646696676, + "grad_norm": 1.3322339057922363, + "learning_rate": 0.0001, + "loss": 89.7791, + "step": 2170 + }, + { + "epoch": 0.352031650552068, + "grad_norm": 1.0212253332138062, + "learning_rate": 0.0001, + "loss": 89.3771, + "step": 2180 + }, + { + "epoch": 0.3536464746371692, + "grad_norm": 1.0446089506149292, + "learning_rate": 0.0001, + "loss": 88.6894, + "step": 2190 + }, + { + "epoch": 0.35526129872227047, + "grad_norm": 1.05767822265625, + "learning_rate": 0.0001, + "loss": 89.5341, + "step": 2200 + }, + { + "epoch": 0.3568761228073717, + "grad_norm": 1.0373681783676147, + "learning_rate": 0.0001, + "loss": 89.7173, + "step": 2210 + }, + { + "epoch": 0.3584909468924729, + "grad_norm": 0.9098039269447327, + "learning_rate": 0.0001, + "loss": 89.2116, + "step": 2220 + }, + { + "epoch": 0.3601057709775741, + "grad_norm": 1.028130292892456, + "learning_rate": 0.0001, + "loss": 88.6627, + "step": 2230 + }, + { + "epoch": 0.36172059506267534, + "grad_norm": 1.0295171737670898, + "learning_rate": 0.0001, + "loss": 89.608, + "step": 2240 + }, + { + "epoch": 0.36333541914777656, + "grad_norm": 0.9720722436904907, + "learning_rate": 0.0001, + "loss": 88.3624, + "step": 2250 + }, + { + "epoch": 0.36495024323287784, + "grad_norm": 0.839654803276062, + "learning_rate": 0.0001, + "loss": 87.2983, + "step": 2260 + }, + { + "epoch": 0.36656506731797905, + "grad_norm": 1.1600219011306763, + "learning_rate": 0.0001, + "loss": 86.6004, + "step": 2270 + }, + { + "epoch": 0.3681798914030803, + "grad_norm": 1.0513924360275269, + "learning_rate": 0.0001, + "loss": 86.6062, + "step": 2280 + }, + { + "epoch": 0.3697947154881815, + "grad_norm": 0.9728474617004395, + "learning_rate": 0.0001, + "loss": 86.1856, + "step": 2290 + }, + { + "epoch": 0.3714095395732827, + "grad_norm": 1.0196386575698853, + "learning_rate": 0.0001, + "loss": 87.6088, + "step": 2300 + }, + { + "epoch": 0.373024363658384, + "grad_norm": 1.0439444780349731, + "learning_rate": 0.0001, + "loss": 85.2374, + "step": 2310 + }, + { + "epoch": 0.3746391877434852, + "grad_norm": 1.0020275115966797, + "learning_rate": 0.0001, + "loss": 86.0596, + "step": 2320 + }, + { + "epoch": 0.3762540118285864, + "grad_norm": 1.0107691287994385, + "learning_rate": 0.0001, + "loss": 86.9513, + "step": 2330 + }, + { + "epoch": 0.37786883591368764, + "grad_norm": 1.680018663406372, + "learning_rate": 0.0001, + "loss": 87.8885, + "step": 2340 + }, + { + "epoch": 0.37948365999878886, + "grad_norm": 0.9181709885597229, + "learning_rate": 0.0001, + "loss": 86.7662, + "step": 2350 + }, + { + "epoch": 0.38109848408389013, + "grad_norm": 0.9598278403282166, + "learning_rate": 0.0001, + "loss": 88.3775, + "step": 2360 + }, + { + "epoch": 0.38271330816899135, + "grad_norm": 1.0707905292510986, + "learning_rate": 0.0001, + "loss": 86.0997, + "step": 2370 + }, + { + "epoch": 0.38432813225409257, + "grad_norm": 1.125797152519226, + "learning_rate": 0.0001, + "loss": 85.8117, + "step": 2380 + }, + { + "epoch": 0.3859429563391938, + "grad_norm": 1.0251439809799194, + "learning_rate": 0.0001, + "loss": 86.0806, + "step": 2390 + }, + { + "epoch": 0.387557780424295, + "grad_norm": 1.0193125009536743, + "learning_rate": 0.0001, + "loss": 87.264, + "step": 2400 + }, + { + "epoch": 0.3891726045093963, + "grad_norm": 0.9309751391410828, + "learning_rate": 0.0001, + "loss": 85.9187, + "step": 2410 + }, + { + "epoch": 0.3907874285944975, + "grad_norm": 1.1025906801223755, + "learning_rate": 0.0001, + "loss": 86.4365, + "step": 2420 + }, + { + "epoch": 0.3924022526795987, + "grad_norm": 0.9880627393722534, + "learning_rate": 0.0001, + "loss": 86.2303, + "step": 2430 + }, + { + "epoch": 0.39401707676469994, + "grad_norm": 0.9520611763000488, + "learning_rate": 0.0001, + "loss": 84.9035, + "step": 2440 + }, + { + "epoch": 0.39563190084980115, + "grad_norm": 0.8641896843910217, + "learning_rate": 0.0001, + "loss": 85.4754, + "step": 2450 + }, + { + "epoch": 0.39724672493490243, + "grad_norm": 1.1162904500961304, + "learning_rate": 0.0001, + "loss": 85.6767, + "step": 2460 + }, + { + "epoch": 0.39886154902000365, + "grad_norm": 0.9073636531829834, + "learning_rate": 0.0001, + "loss": 85.4198, + "step": 2470 + }, + { + "epoch": 0.40047637310510487, + "grad_norm": 1.1135166883468628, + "learning_rate": 0.0001, + "loss": 86.0884, + "step": 2480 + }, + { + "epoch": 0.4020911971902061, + "grad_norm": 0.9273980259895325, + "learning_rate": 0.0001, + "loss": 85.5473, + "step": 2490 + }, + { + "epoch": 0.4037060212753073, + "grad_norm": 0.8653831481933594, + "learning_rate": 0.0001, + "loss": 87.279, + "step": 2500 + }, + { + "epoch": 0.4053208453604086, + "grad_norm": 0.8535159826278687, + "learning_rate": 0.0001, + "loss": 86.7072, + "step": 2510 + }, + { + "epoch": 0.4069356694455098, + "grad_norm": 0.8864684700965881, + "learning_rate": 0.0001, + "loss": 86.441, + "step": 2520 + }, + { + "epoch": 0.408550493530611, + "grad_norm": 0.8042080998420715, + "learning_rate": 0.0001, + "loss": 83.1706, + "step": 2530 + }, + { + "epoch": 0.41016531761571223, + "grad_norm": 1.0486373901367188, + "learning_rate": 0.0001, + "loss": 83.54, + "step": 2540 + }, + { + "epoch": 0.41178014170081345, + "grad_norm": 0.8087959289550781, + "learning_rate": 0.0001, + "loss": 85.7615, + "step": 2550 + }, + { + "epoch": 0.41339496578591467, + "grad_norm": 1.5032018423080444, + "learning_rate": 0.0001, + "loss": 85.7368, + "step": 2560 + }, + { + "epoch": 0.41500978987101594, + "grad_norm": 1.0786182880401611, + "learning_rate": 0.0001, + "loss": 86.3484, + "step": 2570 + }, + { + "epoch": 0.41662461395611716, + "grad_norm": 0.8744294047355652, + "learning_rate": 0.0001, + "loss": 83.5945, + "step": 2580 + }, + { + "epoch": 0.4182394380412184, + "grad_norm": 0.7840721607208252, + "learning_rate": 0.0001, + "loss": 86.3831, + "step": 2590 + }, + { + "epoch": 0.4198542621263196, + "grad_norm": 0.7977836728096008, + "learning_rate": 0.0001, + "loss": 83.6314, + "step": 2600 + }, + { + "epoch": 0.4214690862114208, + "grad_norm": 0.892484188079834, + "learning_rate": 0.0001, + "loss": 86.4223, + "step": 2610 + }, + { + "epoch": 0.4230839102965221, + "grad_norm": 0.912995457649231, + "learning_rate": 0.0001, + "loss": 83.8971, + "step": 2620 + }, + { + "epoch": 0.4246987343816233, + "grad_norm": 0.9566861987113953, + "learning_rate": 0.0001, + "loss": 85.291, + "step": 2630 + }, + { + "epoch": 0.42631355846672453, + "grad_norm": 0.8324154615402222, + "learning_rate": 0.0001, + "loss": 84.6233, + "step": 2640 + }, + { + "epoch": 0.42792838255182575, + "grad_norm": 0.9490360617637634, + "learning_rate": 0.0001, + "loss": 81.799, + "step": 2650 + }, + { + "epoch": 0.42954320663692697, + "grad_norm": 0.9744566679000854, + "learning_rate": 0.0001, + "loss": 85.4837, + "step": 2660 + }, + { + "epoch": 0.43115803072202824, + "grad_norm": 0.941614031791687, + "learning_rate": 0.0001, + "loss": 82.838, + "step": 2670 + }, + { + "epoch": 0.43277285480712946, + "grad_norm": 1.338364601135254, + "learning_rate": 0.0001, + "loss": 84.4224, + "step": 2680 + }, + { + "epoch": 0.4343876788922307, + "grad_norm": 0.8643816113471985, + "learning_rate": 0.0001, + "loss": 84.3959, + "step": 2690 + }, + { + "epoch": 0.4360025029773319, + "grad_norm": 1.0231473445892334, + "learning_rate": 0.0001, + "loss": 82.3144, + "step": 2700 + }, + { + "epoch": 0.4376173270624331, + "grad_norm": 0.9089650511741638, + "learning_rate": 0.0001, + "loss": 83.0326, + "step": 2710 + }, + { + "epoch": 0.4392321511475344, + "grad_norm": 0.893628716468811, + "learning_rate": 0.0001, + "loss": 83.0981, + "step": 2720 + }, + { + "epoch": 0.4408469752326356, + "grad_norm": 0.869076132774353, + "learning_rate": 0.0001, + "loss": 84.2571, + "step": 2730 + }, + { + "epoch": 0.4424617993177368, + "grad_norm": 0.7964235544204712, + "learning_rate": 0.0001, + "loss": 83.3329, + "step": 2740 + }, + { + "epoch": 0.44407662340283804, + "grad_norm": 3.1645138263702393, + "learning_rate": 0.0001, + "loss": 83.7232, + "step": 2750 + }, + { + "epoch": 0.44569144748793926, + "grad_norm": 0.934644877910614, + "learning_rate": 0.0001, + "loss": 85.0688, + "step": 2760 + }, + { + "epoch": 0.44730627157304054, + "grad_norm": 0.8120711445808411, + "learning_rate": 0.0001, + "loss": 82.7706, + "step": 2770 + }, + { + "epoch": 0.44892109565814176, + "grad_norm": 0.8161435127258301, + "learning_rate": 0.0001, + "loss": 84.4826, + "step": 2780 + }, + { + "epoch": 0.450535919743243, + "grad_norm": 0.8659493327140808, + "learning_rate": 0.0001, + "loss": 82.6222, + "step": 2790 + }, + { + "epoch": 0.4521507438283442, + "grad_norm": 0.9306702613830566, + "learning_rate": 0.0001, + "loss": 83.5897, + "step": 2800 + }, + { + "epoch": 0.4537655679134454, + "grad_norm": 2.2544431686401367, + "learning_rate": 0.0001, + "loss": 82.9603, + "step": 2810 + }, + { + "epoch": 0.4553803919985467, + "grad_norm": 0.8868395686149597, + "learning_rate": 0.0001, + "loss": 82.6674, + "step": 2820 + }, + { + "epoch": 0.4569952160836479, + "grad_norm": 0.9336455464363098, + "learning_rate": 0.0001, + "loss": 84.6974, + "step": 2830 + }, + { + "epoch": 0.4586100401687491, + "grad_norm": 0.8814165592193604, + "learning_rate": 0.0001, + "loss": 83.9329, + "step": 2840 + }, + { + "epoch": 0.46022486425385034, + "grad_norm": 0.8125660419464111, + "learning_rate": 0.0001, + "loss": 84.6168, + "step": 2850 + }, + { + "epoch": 0.46183968833895156, + "grad_norm": 0.9447190165519714, + "learning_rate": 0.0001, + "loss": 82.4098, + "step": 2860 + }, + { + "epoch": 0.4634545124240528, + "grad_norm": 0.8080969452857971, + "learning_rate": 0.0001, + "loss": 83.4076, + "step": 2870 + }, + { + "epoch": 0.46506933650915405, + "grad_norm": 1.0586694478988647, + "learning_rate": 0.0001, + "loss": 83.3295, + "step": 2880 + }, + { + "epoch": 0.46668416059425527, + "grad_norm": 0.8207664489746094, + "learning_rate": 0.0001, + "loss": 84.5056, + "step": 2890 + }, + { + "epoch": 0.4682989846793565, + "grad_norm": 1.0647571086883545, + "learning_rate": 0.0001, + "loss": 83.0766, + "step": 2900 + }, + { + "epoch": 0.4699138087644577, + "grad_norm": 0.9416915774345398, + "learning_rate": 0.0001, + "loss": 83.8166, + "step": 2910 + }, + { + "epoch": 0.4715286328495589, + "grad_norm": 0.7479362487792969, + "learning_rate": 0.0001, + "loss": 82.7507, + "step": 2920 + }, + { + "epoch": 0.4731434569346602, + "grad_norm": 1.0775514841079712, + "learning_rate": 0.0001, + "loss": 84.4641, + "step": 2930 + }, + { + "epoch": 0.4747582810197614, + "grad_norm": 0.8736841082572937, + "learning_rate": 0.0001, + "loss": 82.0451, + "step": 2940 + }, + { + "epoch": 0.47637310510486264, + "grad_norm": 0.9500495791435242, + "learning_rate": 0.0001, + "loss": 82.6425, + "step": 2950 + }, + { + "epoch": 0.47798792918996386, + "grad_norm": 0.8551210165023804, + "learning_rate": 0.0001, + "loss": 81.1971, + "step": 2960 + }, + { + "epoch": 0.4796027532750651, + "grad_norm": 0.8149814605712891, + "learning_rate": 0.0001, + "loss": 82.7919, + "step": 2970 + }, + { + "epoch": 0.48121757736016635, + "grad_norm": 0.9012249112129211, + "learning_rate": 0.0001, + "loss": 81.975, + "step": 2980 + }, + { + "epoch": 0.48283240144526757, + "grad_norm": 1.3784620761871338, + "learning_rate": 0.0001, + "loss": 81.541, + "step": 2990 + }, + { + "epoch": 0.4844472255303688, + "grad_norm": 0.7921801209449768, + "learning_rate": 0.0001, + "loss": 83.6215, + "step": 3000 + }, + { + "epoch": 0.48606204961547, + "grad_norm": 0.9051580429077148, + "learning_rate": 0.0001, + "loss": 82.4048, + "step": 3010 + }, + { + "epoch": 0.4876768737005712, + "grad_norm": 1.0717192888259888, + "learning_rate": 0.0001, + "loss": 82.4079, + "step": 3020 + }, + { + "epoch": 0.4892916977856725, + "grad_norm": 0.8971781730651855, + "learning_rate": 0.0001, + "loss": 83.3668, + "step": 3030 + }, + { + "epoch": 0.4909065218707737, + "grad_norm": 0.9635320901870728, + "learning_rate": 0.0001, + "loss": 82.8244, + "step": 3040 + }, + { + "epoch": 0.49252134595587493, + "grad_norm": 1.1009949445724487, + "learning_rate": 0.0001, + "loss": 82.2174, + "step": 3050 + }, + { + "epoch": 0.49413617004097615, + "grad_norm": 0.9805473685264587, + "learning_rate": 0.0001, + "loss": 83.7897, + "step": 3060 + }, + { + "epoch": 0.49575099412607737, + "grad_norm": 1.2107361555099487, + "learning_rate": 0.0001, + "loss": 82.5108, + "step": 3070 + }, + { + "epoch": 0.49736581821117865, + "grad_norm": 0.9171829223632812, + "learning_rate": 0.0001, + "loss": 80.9126, + "step": 3080 + }, + { + "epoch": 0.49898064229627986, + "grad_norm": 0.8093295097351074, + "learning_rate": 0.0001, + "loss": 83.8147, + "step": 3090 + }, + { + "epoch": 0.5005954663813811, + "grad_norm": 1.2166653871536255, + "learning_rate": 0.0001, + "loss": 82.0288, + "step": 3100 + }, + { + "epoch": 0.5022102904664824, + "grad_norm": 1.0471502542495728, + "learning_rate": 0.0001, + "loss": 82.0706, + "step": 3110 + }, + { + "epoch": 0.5038251145515835, + "grad_norm": 1.007645845413208, + "learning_rate": 0.0001, + "loss": 81.7387, + "step": 3120 + }, + { + "epoch": 0.5054399386366848, + "grad_norm": 0.7252638339996338, + "learning_rate": 0.0001, + "loss": 82.9107, + "step": 3130 + }, + { + "epoch": 0.507054762721786, + "grad_norm": 0.8565683364868164, + "learning_rate": 0.0001, + "loss": 80.2935, + "step": 3140 + }, + { + "epoch": 0.5086695868068872, + "grad_norm": 0.9905404448509216, + "learning_rate": 0.0001, + "loss": 82.5201, + "step": 3150 + }, + { + "epoch": 0.5102844108919885, + "grad_norm": 0.887428343296051, + "learning_rate": 0.0001, + "loss": 80.8528, + "step": 3160 + }, + { + "epoch": 0.5118992349770897, + "grad_norm": 1.062942624092102, + "learning_rate": 0.0001, + "loss": 80.7035, + "step": 3170 + }, + { + "epoch": 0.5135140590621909, + "grad_norm": 1.203599214553833, + "learning_rate": 0.0001, + "loss": 81.2521, + "step": 3180 + }, + { + "epoch": 0.5151288831472921, + "grad_norm": 0.8452383875846863, + "learning_rate": 0.0001, + "loss": 81.6079, + "step": 3190 + }, + { + "epoch": 0.5167437072323934, + "grad_norm": 1.2530587911605835, + "learning_rate": 0.0001, + "loss": 82.2148, + "step": 3200 + }, + { + "epoch": 0.5183585313174947, + "grad_norm": 0.9351627826690674, + "learning_rate": 0.0001, + "loss": 82.7857, + "step": 3210 + }, + { + "epoch": 0.5199733554025958, + "grad_norm": 1.0128077268600464, + "learning_rate": 0.0001, + "loss": 81.0532, + "step": 3220 + }, + { + "epoch": 0.5215881794876971, + "grad_norm": 0.9316917061805725, + "learning_rate": 0.0001, + "loss": 80.7443, + "step": 3230 + }, + { + "epoch": 0.5232030035727983, + "grad_norm": 1.502705693244934, + "learning_rate": 0.0001, + "loss": 80.0608, + "step": 3240 + }, + { + "epoch": 0.5248178276578995, + "grad_norm": 0.8166837692260742, + "learning_rate": 0.0001, + "loss": 81.2013, + "step": 3250 + }, + { + "epoch": 0.5264326517430008, + "grad_norm": 0.8210042119026184, + "learning_rate": 0.0001, + "loss": 80.2491, + "step": 3260 + }, + { + "epoch": 0.528047475828102, + "grad_norm": 0.7854771018028259, + "learning_rate": 0.0001, + "loss": 81.1017, + "step": 3270 + }, + { + "epoch": 0.5296622999132032, + "grad_norm": 0.8530565500259399, + "learning_rate": 0.0001, + "loss": 80.8123, + "step": 3280 + }, + { + "epoch": 0.5312771239983044, + "grad_norm": 0.9446498155593872, + "learning_rate": 0.0001, + "loss": 80.0029, + "step": 3290 + }, + { + "epoch": 0.5328919480834057, + "grad_norm": 0.9646537899971008, + "learning_rate": 0.0001, + "loss": 81.5413, + "step": 3300 + }, + { + "epoch": 0.5345067721685068, + "grad_norm": 0.9262019395828247, + "learning_rate": 0.0001, + "loss": 80.9399, + "step": 3310 + }, + { + "epoch": 0.5361215962536081, + "grad_norm": 0.8515486121177673, + "learning_rate": 0.0001, + "loss": 80.9421, + "step": 3320 + }, + { + "epoch": 0.5377364203387094, + "grad_norm": 1.679321050643921, + "learning_rate": 0.0001, + "loss": 82.4851, + "step": 3330 + }, + { + "epoch": 0.5393512444238105, + "grad_norm": 0.8289812207221985, + "learning_rate": 0.0001, + "loss": 80.5664, + "step": 3340 + }, + { + "epoch": 0.5409660685089118, + "grad_norm": 0.7463789582252502, + "learning_rate": 0.0001, + "loss": 81.5193, + "step": 3350 + }, + { + "epoch": 0.542580892594013, + "grad_norm": 0.8332446217536926, + "learning_rate": 0.0001, + "loss": 80.8091, + "step": 3360 + }, + { + "epoch": 0.5441957166791143, + "grad_norm": 0.7612847089767456, + "learning_rate": 0.0001, + "loss": 79.8893, + "step": 3370 + }, + { + "epoch": 0.5458105407642155, + "grad_norm": 1.0557948350906372, + "learning_rate": 0.0001, + "loss": 81.4305, + "step": 3380 + }, + { + "epoch": 0.5474253648493167, + "grad_norm": 0.7975788116455078, + "learning_rate": 0.0001, + "loss": 80.0105, + "step": 3390 + }, + { + "epoch": 0.549040188934418, + "grad_norm": 0.9049227237701416, + "learning_rate": 0.0001, + "loss": 81.2689, + "step": 3400 + }, + { + "epoch": 0.5506550130195191, + "grad_norm": 1.1195895671844482, + "learning_rate": 0.0001, + "loss": 80.7163, + "step": 3410 + }, + { + "epoch": 0.5522698371046204, + "grad_norm": 0.7506540417671204, + "learning_rate": 0.0001, + "loss": 79.0855, + "step": 3420 + }, + { + "epoch": 0.5538846611897217, + "grad_norm": 0.73989337682724, + "learning_rate": 0.0001, + "loss": 81.1865, + "step": 3430 + }, + { + "epoch": 0.5554994852748228, + "grad_norm": 0.8418401479721069, + "learning_rate": 0.0001, + "loss": 79.5714, + "step": 3440 + }, + { + "epoch": 0.5571143093599241, + "grad_norm": 0.9807226061820984, + "learning_rate": 0.0001, + "loss": 78.5699, + "step": 3450 + }, + { + "epoch": 0.5587291334450253, + "grad_norm": 1.2890545129776, + "learning_rate": 0.0001, + "loss": 81.0128, + "step": 3460 + }, + { + "epoch": 0.5603439575301266, + "grad_norm": 0.8593361973762512, + "learning_rate": 0.0001, + "loss": 80.1638, + "step": 3470 + }, + { + "epoch": 0.5619587816152278, + "grad_norm": 0.7766354084014893, + "learning_rate": 0.0001, + "loss": 81.5597, + "step": 3480 + }, + { + "epoch": 0.563573605700329, + "grad_norm": 0.7785626649856567, + "learning_rate": 0.0001, + "loss": 79.3862, + "step": 3490 + }, + { + "epoch": 0.5651884297854303, + "grad_norm": 0.8429279327392578, + "learning_rate": 0.0001, + "loss": 76.6294, + "step": 3500 + }, + { + "epoch": 0.5668032538705314, + "grad_norm": 0.8141765594482422, + "learning_rate": 0.0001, + "loss": 80.2584, + "step": 3510 + }, + { + "epoch": 0.5684180779556327, + "grad_norm": 0.7065490484237671, + "learning_rate": 0.0001, + "loss": 79.3626, + "step": 3520 + }, + { + "epoch": 0.570032902040734, + "grad_norm": 0.7002031803131104, + "learning_rate": 0.0001, + "loss": 79.283, + "step": 3530 + }, + { + "epoch": 0.5716477261258351, + "grad_norm": 0.8715317845344543, + "learning_rate": 0.0001, + "loss": 80.3145, + "step": 3540 + }, + { + "epoch": 0.5732625502109364, + "grad_norm": 0.9584882855415344, + "learning_rate": 0.0001, + "loss": 78.7443, + "step": 3550 + }, + { + "epoch": 0.5748773742960376, + "grad_norm": 0.7908946871757507, + "learning_rate": 0.0001, + "loss": 79.1906, + "step": 3560 + }, + { + "epoch": 0.5764921983811389, + "grad_norm": 0.8416746258735657, + "learning_rate": 0.0001, + "loss": 77.8172, + "step": 3570 + }, + { + "epoch": 0.5781070224662401, + "grad_norm": 0.823667049407959, + "learning_rate": 0.0001, + "loss": 81.864, + "step": 3580 + }, + { + "epoch": 0.5797218465513413, + "grad_norm": 1.4282617568969727, + "learning_rate": 0.0001, + "loss": 79.9052, + "step": 3590 + }, + { + "epoch": 0.5813366706364426, + "grad_norm": 0.8747813701629639, + "learning_rate": 0.0001, + "loss": 78.8234, + "step": 3600 + }, + { + "epoch": 0.5829514947215437, + "grad_norm": 0.8415511846542358, + "learning_rate": 0.0001, + "loss": 78.8171, + "step": 3610 + }, + { + "epoch": 0.584566318806645, + "grad_norm": 0.9014492034912109, + "learning_rate": 0.0001, + "loss": 80.5011, + "step": 3620 + }, + { + "epoch": 0.5861811428917463, + "grad_norm": 0.7535204887390137, + "learning_rate": 0.0001, + "loss": 79.4895, + "step": 3630 + }, + { + "epoch": 0.5877959669768474, + "grad_norm": 0.8196508884429932, + "learning_rate": 0.0001, + "loss": 79.7101, + "step": 3640 + }, + { + "epoch": 0.5894107910619487, + "grad_norm": 1.5975492000579834, + "learning_rate": 0.0001, + "loss": 79.0687, + "step": 3650 + }, + { + "epoch": 0.5910256151470499, + "grad_norm": 0.9251692295074463, + "learning_rate": 0.0001, + "loss": 80.2377, + "step": 3660 + }, + { + "epoch": 0.5926404392321512, + "grad_norm": 0.7143557071685791, + "learning_rate": 0.0001, + "loss": 78.0671, + "step": 3670 + }, + { + "epoch": 0.5942552633172524, + "grad_norm": 0.7876290678977966, + "learning_rate": 0.0001, + "loss": 79.1455, + "step": 3680 + }, + { + "epoch": 0.5958700874023536, + "grad_norm": 1.5361895561218262, + "learning_rate": 0.0001, + "loss": 78.1253, + "step": 3690 + }, + { + "epoch": 0.5974849114874549, + "grad_norm": 0.7637040019035339, + "learning_rate": 0.0001, + "loss": 77.4658, + "step": 3700 + }, + { + "epoch": 0.599099735572556, + "grad_norm": 0.8166250586509705, + "learning_rate": 0.0001, + "loss": 79.5806, + "step": 3710 + }, + { + "epoch": 0.6007145596576573, + "grad_norm": 1.2767380475997925, + "learning_rate": 0.0001, + "loss": 79.1894, + "step": 3720 + }, + { + "epoch": 0.6023293837427586, + "grad_norm": 1.3048640489578247, + "learning_rate": 0.0001, + "loss": 78.0593, + "step": 3730 + }, + { + "epoch": 0.6039442078278597, + "grad_norm": 0.7945445775985718, + "learning_rate": 0.0001, + "loss": 79.7623, + "step": 3740 + }, + { + "epoch": 0.605559031912961, + "grad_norm": 0.7366836667060852, + "learning_rate": 0.0001, + "loss": 77.9571, + "step": 3750 + }, + { + "epoch": 0.6071738559980622, + "grad_norm": 0.9126115441322327, + "learning_rate": 0.0001, + "loss": 79.9502, + "step": 3760 + }, + { + "epoch": 0.6087886800831634, + "grad_norm": 0.800355851650238, + "learning_rate": 0.0001, + "loss": 78.7244, + "step": 3770 + }, + { + "epoch": 0.6104035041682647, + "grad_norm": 0.8290980458259583, + "learning_rate": 0.0001, + "loss": 78.7724, + "step": 3780 + }, + { + "epoch": 0.6120183282533659, + "grad_norm": 0.8512316942214966, + "learning_rate": 0.0001, + "loss": 78.6277, + "step": 3790 + }, + { + "epoch": 0.6136331523384672, + "grad_norm": 0.8909353613853455, + "learning_rate": 0.0001, + "loss": 78.5956, + "step": 3800 + }, + { + "epoch": 0.6152479764235683, + "grad_norm": 0.8328800797462463, + "learning_rate": 0.0001, + "loss": 78.1323, + "step": 3810 + }, + { + "epoch": 0.6168628005086696, + "grad_norm": 0.7950423359870911, + "learning_rate": 0.0001, + "loss": 78.7529, + "step": 3820 + }, + { + "epoch": 0.6184776245937709, + "grad_norm": 0.9988545179367065, + "learning_rate": 0.0001, + "loss": 80.7727, + "step": 3830 + }, + { + "epoch": 0.620092448678872, + "grad_norm": 0.7243085503578186, + "learning_rate": 0.0001, + "loss": 76.7616, + "step": 3840 + }, + { + "epoch": 0.6217072727639733, + "grad_norm": 0.7674855589866638, + "learning_rate": 0.0001, + "loss": 79.1938, + "step": 3850 + }, + { + "epoch": 0.6233220968490745, + "grad_norm": 0.8979483842849731, + "learning_rate": 0.0001, + "loss": 79.1522, + "step": 3860 + }, + { + "epoch": 0.6249369209341757, + "grad_norm": 0.7759745717048645, + "learning_rate": 0.0001, + "loss": 77.7001, + "step": 3870 + }, + { + "epoch": 0.626551745019277, + "grad_norm": 0.9940371513366699, + "learning_rate": 0.0001, + "loss": 79.8712, + "step": 3880 + }, + { + "epoch": 0.6281665691043782, + "grad_norm": 0.8103354573249817, + "learning_rate": 0.0001, + "loss": 77.5229, + "step": 3890 + }, + { + "epoch": 0.6297813931894795, + "grad_norm": 0.9188411831855774, + "learning_rate": 0.0001, + "loss": 78.2933, + "step": 3900 + }, + { + "epoch": 0.6313962172745806, + "grad_norm": 0.823017954826355, + "learning_rate": 0.0001, + "loss": 77.8692, + "step": 3910 + }, + { + "epoch": 0.6330110413596819, + "grad_norm": 0.8833358287811279, + "learning_rate": 0.0001, + "loss": 78.4465, + "step": 3920 + }, + { + "epoch": 0.634625865444783, + "grad_norm": 0.6880685687065125, + "learning_rate": 0.0001, + "loss": 78.8881, + "step": 3930 + }, + { + "epoch": 0.6362406895298843, + "grad_norm": 0.778455913066864, + "learning_rate": 0.0001, + "loss": 76.0277, + "step": 3940 + }, + { + "epoch": 0.6378555136149856, + "grad_norm": 0.8132772445678711, + "learning_rate": 0.0001, + "loss": 79.1284, + "step": 3950 + }, + { + "epoch": 0.6394703377000868, + "grad_norm": 0.7703788876533508, + "learning_rate": 0.0001, + "loss": 76.9013, + "step": 3960 + }, + { + "epoch": 0.641085161785188, + "grad_norm": 0.7876151204109192, + "learning_rate": 0.0001, + "loss": 77.1354, + "step": 3970 + }, + { + "epoch": 0.6426999858702892, + "grad_norm": 0.8027064800262451, + "learning_rate": 0.0001, + "loss": 77.4936, + "step": 3980 + }, + { + "epoch": 0.6443148099553905, + "grad_norm": 0.8261818885803223, + "learning_rate": 0.0001, + "loss": 76.7634, + "step": 3990 + }, + { + "epoch": 0.6459296340404918, + "grad_norm": 0.7024852633476257, + "learning_rate": 0.0001, + "loss": 79.951, + "step": 4000 + }, + { + "epoch": 0.6475444581255929, + "grad_norm": 1.015307068824768, + "learning_rate": 0.0001, + "loss": 76.0354, + "step": 4010 + }, + { + "epoch": 0.6491592822106942, + "grad_norm": 0.715346097946167, + "learning_rate": 0.0001, + "loss": 78.9021, + "step": 4020 + }, + { + "epoch": 0.6507741062957954, + "grad_norm": 0.8994929790496826, + "learning_rate": 0.0001, + "loss": 77.0587, + "step": 4030 + }, + { + "epoch": 0.6523889303808966, + "grad_norm": 0.7683436274528503, + "learning_rate": 0.0001, + "loss": 78.6943, + "step": 4040 + }, + { + "epoch": 0.6540037544659979, + "grad_norm": 0.7273952960968018, + "learning_rate": 0.0001, + "loss": 77.8396, + "step": 4050 + }, + { + "epoch": 0.6556185785510991, + "grad_norm": 1.0676878690719604, + "learning_rate": 0.0001, + "loss": 79.3449, + "step": 4060 + }, + { + "epoch": 0.6572334026362003, + "grad_norm": 0.6916168928146362, + "learning_rate": 0.0001, + "loss": 76.5462, + "step": 4070 + }, + { + "epoch": 0.6588482267213015, + "grad_norm": 0.6466719508171082, + "learning_rate": 0.0001, + "loss": 77.9354, + "step": 4080 + }, + { + "epoch": 0.6604630508064028, + "grad_norm": 0.7406665086746216, + "learning_rate": 0.0001, + "loss": 76.0135, + "step": 4090 + }, + { + "epoch": 0.662077874891504, + "grad_norm": 0.7104353904724121, + "learning_rate": 0.0001, + "loss": 78.3727, + "step": 4100 + }, + { + "epoch": 0.6636926989766052, + "grad_norm": 0.8328452110290527, + "learning_rate": 0.0001, + "loss": 77.5354, + "step": 4110 + }, + { + "epoch": 0.6653075230617065, + "grad_norm": 0.7012260556221008, + "learning_rate": 0.0001, + "loss": 78.7406, + "step": 4120 + }, + { + "epoch": 0.6669223471468076, + "grad_norm": 1.0436632633209229, + "learning_rate": 0.0001, + "loss": 78.0063, + "step": 4130 + }, + { + "epoch": 0.6685371712319089, + "grad_norm": 0.8471246361732483, + "learning_rate": 0.0001, + "loss": 77.8871, + "step": 4140 + }, + { + "epoch": 0.6701519953170102, + "grad_norm": 0.7378994822502136, + "learning_rate": 0.0001, + "loss": 76.9109, + "step": 4150 + }, + { + "epoch": 0.6717668194021114, + "grad_norm": 0.7778745889663696, + "learning_rate": 0.0001, + "loss": 78.3374, + "step": 4160 + }, + { + "epoch": 0.6733816434872126, + "grad_norm": 0.7841216325759888, + "learning_rate": 0.0001, + "loss": 78.1657, + "step": 4170 + }, + { + "epoch": 0.6749964675723138, + "grad_norm": 1.0027995109558105, + "learning_rate": 0.0001, + "loss": 76.6232, + "step": 4180 + }, + { + "epoch": 0.6766112916574151, + "grad_norm": 1.0034840106964111, + "learning_rate": 0.0001, + "loss": 79.5785, + "step": 4190 + }, + { + "epoch": 0.6782261157425163, + "grad_norm": 0.8399350643157959, + "learning_rate": 0.0001, + "loss": 76.8101, + "step": 4200 + }, + { + "epoch": 0.6798409398276175, + "grad_norm": 0.8864424824714661, + "learning_rate": 0.0001, + "loss": 77.8671, + "step": 4210 + }, + { + "epoch": 0.6814557639127188, + "grad_norm": 0.8322188258171082, + "learning_rate": 0.0001, + "loss": 77.6597, + "step": 4220 + }, + { + "epoch": 0.68307058799782, + "grad_norm": 0.8962367177009583, + "learning_rate": 0.0001, + "loss": 77.7324, + "step": 4230 + }, + { + "epoch": 0.6846854120829212, + "grad_norm": 0.9727205038070679, + "learning_rate": 0.0001, + "loss": 77.8531, + "step": 4240 + }, + { + "epoch": 0.6863002361680225, + "grad_norm": 0.956584095954895, + "learning_rate": 0.0001, + "loss": 78.709, + "step": 4250 + }, + { + "epoch": 0.6879150602531237, + "grad_norm": 0.8531600832939148, + "learning_rate": 0.0001, + "loss": 78.7755, + "step": 4260 + }, + { + "epoch": 0.6895298843382249, + "grad_norm": 0.8059844374656677, + "learning_rate": 0.0001, + "loss": 77.1636, + "step": 4270 + }, + { + "epoch": 0.6911447084233261, + "grad_norm": 0.8373456597328186, + "learning_rate": 0.0001, + "loss": 78.6561, + "step": 4280 + }, + { + "epoch": 0.6927595325084274, + "grad_norm": 0.7819933891296387, + "learning_rate": 0.0001, + "loss": 77.6061, + "step": 4290 + }, + { + "epoch": 0.6943743565935286, + "grad_norm": 0.8829817771911621, + "learning_rate": 0.0001, + "loss": 77.3387, + "step": 4300 + }, + { + "epoch": 0.6959891806786298, + "grad_norm": 0.791772723197937, + "learning_rate": 0.0001, + "loss": 77.4368, + "step": 4310 + }, + { + "epoch": 0.6976040047637311, + "grad_norm": 0.9949231147766113, + "learning_rate": 0.0001, + "loss": 73.4359, + "step": 4320 + }, + { + "epoch": 0.6992188288488322, + "grad_norm": 0.8676188588142395, + "learning_rate": 0.0001, + "loss": 78.4836, + "step": 4330 + }, + { + "epoch": 0.7008336529339335, + "grad_norm": 1.1704007387161255, + "learning_rate": 0.0001, + "loss": 76.5091, + "step": 4340 + }, + { + "epoch": 0.7024484770190348, + "grad_norm": 0.8877903819084167, + "learning_rate": 0.0001, + "loss": 78.021, + "step": 4350 + }, + { + "epoch": 0.704063301104136, + "grad_norm": 1.381914496421814, + "learning_rate": 0.0001, + "loss": 77.8544, + "step": 4360 + }, + { + "epoch": 0.7056781251892372, + "grad_norm": 0.835023045539856, + "learning_rate": 0.0001, + "loss": 76.1552, + "step": 4370 + }, + { + "epoch": 0.7072929492743384, + "grad_norm": 0.9814149141311646, + "learning_rate": 0.0001, + "loss": 76.7646, + "step": 4380 + }, + { + "epoch": 0.7089077733594397, + "grad_norm": 0.8012816309928894, + "learning_rate": 0.0001, + "loss": 75.794, + "step": 4390 + }, + { + "epoch": 0.7105225974445409, + "grad_norm": 0.8340234160423279, + "learning_rate": 0.0001, + "loss": 77.4417, + "step": 4400 + }, + { + "epoch": 0.7121374215296421, + "grad_norm": 0.7257950305938721, + "learning_rate": 0.0001, + "loss": 77.1807, + "step": 4410 + }, + { + "epoch": 0.7137522456147434, + "grad_norm": 0.7342708706855774, + "learning_rate": 0.0001, + "loss": 75.5091, + "step": 4420 + }, + { + "epoch": 0.7153670696998445, + "grad_norm": 0.8298904895782471, + "learning_rate": 0.0001, + "loss": 76.6651, + "step": 4430 + }, + { + "epoch": 0.7169818937849458, + "grad_norm": 0.8564396500587463, + "learning_rate": 0.0001, + "loss": 76.0055, + "step": 4440 + }, + { + "epoch": 0.7185967178700471, + "grad_norm": 0.9145296812057495, + "learning_rate": 0.0001, + "loss": 76.0369, + "step": 4450 + }, + { + "epoch": 0.7202115419551482, + "grad_norm": 0.8074191212654114, + "learning_rate": 0.0001, + "loss": 77.4436, + "step": 4460 + }, + { + "epoch": 0.7218263660402495, + "grad_norm": 0.8212682008743286, + "learning_rate": 0.0001, + "loss": 76.3129, + "step": 4470 + }, + { + "epoch": 0.7234411901253507, + "grad_norm": 0.751323938369751, + "learning_rate": 0.0001, + "loss": 77.603, + "step": 4480 + }, + { + "epoch": 0.725056014210452, + "grad_norm": 0.9664170145988464, + "learning_rate": 0.0001, + "loss": 77.481, + "step": 4490 + }, + { + "epoch": 0.7266708382955531, + "grad_norm": 0.7369455695152283, + "learning_rate": 0.0001, + "loss": 76.3238, + "step": 4500 + }, + { + "epoch": 0.7282856623806544, + "grad_norm": 0.8448513746261597, + "learning_rate": 0.0001, + "loss": 76.5679, + "step": 4510 + }, + { + "epoch": 0.7299004864657557, + "grad_norm": 0.8605676293373108, + "learning_rate": 0.0001, + "loss": 75.7667, + "step": 4520 + }, + { + "epoch": 0.7315153105508568, + "grad_norm": 0.8034664392471313, + "learning_rate": 0.0001, + "loss": 76.2163, + "step": 4530 + }, + { + "epoch": 0.7331301346359581, + "grad_norm": 0.7846826910972595, + "learning_rate": 0.0001, + "loss": 73.5979, + "step": 4540 + }, + { + "epoch": 0.7347449587210593, + "grad_norm": 0.7422606348991394, + "learning_rate": 0.0001, + "loss": 76.2334, + "step": 4550 + }, + { + "epoch": 0.7363597828061605, + "grad_norm": 1.4478483200073242, + "learning_rate": 0.0001, + "loss": 77.6168, + "step": 4560 + }, + { + "epoch": 0.7379746068912618, + "grad_norm": 2.0957767963409424, + "learning_rate": 0.0001, + "loss": 76.3199, + "step": 4570 + }, + { + "epoch": 0.739589430976363, + "grad_norm": 0.6757177114486694, + "learning_rate": 0.0001, + "loss": 77.2923, + "step": 4580 + }, + { + "epoch": 0.7412042550614643, + "grad_norm": 0.9281920790672302, + "learning_rate": 0.0001, + "loss": 76.9963, + "step": 4590 + }, + { + "epoch": 0.7428190791465654, + "grad_norm": 0.8702669143676758, + "learning_rate": 0.0001, + "loss": 76.2553, + "step": 4600 + }, + { + "epoch": 0.7444339032316667, + "grad_norm": 0.7976512312889099, + "learning_rate": 0.0001, + "loss": 74.7622, + "step": 4610 + }, + { + "epoch": 0.746048727316768, + "grad_norm": 0.9116189479827881, + "learning_rate": 0.0001, + "loss": 76.3198, + "step": 4620 + }, + { + "epoch": 0.7476635514018691, + "grad_norm": 0.6239303946495056, + "learning_rate": 0.0001, + "loss": 75.7678, + "step": 4630 + }, + { + "epoch": 0.7492783754869704, + "grad_norm": 0.8380976915359497, + "learning_rate": 0.0001, + "loss": 76.8305, + "step": 4640 + }, + { + "epoch": 0.7508931995720716, + "grad_norm": 0.7394141554832458, + "learning_rate": 0.0001, + "loss": 76.5033, + "step": 4650 + }, + { + "epoch": 0.7525080236571728, + "grad_norm": 1.0694036483764648, + "learning_rate": 0.0001, + "loss": 75.0084, + "step": 4660 + }, + { + "epoch": 0.7541228477422741, + "grad_norm": 0.7530115246772766, + "learning_rate": 0.0001, + "loss": 74.4977, + "step": 4670 + }, + { + "epoch": 0.7557376718273753, + "grad_norm": 1.9561909437179565, + "learning_rate": 0.0001, + "loss": 74.7077, + "step": 4680 + }, + { + "epoch": 0.7573524959124766, + "grad_norm": 0.625341534614563, + "learning_rate": 0.0001, + "loss": 76.1238, + "step": 4690 + }, + { + "epoch": 0.7589673199975777, + "grad_norm": 0.9790500998497009, + "learning_rate": 0.0001, + "loss": 76.3997, + "step": 4700 + }, + { + "epoch": 0.760582144082679, + "grad_norm": 0.7087915539741516, + "learning_rate": 0.0001, + "loss": 75.6893, + "step": 4710 + }, + { + "epoch": 0.7621969681677803, + "grad_norm": 1.0995875597000122, + "learning_rate": 0.0001, + "loss": 74.8588, + "step": 4720 + }, + { + "epoch": 0.7638117922528814, + "grad_norm": 1.0309056043624878, + "learning_rate": 0.0001, + "loss": 73.5934, + "step": 4730 + }, + { + "epoch": 0.7654266163379827, + "grad_norm": 1.0132921934127808, + "learning_rate": 0.0001, + "loss": 73.9868, + "step": 4740 + }, + { + "epoch": 0.7670414404230839, + "grad_norm": 0.7138039469718933, + "learning_rate": 0.0001, + "loss": 76.4459, + "step": 4750 + }, + { + "epoch": 0.7686562645081851, + "grad_norm": 0.7055657505989075, + "learning_rate": 0.0001, + "loss": 75.1068, + "step": 4760 + }, + { + "epoch": 0.7702710885932864, + "grad_norm": 0.7398490905761719, + "learning_rate": 0.0001, + "loss": 75.3697, + "step": 4770 + }, + { + "epoch": 0.7718859126783876, + "grad_norm": 0.6836295127868652, + "learning_rate": 0.0001, + "loss": 77.4988, + "step": 4780 + }, + { + "epoch": 0.7735007367634888, + "grad_norm": 0.7552574276924133, + "learning_rate": 0.0001, + "loss": 75.149, + "step": 4790 + }, + { + "epoch": 0.77511556084859, + "grad_norm": 0.7930851578712463, + "learning_rate": 0.0001, + "loss": 73.8661, + "step": 4800 + }, + { + "epoch": 0.7767303849336913, + "grad_norm": 0.7549886107444763, + "learning_rate": 0.0001, + "loss": 75.042, + "step": 4810 + }, + { + "epoch": 0.7783452090187926, + "grad_norm": 0.9932214617729187, + "learning_rate": 0.0001, + "loss": 75.6619, + "step": 4820 + }, + { + "epoch": 0.7799600331038937, + "grad_norm": 0.7685471773147583, + "learning_rate": 0.0001, + "loss": 75.8404, + "step": 4830 + }, + { + "epoch": 0.781574857188995, + "grad_norm": 0.828737199306488, + "learning_rate": 0.0001, + "loss": 75.5839, + "step": 4840 + }, + { + "epoch": 0.7831896812740962, + "grad_norm": 0.8575420379638672, + "learning_rate": 0.0001, + "loss": 75.7889, + "step": 4850 + }, + { + "epoch": 0.7848045053591974, + "grad_norm": 0.7053834199905396, + "learning_rate": 0.0001, + "loss": 75.9372, + "step": 4860 + }, + { + "epoch": 0.7864193294442987, + "grad_norm": 0.7861505150794983, + "learning_rate": 0.0001, + "loss": 76.1532, + "step": 4870 + }, + { + "epoch": 0.7880341535293999, + "grad_norm": 0.8357386589050293, + "learning_rate": 0.0001, + "loss": 75.8536, + "step": 4880 + }, + { + "epoch": 0.7896489776145011, + "grad_norm": 3.448834180831909, + "learning_rate": 0.0001, + "loss": 74.1794, + "step": 4890 + }, + { + "epoch": 0.7912638016996023, + "grad_norm": 0.813522458076477, + "learning_rate": 0.0001, + "loss": 74.5202, + "step": 4900 + }, + { + "epoch": 0.7928786257847036, + "grad_norm": 0.840230405330658, + "learning_rate": 0.0001, + "loss": 74.6634, + "step": 4910 + }, + { + "epoch": 0.7944934498698049, + "grad_norm": 0.7855745553970337, + "learning_rate": 0.0001, + "loss": 73.5369, + "step": 4920 + }, + { + "epoch": 0.796108273954906, + "grad_norm": 0.7708212733268738, + "learning_rate": 0.0001, + "loss": 74.6225, + "step": 4930 + }, + { + "epoch": 0.7977230980400073, + "grad_norm": 0.7140830159187317, + "learning_rate": 0.0001, + "loss": 75.0585, + "step": 4940 + }, + { + "epoch": 0.7993379221251085, + "grad_norm": 0.7960936427116394, + "learning_rate": 0.0001, + "loss": 74.0574, + "step": 4950 + }, + { + "epoch": 0.8009527462102097, + "grad_norm": 0.7269165515899658, + "learning_rate": 0.0001, + "loss": 74.0379, + "step": 4960 + }, + { + "epoch": 0.802567570295311, + "grad_norm": 0.8505013585090637, + "learning_rate": 0.0001, + "loss": 74.7448, + "step": 4970 + }, + { + "epoch": 0.8041823943804122, + "grad_norm": 0.8935654759407043, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 4980 + }, + { + "epoch": 0.8057972184655134, + "grad_norm": 0.9447595477104187, + "learning_rate": 0.0001, + "loss": 75.9674, + "step": 4990 + }, + { + "epoch": 0.8074120425506146, + "grad_norm": 1.6739715337753296, + "learning_rate": 0.0001, + "loss": 73.9035, + "step": 5000 + }, + { + "epoch": 0.8090268666357159, + "grad_norm": 0.861902117729187, + "learning_rate": 0.0001, + "loss": 74.906, + "step": 5010 + }, + { + "epoch": 0.8106416907208172, + "grad_norm": 0.8328655362129211, + "learning_rate": 0.0001, + "loss": 74.1493, + "step": 5020 + }, + { + "epoch": 0.8122565148059183, + "grad_norm": 0.8193135261535645, + "learning_rate": 0.0001, + "loss": 74.6614, + "step": 5030 + }, + { + "epoch": 0.8138713388910196, + "grad_norm": 0.6954565048217773, + "learning_rate": 0.0001, + "loss": 73.7366, + "step": 5040 + }, + { + "epoch": 0.8154861629761208, + "grad_norm": 0.7381289005279541, + "learning_rate": 0.0001, + "loss": 73.5086, + "step": 5050 + }, + { + "epoch": 0.817100987061222, + "grad_norm": 1.090443730354309, + "learning_rate": 0.0001, + "loss": 75.0695, + "step": 5060 + }, + { + "epoch": 0.8187158111463233, + "grad_norm": 1.2544814348220825, + "learning_rate": 0.0001, + "loss": 72.89, + "step": 5070 + }, + { + "epoch": 0.8203306352314245, + "grad_norm": 0.632146418094635, + "learning_rate": 0.0001, + "loss": 74.8604, + "step": 5080 + }, + { + "epoch": 0.8219454593165257, + "grad_norm": 1.1038259267807007, + "learning_rate": 0.0001, + "loss": 74.7354, + "step": 5090 + }, + { + "epoch": 0.8235602834016269, + "grad_norm": 0.6886508464813232, + "learning_rate": 0.0001, + "loss": 75.4243, + "step": 5100 + }, + { + "epoch": 0.8251751074867282, + "grad_norm": 0.8367568850517273, + "learning_rate": 0.0001, + "loss": 75.2732, + "step": 5110 + }, + { + "epoch": 0.8267899315718293, + "grad_norm": 0.9302715063095093, + "learning_rate": 0.0001, + "loss": 75.7924, + "step": 5120 + }, + { + "epoch": 0.8284047556569306, + "grad_norm": 0.6871219873428345, + "learning_rate": 0.0001, + "loss": 74.2905, + "step": 5130 + }, + { + "epoch": 0.8300195797420319, + "grad_norm": 0.7319445013999939, + "learning_rate": 0.0001, + "loss": 73.7774, + "step": 5140 + }, + { + "epoch": 0.831634403827133, + "grad_norm": 0.7184522151947021, + "learning_rate": 0.0001, + "loss": 73.0554, + "step": 5150 + }, + { + "epoch": 0.8332492279122343, + "grad_norm": 0.8462343811988831, + "learning_rate": 0.0001, + "loss": 73.7615, + "step": 5160 + }, + { + "epoch": 0.8348640519973355, + "grad_norm": 0.9436241984367371, + "learning_rate": 0.0001, + "loss": 73.9464, + "step": 5170 + }, + { + "epoch": 0.8364788760824368, + "grad_norm": 0.9079630374908447, + "learning_rate": 0.0001, + "loss": 73.2727, + "step": 5180 + }, + { + "epoch": 0.838093700167538, + "grad_norm": 0.8280741572380066, + "learning_rate": 0.0001, + "loss": 73.9751, + "step": 5190 + }, + { + "epoch": 0.8397085242526392, + "grad_norm": 0.6475157141685486, + "learning_rate": 0.0001, + "loss": 74.7105, + "step": 5200 + }, + { + "epoch": 0.8413233483377405, + "grad_norm": 0.7090312838554382, + "learning_rate": 0.0001, + "loss": 73.6489, + "step": 5210 + }, + { + "epoch": 0.8429381724228416, + "grad_norm": 1.20967698097229, + "learning_rate": 0.0001, + "loss": 75.1335, + "step": 5220 + }, + { + "epoch": 0.8445529965079429, + "grad_norm": 2.053532600402832, + "learning_rate": 0.0001, + "loss": 75.3111, + "step": 5230 + }, + { + "epoch": 0.8461678205930442, + "grad_norm": 0.7563302516937256, + "learning_rate": 0.0001, + "loss": 72.9935, + "step": 5240 + }, + { + "epoch": 0.8477826446781453, + "grad_norm": 0.7801489233970642, + "learning_rate": 0.0001, + "loss": 74.7856, + "step": 5250 + }, + { + "epoch": 0.8493974687632466, + "grad_norm": 0.737870454788208, + "learning_rate": 0.0001, + "loss": 75.6281, + "step": 5260 + }, + { + "epoch": 0.8510122928483478, + "grad_norm": 0.704352617263794, + "learning_rate": 0.0001, + "loss": 75.4598, + "step": 5270 + }, + { + "epoch": 0.8526271169334491, + "grad_norm": 0.6452057361602783, + "learning_rate": 0.0001, + "loss": 74.0341, + "step": 5280 + }, + { + "epoch": 0.8542419410185503, + "grad_norm": 0.6863471865653992, + "learning_rate": 0.0001, + "loss": 74.3727, + "step": 5290 + }, + { + "epoch": 0.8558567651036515, + "grad_norm": 9.756170272827148, + "learning_rate": 0.0001, + "loss": 75.5231, + "step": 5300 + }, + { + "epoch": 0.8574715891887528, + "grad_norm": 0.8624544739723206, + "learning_rate": 0.0001, + "loss": 74.7821, + "step": 5310 + }, + { + "epoch": 0.8590864132738539, + "grad_norm": 0.6637155413627625, + "learning_rate": 0.0001, + "loss": 74.1153, + "step": 5320 + }, + { + "epoch": 0.8607012373589552, + "grad_norm": 1.0764445066452026, + "learning_rate": 0.0001, + "loss": 73.7208, + "step": 5330 + }, + { + "epoch": 0.8623160614440565, + "grad_norm": 0.8567569255828857, + "learning_rate": 0.0001, + "loss": 73.1399, + "step": 5340 + }, + { + "epoch": 0.8639308855291576, + "grad_norm": 0.7681576013565063, + "learning_rate": 0.0001, + "loss": 73.7577, + "step": 5350 + }, + { + "epoch": 0.8655457096142589, + "grad_norm": 0.9336384534835815, + "learning_rate": 0.0001, + "loss": 74.1022, + "step": 5360 + }, + { + "epoch": 0.8671605336993601, + "grad_norm": 0.8603724837303162, + "learning_rate": 0.0001, + "loss": 75.9579, + "step": 5370 + }, + { + "epoch": 0.8687753577844614, + "grad_norm": 0.77194744348526, + "learning_rate": 0.0001, + "loss": 72.4799, + "step": 5380 + }, + { + "epoch": 0.8703901818695626, + "grad_norm": 0.7480376958847046, + "learning_rate": 0.0001, + "loss": 76.1257, + "step": 5390 + }, + { + "epoch": 0.8720050059546638, + "grad_norm": 0.7841023206710815, + "learning_rate": 0.0001, + "loss": 73.1512, + "step": 5400 + }, + { + "epoch": 0.8736198300397651, + "grad_norm": 0.8741798996925354, + "learning_rate": 0.0001, + "loss": 74.2151, + "step": 5410 + }, + { + "epoch": 0.8752346541248662, + "grad_norm": 0.9762082099914551, + "learning_rate": 0.0001, + "loss": 76.7138, + "step": 5420 + }, + { + "epoch": 0.8768494782099675, + "grad_norm": 0.7670348286628723, + "learning_rate": 0.0001, + "loss": 72.9567, + "step": 5430 + }, + { + "epoch": 0.8784643022950688, + "grad_norm": 0.7449469566345215, + "learning_rate": 0.0001, + "loss": 74.8841, + "step": 5440 + }, + { + "epoch": 0.8800791263801699, + "grad_norm": 0.6984641551971436, + "learning_rate": 0.0001, + "loss": 73.8906, + "step": 5450 + }, + { + "epoch": 0.8816939504652712, + "grad_norm": 0.7854361534118652, + "learning_rate": 0.0001, + "loss": 74.626, + "step": 5460 + }, + { + "epoch": 0.8833087745503724, + "grad_norm": 0.9584240317344666, + "learning_rate": 0.0001, + "loss": 74.7048, + "step": 5470 + }, + { + "epoch": 0.8849235986354737, + "grad_norm": 0.6040284037590027, + "learning_rate": 0.0001, + "loss": 74.7194, + "step": 5480 + }, + { + "epoch": 0.8865384227205749, + "grad_norm": 0.7756215929985046, + "learning_rate": 0.0001, + "loss": 73.8178, + "step": 5490 + }, + { + "epoch": 0.8881532468056761, + "grad_norm": 0.9070586562156677, + "learning_rate": 0.0001, + "loss": 72.8309, + "step": 5500 + }, + { + "epoch": 0.8897680708907774, + "grad_norm": 0.8464986681938171, + "learning_rate": 0.0001, + "loss": 73.8847, + "step": 5510 + }, + { + "epoch": 0.8913828949758785, + "grad_norm": 0.7968605160713196, + "learning_rate": 0.0001, + "loss": 73.1691, + "step": 5520 + }, + { + "epoch": 0.8929977190609798, + "grad_norm": 0.9934583306312561, + "learning_rate": 0.0001, + "loss": 76.2662, + "step": 5530 + }, + { + "epoch": 0.8946125431460811, + "grad_norm": 0.7272152900695801, + "learning_rate": 0.0001, + "loss": 75.5295, + "step": 5540 + }, + { + "epoch": 0.8962273672311822, + "grad_norm": 0.857345461845398, + "learning_rate": 0.0001, + "loss": 75.3703, + "step": 5550 + }, + { + "epoch": 0.8978421913162835, + "grad_norm": 0.713918924331665, + "learning_rate": 0.0001, + "loss": 73.4563, + "step": 5560 + }, + { + "epoch": 0.8994570154013847, + "grad_norm": 0.759233832359314, + "learning_rate": 0.0001, + "loss": 73.2744, + "step": 5570 + }, + { + "epoch": 0.901071839486486, + "grad_norm": 0.7085754871368408, + "learning_rate": 0.0001, + "loss": 74.7792, + "step": 5580 + }, + { + "epoch": 0.9026866635715872, + "grad_norm": 0.7370456457138062, + "learning_rate": 0.0001, + "loss": 74.4216, + "step": 5590 + }, + { + "epoch": 0.9043014876566884, + "grad_norm": 0.779746949672699, + "learning_rate": 0.0001, + "loss": 75.9822, + "step": 5600 + }, + { + "epoch": 0.9059163117417897, + "grad_norm": 0.7915763854980469, + "learning_rate": 0.0001, + "loss": 74.3706, + "step": 5610 + }, + { + "epoch": 0.9075311358268908, + "grad_norm": 0.7886125445365906, + "learning_rate": 0.0001, + "loss": 72.9453, + "step": 5620 + }, + { + "epoch": 0.9091459599119921, + "grad_norm": 0.7804751992225647, + "learning_rate": 0.0001, + "loss": 73.514, + "step": 5630 + }, + { + "epoch": 0.9107607839970934, + "grad_norm": 0.7168806195259094, + "learning_rate": 0.0001, + "loss": 71.4159, + "step": 5640 + }, + { + "epoch": 0.9123756080821945, + "grad_norm": 0.7553626894950867, + "learning_rate": 0.0001, + "loss": 74.4353, + "step": 5650 + }, + { + "epoch": 0.9139904321672958, + "grad_norm": 0.6688916683197021, + "learning_rate": 0.0001, + "loss": 74.3786, + "step": 5660 + }, + { + "epoch": 0.915605256252397, + "grad_norm": 0.7425532341003418, + "learning_rate": 0.0001, + "loss": 73.3355, + "step": 5670 + }, + { + "epoch": 0.9172200803374982, + "grad_norm": 0.6473882794380188, + "learning_rate": 0.0001, + "loss": 74.2315, + "step": 5680 + }, + { + "epoch": 0.9188349044225994, + "grad_norm": 0.7222439050674438, + "learning_rate": 0.0001, + "loss": 74.2885, + "step": 5690 + }, + { + "epoch": 0.9204497285077007, + "grad_norm": 1.5716948509216309, + "learning_rate": 0.0001, + "loss": 74.6739, + "step": 5700 + }, + { + "epoch": 0.922064552592802, + "grad_norm": 0.8077759742736816, + "learning_rate": 0.0001, + "loss": 72.6617, + "step": 5710 + }, + { + "epoch": 0.9236793766779031, + "grad_norm": 0.6948631405830383, + "learning_rate": 0.0001, + "loss": 73.6247, + "step": 5720 + }, + { + "epoch": 0.9252942007630044, + "grad_norm": 0.8823757767677307, + "learning_rate": 0.0001, + "loss": 73.2536, + "step": 5730 + }, + { + "epoch": 0.9269090248481056, + "grad_norm": 0.7066764235496521, + "learning_rate": 0.0001, + "loss": 74.1748, + "step": 5740 + }, + { + "epoch": 0.9285238489332068, + "grad_norm": 0.6979793906211853, + "learning_rate": 0.0001, + "loss": 72.3189, + "step": 5750 + }, + { + "epoch": 0.9301386730183081, + "grad_norm": 0.6961462497711182, + "learning_rate": 0.0001, + "loss": 74.6196, + "step": 5760 + }, + { + "epoch": 0.9317534971034093, + "grad_norm": 0.7797656655311584, + "learning_rate": 0.0001, + "loss": 74.5472, + "step": 5770 + }, + { + "epoch": 0.9333683211885105, + "grad_norm": 1.0671838521957397, + "learning_rate": 0.0001, + "loss": 73.241, + "step": 5780 + }, + { + "epoch": 0.9349831452736117, + "grad_norm": 0.7566089034080505, + "learning_rate": 0.0001, + "loss": 73.6784, + "step": 5790 + }, + { + "epoch": 0.936597969358713, + "grad_norm": 0.8563273549079895, + "learning_rate": 0.0001, + "loss": 73.6871, + "step": 5800 + }, + { + "epoch": 0.9382127934438143, + "grad_norm": 0.7885987162590027, + "learning_rate": 0.0001, + "loss": 73.4447, + "step": 5810 + }, + { + "epoch": 0.9398276175289154, + "grad_norm": 0.8751112818717957, + "learning_rate": 0.0001, + "loss": 74.0531, + "step": 5820 + }, + { + "epoch": 0.9414424416140167, + "grad_norm": 0.6663336157798767, + "learning_rate": 0.0001, + "loss": 73.0557, + "step": 5830 + }, + { + "epoch": 0.9430572656991179, + "grad_norm": 0.7993099689483643, + "learning_rate": 0.0001, + "loss": 74.4467, + "step": 5840 + }, + { + "epoch": 0.9446720897842191, + "grad_norm": 0.6987188458442688, + "learning_rate": 0.0001, + "loss": 74.0327, + "step": 5850 + }, + { + "epoch": 0.9462869138693204, + "grad_norm": 0.8458583354949951, + "learning_rate": 0.0001, + "loss": 73.1757, + "step": 5860 + }, + { + "epoch": 0.9479017379544216, + "grad_norm": 0.6372436881065369, + "learning_rate": 0.0001, + "loss": 73.1067, + "step": 5870 + }, + { + "epoch": 0.9495165620395228, + "grad_norm": 0.7447753548622131, + "learning_rate": 0.0001, + "loss": 75.5461, + "step": 5880 + }, + { + "epoch": 0.951131386124624, + "grad_norm": 0.6633971929550171, + "learning_rate": 0.0001, + "loss": 72.9966, + "step": 5890 + }, + { + "epoch": 0.9527462102097253, + "grad_norm": 0.8576784729957581, + "learning_rate": 0.0001, + "loss": 71.5712, + "step": 5900 + }, + { + "epoch": 0.9543610342948265, + "grad_norm": 1.4696853160858154, + "learning_rate": 0.0001, + "loss": 74.7893, + "step": 5910 + }, + { + "epoch": 0.9559758583799277, + "grad_norm": 0.611210286617279, + "learning_rate": 0.0001, + "loss": 74.3426, + "step": 5920 + }, + { + "epoch": 0.957590682465029, + "grad_norm": 0.5947410464286804, + "learning_rate": 0.0001, + "loss": 71.2563, + "step": 5930 + }, + { + "epoch": 0.9592055065501301, + "grad_norm": 0.7169992923736572, + "learning_rate": 0.0001, + "loss": 71.6049, + "step": 5940 + }, + { + "epoch": 0.9608203306352314, + "grad_norm": 0.6773818731307983, + "learning_rate": 0.0001, + "loss": 75.0328, + "step": 5950 + }, + { + "epoch": 0.9624351547203327, + "grad_norm": 0.6714807748794556, + "learning_rate": 0.0001, + "loss": 71.5105, + "step": 5960 + }, + { + "epoch": 0.9640499788054339, + "grad_norm": 1.0064337253570557, + "learning_rate": 0.0001, + "loss": 75.8887, + "step": 5970 + }, + { + "epoch": 0.9656648028905351, + "grad_norm": 0.9802227020263672, + "learning_rate": 0.0001, + "loss": 73.5655, + "step": 5980 + }, + { + "epoch": 0.9672796269756363, + "grad_norm": 0.7097306251525879, + "learning_rate": 0.0001, + "loss": 73.298, + "step": 5990 + }, + { + "epoch": 0.9688944510607376, + "grad_norm": 0.6680361032485962, + "learning_rate": 0.0001, + "loss": 73.8873, + "step": 6000 + }, + { + "epoch": 0.9705092751458388, + "grad_norm": 1.1302506923675537, + "learning_rate": 0.0001, + "loss": 73.3328, + "step": 6010 + }, + { + "epoch": 0.97212409923094, + "grad_norm": 0.7163954377174377, + "learning_rate": 0.0001, + "loss": 70.5955, + "step": 6020 + }, + { + "epoch": 0.9737389233160413, + "grad_norm": 0.8182240128517151, + "learning_rate": 0.0001, + "loss": 72.6397, + "step": 6030 + }, + { + "epoch": 0.9753537474011424, + "grad_norm": 0.6752052903175354, + "learning_rate": 0.0001, + "loss": 73.1188, + "step": 6040 + }, + { + "epoch": 0.9769685714862437, + "grad_norm": 0.8829680681228638, + "learning_rate": 0.0001, + "loss": 75.7346, + "step": 6050 + }, + { + "epoch": 0.978583395571345, + "grad_norm": 0.9254767894744873, + "learning_rate": 0.0001, + "loss": 73.6221, + "step": 6060 + }, + { + "epoch": 0.9801982196564462, + "grad_norm": 0.7527495622634888, + "learning_rate": 0.0001, + "loss": 73.948, + "step": 6070 + }, + { + "epoch": 0.9818130437415474, + "grad_norm": 1.0135066509246826, + "learning_rate": 0.0001, + "loss": 72.8929, + "step": 6080 + }, + { + "epoch": 0.9834278678266486, + "grad_norm": 0.7093394994735718, + "learning_rate": 0.0001, + "loss": 73.5826, + "step": 6090 + }, + { + "epoch": 0.9850426919117499, + "grad_norm": 0.6651231646537781, + "learning_rate": 0.0001, + "loss": 73.167, + "step": 6100 + }, + { + "epoch": 0.9866575159968511, + "grad_norm": 0.720552921295166, + "learning_rate": 0.0001, + "loss": 74.8985, + "step": 6110 + }, + { + "epoch": 0.9882723400819523, + "grad_norm": 1.7821632623672485, + "learning_rate": 0.0001, + "loss": 74.1414, + "step": 6120 + }, + { + "epoch": 0.9898871641670536, + "grad_norm": 0.6405150890350342, + "learning_rate": 0.0001, + "loss": 73.5989, + "step": 6130 + }, + { + "epoch": 0.9915019882521547, + "grad_norm": 0.8207482695579529, + "learning_rate": 0.0001, + "loss": 73.0977, + "step": 6140 + }, + { + "epoch": 0.993116812337256, + "grad_norm": 0.7320757508277893, + "learning_rate": 0.0001, + "loss": 72.0241, + "step": 6150 + }, + { + "epoch": 0.9947316364223573, + "grad_norm": 0.9522950649261475, + "learning_rate": 0.0001, + "loss": 73.9831, + "step": 6160 + }, + { + "epoch": 0.9963464605074585, + "grad_norm": 0.7437159419059753, + "learning_rate": 0.0001, + "loss": 72.1054, + "step": 6170 + }, + { + "epoch": 0.9979612845925597, + "grad_norm": 0.7035623788833618, + "learning_rate": 0.0001, + "loss": 73.959, + "step": 6180 + }, + { + "epoch": 0.9995761086776609, + "grad_norm": 0.7559277415275574, + "learning_rate": 0.0001, + "loss": 71.4651, + "step": 6190 + }, + { + "epoch": 1.001130376859571, + "grad_norm": 0.852342426776886, + "learning_rate": 0.0001, + "loss": 71.8844, + "step": 6200 + }, + { + "epoch": 1.002745200944672, + "grad_norm": 0.8018911480903625, + "learning_rate": 0.0001, + "loss": 74.5093, + "step": 6210 + }, + { + "epoch": 1.0043600250297733, + "grad_norm": 0.6613730192184448, + "learning_rate": 0.0001, + "loss": 71.9953, + "step": 6220 + }, + { + "epoch": 1.0059748491148746, + "grad_norm": 0.9815977215766907, + "learning_rate": 0.0001, + "loss": 70.8632, + "step": 6230 + }, + { + "epoch": 1.0075896731999758, + "grad_norm": 0.781581699848175, + "learning_rate": 0.0001, + "loss": 72.3603, + "step": 6240 + }, + { + "epoch": 1.009204497285077, + "grad_norm": 0.8079054355621338, + "learning_rate": 0.0001, + "loss": 73.6153, + "step": 6250 + }, + { + "epoch": 1.0108193213701782, + "grad_norm": 0.8356053233146667, + "learning_rate": 0.0001, + "loss": 73.1562, + "step": 6260 + }, + { + "epoch": 1.0124341454552794, + "grad_norm": 0.7107778787612915, + "learning_rate": 0.0001, + "loss": 74.5276, + "step": 6270 + }, + { + "epoch": 1.0140489695403807, + "grad_norm": 0.8601720929145813, + "learning_rate": 0.0001, + "loss": 70.103, + "step": 6280 + }, + { + "epoch": 1.015663793625482, + "grad_norm": 0.675837516784668, + "learning_rate": 0.0001, + "loss": 72.6102, + "step": 6290 + }, + { + "epoch": 1.0172786177105833, + "grad_norm": 0.7160016298294067, + "learning_rate": 0.0001, + "loss": 72.2404, + "step": 6300 + }, + { + "epoch": 1.0188934417956843, + "grad_norm": 0.7286407351493835, + "learning_rate": 0.0001, + "loss": 72.7149, + "step": 6310 + }, + { + "epoch": 1.0205082658807856, + "grad_norm": 0.8183980584144592, + "learning_rate": 0.0001, + "loss": 73.5827, + "step": 6320 + }, + { + "epoch": 1.0221230899658869, + "grad_norm": 0.7300626039505005, + "learning_rate": 0.0001, + "loss": 72.3459, + "step": 6330 + }, + { + "epoch": 1.0237379140509881, + "grad_norm": 0.7222613096237183, + "learning_rate": 0.0001, + "loss": 71.881, + "step": 6340 + }, + { + "epoch": 1.0253527381360894, + "grad_norm": 0.7324674725532532, + "learning_rate": 0.0001, + "loss": 74.4781, + "step": 6350 + }, + { + "epoch": 1.0269675622211905, + "grad_norm": 0.9173555374145508, + "learning_rate": 0.0001, + "loss": 74.3222, + "step": 6360 + }, + { + "epoch": 1.0285823863062917, + "grad_norm": 0.8404964208602905, + "learning_rate": 0.0001, + "loss": 71.8959, + "step": 6370 + }, + { + "epoch": 1.030197210391393, + "grad_norm": 0.8735223412513733, + "learning_rate": 0.0001, + "loss": 72.7251, + "step": 6380 + }, + { + "epoch": 1.0318120344764943, + "grad_norm": 0.8467233777046204, + "learning_rate": 0.0001, + "loss": 72.5218, + "step": 6390 + }, + { + "epoch": 1.0334268585615956, + "grad_norm": 0.673565149307251, + "learning_rate": 0.0001, + "loss": 70.6392, + "step": 6400 + }, + { + "epoch": 1.0350416826466966, + "grad_norm": 0.6558571457862854, + "learning_rate": 0.0001, + "loss": 73.5305, + "step": 6410 + }, + { + "epoch": 1.0366565067317979, + "grad_norm": 0.7251600623130798, + "learning_rate": 0.0001, + "loss": 72.76, + "step": 6420 + }, + { + "epoch": 1.0382713308168992, + "grad_norm": 0.7458305358886719, + "learning_rate": 0.0001, + "loss": 73.6812, + "step": 6430 + }, + { + "epoch": 1.0398861549020004, + "grad_norm": 1.493802547454834, + "learning_rate": 0.0001, + "loss": 74.2455, + "step": 6440 + }, + { + "epoch": 1.0415009789871017, + "grad_norm": 0.894632875919342, + "learning_rate": 0.0001, + "loss": 74.2072, + "step": 6450 + }, + { + "epoch": 1.0431158030722028, + "grad_norm": 0.7142218947410583, + "learning_rate": 0.0001, + "loss": 71.4293, + "step": 6460 + }, + { + "epoch": 1.044730627157304, + "grad_norm": 0.702639639377594, + "learning_rate": 0.0001, + "loss": 71.5383, + "step": 6470 + }, + { + "epoch": 1.0463454512424053, + "grad_norm": 0.9378212094306946, + "learning_rate": 0.0001, + "loss": 72.8916, + "step": 6480 + }, + { + "epoch": 1.0479602753275066, + "grad_norm": 0.8298363089561462, + "learning_rate": 0.0001, + "loss": 72.3344, + "step": 6490 + }, + { + "epoch": 1.0495750994126078, + "grad_norm": 0.6617016792297363, + "learning_rate": 0.0001, + "loss": 71.0487, + "step": 6500 + }, + { + "epoch": 1.051189923497709, + "grad_norm": 0.638520359992981, + "learning_rate": 0.0001, + "loss": 72.2583, + "step": 6510 + }, + { + "epoch": 1.0528047475828102, + "grad_norm": 0.7101534008979797, + "learning_rate": 0.0001, + "loss": 70.8697, + "step": 6520 + }, + { + "epoch": 1.0544195716679114, + "grad_norm": 0.9425339698791504, + "learning_rate": 0.0001, + "loss": 71.0641, + "step": 6530 + }, + { + "epoch": 1.0560343957530127, + "grad_norm": 0.6572297811508179, + "learning_rate": 0.0001, + "loss": 72.6265, + "step": 6540 + }, + { + "epoch": 1.057649219838114, + "grad_norm": 0.745622456073761, + "learning_rate": 0.0001, + "loss": 73.0587, + "step": 6550 + }, + { + "epoch": 1.059264043923215, + "grad_norm": 0.7667096257209778, + "learning_rate": 0.0001, + "loss": 72.1779, + "step": 6560 + }, + { + "epoch": 1.0608788680083163, + "grad_norm": 0.7322410941123962, + "learning_rate": 0.0001, + "loss": 71.5503, + "step": 6570 + }, + { + "epoch": 1.0624936920934176, + "grad_norm": 0.5739616751670837, + "learning_rate": 0.0001, + "loss": 72.3846, + "step": 6580 + }, + { + "epoch": 1.0641085161785189, + "grad_norm": 0.7149553894996643, + "learning_rate": 0.0001, + "loss": 71.5459, + "step": 6590 + }, + { + "epoch": 1.0657233402636201, + "grad_norm": 0.6634597778320312, + "learning_rate": 0.0001, + "loss": 72.2528, + "step": 6600 + }, + { + "epoch": 1.0673381643487212, + "grad_norm": 0.7208803296089172, + "learning_rate": 0.0001, + "loss": 71.1115, + "step": 6610 + }, + { + "epoch": 1.0689529884338225, + "grad_norm": 0.9845868945121765, + "learning_rate": 0.0001, + "loss": 72.6418, + "step": 6620 + }, + { + "epoch": 1.0705678125189237, + "grad_norm": 0.9322167634963989, + "learning_rate": 0.0001, + "loss": 71.937, + "step": 6630 + }, + { + "epoch": 1.072182636604025, + "grad_norm": 0.6323093771934509, + "learning_rate": 0.0001, + "loss": 71.1537, + "step": 6640 + }, + { + "epoch": 1.0737974606891263, + "grad_norm": 0.6476802229881287, + "learning_rate": 0.0001, + "loss": 73.7811, + "step": 6650 + }, + { + "epoch": 1.0754122847742273, + "grad_norm": 0.7882822751998901, + "learning_rate": 0.0001, + "loss": 71.9548, + "step": 6660 + }, + { + "epoch": 1.0770271088593286, + "grad_norm": 0.7651168704032898, + "learning_rate": 0.0001, + "loss": 73.4066, + "step": 6670 + }, + { + "epoch": 1.07864193294443, + "grad_norm": 0.7536904215812683, + "learning_rate": 0.0001, + "loss": 72.1063, + "step": 6680 + }, + { + "epoch": 1.0802567570295312, + "grad_norm": 0.7908427119255066, + "learning_rate": 0.0001, + "loss": 72.6374, + "step": 6690 + }, + { + "epoch": 1.0818715811146324, + "grad_norm": 0.7796691060066223, + "learning_rate": 0.0001, + "loss": 72.2938, + "step": 6700 + }, + { + "epoch": 1.0834864051997335, + "grad_norm": 0.8512274622917175, + "learning_rate": 0.0001, + "loss": 71.556, + "step": 6710 + }, + { + "epoch": 1.0851012292848348, + "grad_norm": 0.7285851836204529, + "learning_rate": 0.0001, + "loss": 72.406, + "step": 6720 + }, + { + "epoch": 1.086716053369936, + "grad_norm": 0.6613008975982666, + "learning_rate": 0.0001, + "loss": 70.6509, + "step": 6730 + }, + { + "epoch": 1.0883308774550373, + "grad_norm": 0.7158271074295044, + "learning_rate": 0.0001, + "loss": 71.8434, + "step": 6740 + }, + { + "epoch": 1.0899457015401384, + "grad_norm": 0.8635967373847961, + "learning_rate": 0.0001, + "loss": 71.8279, + "step": 6750 + }, + { + "epoch": 1.0915605256252396, + "grad_norm": 0.7020159959793091, + "learning_rate": 0.0001, + "loss": 72.2417, + "step": 6760 + }, + { + "epoch": 1.093175349710341, + "grad_norm": 0.6389853954315186, + "learning_rate": 0.0001, + "loss": 72.1879, + "step": 6770 + }, + { + "epoch": 1.0947901737954422, + "grad_norm": 0.6460697650909424, + "learning_rate": 0.0001, + "loss": 72.3356, + "step": 6780 + }, + { + "epoch": 1.0964049978805435, + "grad_norm": 0.7152006030082703, + "learning_rate": 0.0001, + "loss": 71.3725, + "step": 6790 + }, + { + "epoch": 1.0980198219656445, + "grad_norm": 1.0050997734069824, + "learning_rate": 0.0001, + "loss": 72.5452, + "step": 6800 + }, + { + "epoch": 1.0996346460507458, + "grad_norm": 0.6967180371284485, + "learning_rate": 0.0001, + "loss": 72.0832, + "step": 6810 + }, + { + "epoch": 1.101249470135847, + "grad_norm": 0.7472837567329407, + "learning_rate": 0.0001, + "loss": 72.8122, + "step": 6820 + }, + { + "epoch": 1.1028642942209483, + "grad_norm": 0.8198382258415222, + "learning_rate": 0.0001, + "loss": 72.5116, + "step": 6830 + }, + { + "epoch": 1.1044791183060496, + "grad_norm": 0.8112207651138306, + "learning_rate": 0.0001, + "loss": 72.4218, + "step": 6840 + }, + { + "epoch": 1.1060939423911507, + "grad_norm": 0.8412114977836609, + "learning_rate": 0.0001, + "loss": 71.0198, + "step": 6850 + }, + { + "epoch": 1.107708766476252, + "grad_norm": 0.6730719804763794, + "learning_rate": 0.0001, + "loss": 71.2045, + "step": 6860 + }, + { + "epoch": 1.1093235905613532, + "grad_norm": 0.6895949244499207, + "learning_rate": 0.0001, + "loss": 72.3056, + "step": 6870 + }, + { + "epoch": 1.1109384146464545, + "grad_norm": 2.476609945297241, + "learning_rate": 0.0001, + "loss": 70.3865, + "step": 6880 + }, + { + "epoch": 1.1125532387315558, + "grad_norm": 0.7285001873970032, + "learning_rate": 0.0001, + "loss": 71.5761, + "step": 6890 + }, + { + "epoch": 1.1141680628166568, + "grad_norm": 0.7546679973602295, + "learning_rate": 0.0001, + "loss": 72.5979, + "step": 6900 + }, + { + "epoch": 1.115782886901758, + "grad_norm": 0.8974841237068176, + "learning_rate": 0.0001, + "loss": 74.5888, + "step": 6910 + }, + { + "epoch": 1.1173977109868594, + "grad_norm": 0.8785758018493652, + "learning_rate": 0.0001, + "loss": 71.662, + "step": 6920 + }, + { + "epoch": 1.1190125350719606, + "grad_norm": 0.6805328130722046, + "learning_rate": 0.0001, + "loss": 72.201, + "step": 6930 + }, + { + "epoch": 1.120627359157062, + "grad_norm": 0.8132315278053284, + "learning_rate": 0.0001, + "loss": 72.7473, + "step": 6940 + }, + { + "epoch": 1.122242183242163, + "grad_norm": 1.1625338792800903, + "learning_rate": 0.0001, + "loss": 70.5987, + "step": 6950 + }, + { + "epoch": 1.1238570073272642, + "grad_norm": 1.0984737873077393, + "learning_rate": 0.0001, + "loss": 72.7335, + "step": 6960 + }, + { + "epoch": 1.1254718314123655, + "grad_norm": 0.6798779368400574, + "learning_rate": 0.0001, + "loss": 72.1942, + "step": 6970 + }, + { + "epoch": 1.1270866554974668, + "grad_norm": 0.7546010613441467, + "learning_rate": 0.0001, + "loss": 70.8014, + "step": 6980 + }, + { + "epoch": 1.128701479582568, + "grad_norm": 1.0238053798675537, + "learning_rate": 0.0001, + "loss": 72.5617, + "step": 6990 + }, + { + "epoch": 1.130316303667669, + "grad_norm": 0.6535419821739197, + "learning_rate": 0.0001, + "loss": 70.2193, + "step": 7000 + }, + { + "epoch": 1.1319311277527704, + "grad_norm": 0.7354230284690857, + "learning_rate": 0.0001, + "loss": 72.0685, + "step": 7010 + }, + { + "epoch": 1.1335459518378717, + "grad_norm": 0.7848799824714661, + "learning_rate": 0.0001, + "loss": 72.2869, + "step": 7020 + }, + { + "epoch": 1.135160775922973, + "grad_norm": 0.6519262790679932, + "learning_rate": 0.0001, + "loss": 72.2023, + "step": 7030 + }, + { + "epoch": 1.1367756000080742, + "grad_norm": 0.8428511023521423, + "learning_rate": 0.0001, + "loss": 71.3396, + "step": 7040 + }, + { + "epoch": 1.1383904240931753, + "grad_norm": 0.8873165249824524, + "learning_rate": 0.0001, + "loss": 69.8284, + "step": 7050 + }, + { + "epoch": 1.1400052481782765, + "grad_norm": 0.9141374230384827, + "learning_rate": 0.0001, + "loss": 70.3666, + "step": 7060 + }, + { + "epoch": 1.1416200722633778, + "grad_norm": 0.6556552648544312, + "learning_rate": 0.0001, + "loss": 71.1316, + "step": 7070 + }, + { + "epoch": 1.143234896348479, + "grad_norm": 0.6736201047897339, + "learning_rate": 0.0001, + "loss": 70.2601, + "step": 7080 + }, + { + "epoch": 1.1448497204335804, + "grad_norm": 0.6965599656105042, + "learning_rate": 0.0001, + "loss": 71.9409, + "step": 7090 + }, + { + "epoch": 1.1464645445186814, + "grad_norm": 1.0600581169128418, + "learning_rate": 0.0001, + "loss": 72.7344, + "step": 7100 + }, + { + "epoch": 1.1480793686037827, + "grad_norm": 0.7501764297485352, + "learning_rate": 0.0001, + "loss": 72.3129, + "step": 7110 + }, + { + "epoch": 1.149694192688884, + "grad_norm": 0.7105675935745239, + "learning_rate": 0.0001, + "loss": 71.1356, + "step": 7120 + }, + { + "epoch": 1.1513090167739852, + "grad_norm": 0.7446026802062988, + "learning_rate": 0.0001, + "loss": 69.8208, + "step": 7130 + }, + { + "epoch": 1.1529238408590865, + "grad_norm": 0.6664865016937256, + "learning_rate": 0.0001, + "loss": 72.2466, + "step": 7140 + }, + { + "epoch": 1.1545386649441876, + "grad_norm": 1.7590374946594238, + "learning_rate": 0.0001, + "loss": 71.4987, + "step": 7150 + }, + { + "epoch": 1.1561534890292888, + "grad_norm": 0.7941271066665649, + "learning_rate": 0.0001, + "loss": 70.787, + "step": 7160 + }, + { + "epoch": 1.15776831311439, + "grad_norm": 0.7473573088645935, + "learning_rate": 0.0001, + "loss": 71.2516, + "step": 7170 + }, + { + "epoch": 1.1593831371994914, + "grad_norm": 0.7589637041091919, + "learning_rate": 0.0001, + "loss": 71.4781, + "step": 7180 + }, + { + "epoch": 1.1609979612845926, + "grad_norm": 0.8148510456085205, + "learning_rate": 0.0001, + "loss": 71.1491, + "step": 7190 + }, + { + "epoch": 1.1626127853696937, + "grad_norm": 0.6803365349769592, + "learning_rate": 0.0001, + "loss": 71.7937, + "step": 7200 + }, + { + "epoch": 1.164227609454795, + "grad_norm": 0.7314755320549011, + "learning_rate": 0.0001, + "loss": 70.6566, + "step": 7210 + }, + { + "epoch": 1.1658424335398963, + "grad_norm": 0.6494631171226501, + "learning_rate": 0.0001, + "loss": 70.5057, + "step": 7220 + }, + { + "epoch": 1.1674572576249975, + "grad_norm": 0.6076239943504333, + "learning_rate": 0.0001, + "loss": 72.2985, + "step": 7230 + }, + { + "epoch": 1.1690720817100988, + "grad_norm": 0.6650206446647644, + "learning_rate": 0.0001, + "loss": 71.0719, + "step": 7240 + }, + { + "epoch": 1.1706869057951999, + "grad_norm": 0.6926266551017761, + "learning_rate": 0.0001, + "loss": 70.3714, + "step": 7250 + }, + { + "epoch": 1.1723017298803011, + "grad_norm": 0.6534716486930847, + "learning_rate": 0.0001, + "loss": 71.3194, + "step": 7260 + }, + { + "epoch": 1.1739165539654024, + "grad_norm": 0.7194528579711914, + "learning_rate": 0.0001, + "loss": 72.3277, + "step": 7270 + }, + { + "epoch": 1.1755313780505037, + "grad_norm": 0.7496437430381775, + "learning_rate": 0.0001, + "loss": 71.1136, + "step": 7280 + }, + { + "epoch": 1.177146202135605, + "grad_norm": 0.690522313117981, + "learning_rate": 0.0001, + "loss": 70.065, + "step": 7290 + }, + { + "epoch": 1.178761026220706, + "grad_norm": 0.6966914534568787, + "learning_rate": 0.0001, + "loss": 73.1137, + "step": 7300 + }, + { + "epoch": 1.1803758503058073, + "grad_norm": 0.7304641604423523, + "learning_rate": 0.0001, + "loss": 71.2466, + "step": 7310 + }, + { + "epoch": 1.1819906743909085, + "grad_norm": 0.7564008235931396, + "learning_rate": 0.0001, + "loss": 71.4147, + "step": 7320 + }, + { + "epoch": 1.1836054984760098, + "grad_norm": 0.61886066198349, + "learning_rate": 0.0001, + "loss": 70.3297, + "step": 7330 + }, + { + "epoch": 1.185220322561111, + "grad_norm": 0.6281574964523315, + "learning_rate": 0.0001, + "loss": 70.9607, + "step": 7340 + }, + { + "epoch": 1.1868351466462121, + "grad_norm": 0.6583316326141357, + "learning_rate": 0.0001, + "loss": 70.3443, + "step": 7350 + }, + { + "epoch": 1.1884499707313134, + "grad_norm": 1.8159942626953125, + "learning_rate": 0.0001, + "loss": 71.4792, + "step": 7360 + }, + { + "epoch": 1.1900647948164147, + "grad_norm": 0.7253140807151794, + "learning_rate": 0.0001, + "loss": 72.2182, + "step": 7370 + }, + { + "epoch": 1.191679618901516, + "grad_norm": 0.9141016602516174, + "learning_rate": 0.0001, + "loss": 68.7443, + "step": 7380 + }, + { + "epoch": 1.193294442986617, + "grad_norm": 0.7392430901527405, + "learning_rate": 0.0001, + "loss": 69.1888, + "step": 7390 + }, + { + "epoch": 1.1949092670717183, + "grad_norm": 0.7713924646377563, + "learning_rate": 0.0001, + "loss": 69.1301, + "step": 7400 + }, + { + "epoch": 1.1965240911568196, + "grad_norm": 0.6000798344612122, + "learning_rate": 0.0001, + "loss": 71.2576, + "step": 7410 + }, + { + "epoch": 1.1981389152419208, + "grad_norm": 0.687111496925354, + "learning_rate": 0.0001, + "loss": 70.2919, + "step": 7420 + }, + { + "epoch": 1.1997537393270221, + "grad_norm": 1.1718226671218872, + "learning_rate": 0.0001, + "loss": 71.9498, + "step": 7430 + }, + { + "epoch": 1.2013685634121232, + "grad_norm": 0.7176038026809692, + "learning_rate": 0.0001, + "loss": 71.91, + "step": 7440 + }, + { + "epoch": 1.2029833874972244, + "grad_norm": 0.7705256938934326, + "learning_rate": 0.0001, + "loss": 70.1241, + "step": 7450 + }, + { + "epoch": 1.2045982115823257, + "grad_norm": 0.8004609942436218, + "learning_rate": 0.0001, + "loss": 69.5121, + "step": 7460 + }, + { + "epoch": 1.206213035667427, + "grad_norm": 0.6708329319953918, + "learning_rate": 0.0001, + "loss": 73.0173, + "step": 7470 + }, + { + "epoch": 1.2078278597525283, + "grad_norm": 0.8777983784675598, + "learning_rate": 0.0001, + "loss": 71.6843, + "step": 7480 + }, + { + "epoch": 1.2094426838376293, + "grad_norm": 0.7683864831924438, + "learning_rate": 0.0001, + "loss": 71.6914, + "step": 7490 + }, + { + "epoch": 1.2110575079227306, + "grad_norm": 0.8022210597991943, + "learning_rate": 0.0001, + "loss": 69.1724, + "step": 7500 + }, + { + "epoch": 1.2126723320078319, + "grad_norm": 0.7250640392303467, + "learning_rate": 0.0001, + "loss": 70.6568, + "step": 7510 + }, + { + "epoch": 1.2142871560929331, + "grad_norm": 0.5625214576721191, + "learning_rate": 0.0001, + "loss": 69.4849, + "step": 7520 + }, + { + "epoch": 1.2159019801780344, + "grad_norm": 0.7980468273162842, + "learning_rate": 0.0001, + "loss": 70.8164, + "step": 7530 + }, + { + "epoch": 1.2175168042631355, + "grad_norm": 0.6900951862335205, + "learning_rate": 0.0001, + "loss": 71.1719, + "step": 7540 + }, + { + "epoch": 1.2191316283482367, + "grad_norm": 0.6284971833229065, + "learning_rate": 0.0001, + "loss": 70.1751, + "step": 7550 + }, + { + "epoch": 1.220746452433338, + "grad_norm": 0.6641798615455627, + "learning_rate": 0.0001, + "loss": 71.9043, + "step": 7560 + }, + { + "epoch": 1.2223612765184393, + "grad_norm": 0.7439492344856262, + "learning_rate": 0.0001, + "loss": 70.7286, + "step": 7570 + }, + { + "epoch": 1.2239761006035406, + "grad_norm": 0.8018355369567871, + "learning_rate": 0.0001, + "loss": 71.638, + "step": 7580 + }, + { + "epoch": 1.2255909246886416, + "grad_norm": 0.7318522930145264, + "learning_rate": 0.0001, + "loss": 70.0867, + "step": 7590 + }, + { + "epoch": 1.2272057487737429, + "grad_norm": 0.6799285411834717, + "learning_rate": 0.0001, + "loss": 71.3427, + "step": 7600 + }, + { + "epoch": 1.2288205728588442, + "grad_norm": 0.6796864867210388, + "learning_rate": 0.0001, + "loss": 71.1698, + "step": 7610 + }, + { + "epoch": 1.2304353969439454, + "grad_norm": 0.7256141304969788, + "learning_rate": 0.0001, + "loss": 71.1051, + "step": 7620 + }, + { + "epoch": 1.2320502210290467, + "grad_norm": 0.8318199515342712, + "learning_rate": 0.0001, + "loss": 71.3179, + "step": 7630 + }, + { + "epoch": 1.2336650451141478, + "grad_norm": 0.6594699025154114, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 7640 + }, + { + "epoch": 1.235279869199249, + "grad_norm": 0.7900195717811584, + "learning_rate": 0.0001, + "loss": 72.0139, + "step": 7650 + }, + { + "epoch": 1.2368946932843503, + "grad_norm": 0.7845672965049744, + "learning_rate": 0.0001, + "loss": 70.808, + "step": 7660 + }, + { + "epoch": 1.2385095173694516, + "grad_norm": 0.6598045229911804, + "learning_rate": 0.0001, + "loss": 69.8953, + "step": 7670 + }, + { + "epoch": 1.2401243414545529, + "grad_norm": 1.3222978115081787, + "learning_rate": 0.0001, + "loss": 68.787, + "step": 7680 + }, + { + "epoch": 1.241739165539654, + "grad_norm": 0.6660508513450623, + "learning_rate": 0.0001, + "loss": 71.5831, + "step": 7690 + }, + { + "epoch": 1.2433539896247552, + "grad_norm": 0.6305698156356812, + "learning_rate": 0.0001, + "loss": 70.3232, + "step": 7700 + }, + { + "epoch": 1.2449688137098565, + "grad_norm": 1.2648323774337769, + "learning_rate": 0.0001, + "loss": 67.8901, + "step": 7710 + }, + { + "epoch": 1.2465836377949577, + "grad_norm": 0.7177619934082031, + "learning_rate": 0.0001, + "loss": 69.6582, + "step": 7720 + }, + { + "epoch": 1.248198461880059, + "grad_norm": 0.6716091632843018, + "learning_rate": 0.0001, + "loss": 70.2034, + "step": 7730 + }, + { + "epoch": 1.24981328596516, + "grad_norm": 0.6050158143043518, + "learning_rate": 0.0001, + "loss": 69.3942, + "step": 7740 + }, + { + "epoch": 1.2514281100502613, + "grad_norm": 0.7001014947891235, + "learning_rate": 0.0001, + "loss": 69.5971, + "step": 7750 + }, + { + "epoch": 1.2530429341353626, + "grad_norm": 0.7221280932426453, + "learning_rate": 0.0001, + "loss": 71.3651, + "step": 7760 + }, + { + "epoch": 1.2546577582204639, + "grad_norm": 0.9130340218544006, + "learning_rate": 0.0001, + "loss": 68.3038, + "step": 7770 + }, + { + "epoch": 1.2562725823055652, + "grad_norm": 0.7703199982643127, + "learning_rate": 0.0001, + "loss": 71.6127, + "step": 7780 + }, + { + "epoch": 1.2578874063906662, + "grad_norm": 0.6334695816040039, + "learning_rate": 0.0001, + "loss": 68.9074, + "step": 7790 + }, + { + "epoch": 1.2595022304757675, + "grad_norm": 0.7074242234230042, + "learning_rate": 0.0001, + "loss": 70.0569, + "step": 7800 + }, + { + "epoch": 1.2611170545608688, + "grad_norm": 0.6956025958061218, + "learning_rate": 0.0001, + "loss": 69.892, + "step": 7810 + }, + { + "epoch": 1.26273187864597, + "grad_norm": 0.7235708236694336, + "learning_rate": 0.0001, + "loss": 71.1777, + "step": 7820 + }, + { + "epoch": 1.2643467027310713, + "grad_norm": 0.6157007813453674, + "learning_rate": 0.0001, + "loss": 71.5091, + "step": 7830 + }, + { + "epoch": 1.2659615268161724, + "grad_norm": 0.5352533459663391, + "learning_rate": 0.0001, + "loss": 70.083, + "step": 7840 + }, + { + "epoch": 1.2675763509012736, + "grad_norm": 0.7171182632446289, + "learning_rate": 0.0001, + "loss": 68.4585, + "step": 7850 + }, + { + "epoch": 1.269191174986375, + "grad_norm": 0.7503113746643066, + "learning_rate": 0.0001, + "loss": 70.4206, + "step": 7860 + }, + { + "epoch": 1.2708059990714762, + "grad_norm": 0.6552693843841553, + "learning_rate": 0.0001, + "loss": 70.8455, + "step": 7870 + }, + { + "epoch": 1.2724208231565775, + "grad_norm": 0.7173179984092712, + "learning_rate": 0.0001, + "loss": 68.82, + "step": 7880 + }, + { + "epoch": 1.2740356472416785, + "grad_norm": 0.7352942824363708, + "learning_rate": 0.0001, + "loss": 70.4974, + "step": 7890 + }, + { + "epoch": 1.2756504713267798, + "grad_norm": 0.6120197176933289, + "learning_rate": 0.0001, + "loss": 72.1638, + "step": 7900 + }, + { + "epoch": 1.277265295411881, + "grad_norm": 0.6271925568580627, + "learning_rate": 0.0001, + "loss": 69.3267, + "step": 7910 + }, + { + "epoch": 1.2788801194969823, + "grad_norm": 0.752636194229126, + "learning_rate": 0.0001, + "loss": 70.6761, + "step": 7920 + }, + { + "epoch": 1.2804949435820836, + "grad_norm": 0.7307374477386475, + "learning_rate": 0.0001, + "loss": 70.2909, + "step": 7930 + }, + { + "epoch": 1.2821097676671847, + "grad_norm": 0.5977495908737183, + "learning_rate": 0.0001, + "loss": 72.2969, + "step": 7940 + }, + { + "epoch": 1.283724591752286, + "grad_norm": 0.7927457094192505, + "learning_rate": 0.0001, + "loss": 70.3784, + "step": 7950 + }, + { + "epoch": 1.2853394158373872, + "grad_norm": 0.7520087957382202, + "learning_rate": 0.0001, + "loss": 70.7122, + "step": 7960 + }, + { + "epoch": 1.2869542399224885, + "grad_norm": 0.6036237478256226, + "learning_rate": 0.0001, + "loss": 69.8718, + "step": 7970 + }, + { + "epoch": 1.2885690640075897, + "grad_norm": 0.6805875897407532, + "learning_rate": 0.0001, + "loss": 70.2391, + "step": 7980 + }, + { + "epoch": 1.2901838880926908, + "grad_norm": 0.774559497833252, + "learning_rate": 0.0001, + "loss": 71.1605, + "step": 7990 + }, + { + "epoch": 1.291798712177792, + "grad_norm": 0.6991063356399536, + "learning_rate": 0.0001, + "loss": 69.9779, + "step": 8000 + }, + { + "epoch": 1.2934135362628933, + "grad_norm": 0.5877949595451355, + "learning_rate": 0.0001, + "loss": 71.16, + "step": 8010 + }, + { + "epoch": 1.2950283603479946, + "grad_norm": 0.8840436339378357, + "learning_rate": 0.0001, + "loss": 70.9454, + "step": 8020 + }, + { + "epoch": 1.296643184433096, + "grad_norm": 0.7532126307487488, + "learning_rate": 0.0001, + "loss": 70.141, + "step": 8030 + }, + { + "epoch": 1.298258008518197, + "grad_norm": 0.6880044341087341, + "learning_rate": 0.0001, + "loss": 70.0746, + "step": 8040 + }, + { + "epoch": 1.2998728326032982, + "grad_norm": 0.6319366097450256, + "learning_rate": 0.0001, + "loss": 69.9396, + "step": 8050 + }, + { + "epoch": 1.3014876566883995, + "grad_norm": 0.6329445838928223, + "learning_rate": 0.0001, + "loss": 69.9043, + "step": 8060 + }, + { + "epoch": 1.3031024807735008, + "grad_norm": 0.8892332315444946, + "learning_rate": 0.0001, + "loss": 69.4187, + "step": 8070 + }, + { + "epoch": 1.304717304858602, + "grad_norm": 0.696500301361084, + "learning_rate": 0.0001, + "loss": 68.1885, + "step": 8080 + }, + { + "epoch": 1.306332128943703, + "grad_norm": 0.5933213233947754, + "learning_rate": 0.0001, + "loss": 68.8017, + "step": 8090 + }, + { + "epoch": 1.3079469530288044, + "grad_norm": 0.9274749159812927, + "learning_rate": 0.0001, + "loss": 71.5875, + "step": 8100 + }, + { + "epoch": 1.3095617771139056, + "grad_norm": 0.7540446519851685, + "learning_rate": 0.0001, + "loss": 71.1233, + "step": 8110 + }, + { + "epoch": 1.311176601199007, + "grad_norm": 0.6701658964157104, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8120 + }, + { + "epoch": 1.3127914252841082, + "grad_norm": 0.6096424460411072, + "learning_rate": 0.0001, + "loss": 69.9583, + "step": 8130 + }, + { + "epoch": 1.3144062493692092, + "grad_norm": 0.809524416923523, + "learning_rate": 0.0001, + "loss": 71.8149, + "step": 8140 + }, + { + "epoch": 1.3160210734543105, + "grad_norm": 0.7928242683410645, + "learning_rate": 0.0001, + "loss": 70.2084, + "step": 8150 + }, + { + "epoch": 1.3176358975394118, + "grad_norm": 0.7462850213050842, + "learning_rate": 0.0001, + "loss": 71.4789, + "step": 8160 + }, + { + "epoch": 1.319250721624513, + "grad_norm": 0.8166030049324036, + "learning_rate": 0.0001, + "loss": 69.3154, + "step": 8170 + }, + { + "epoch": 1.3208655457096143, + "grad_norm": 0.7002414464950562, + "learning_rate": 0.0001, + "loss": 70.8048, + "step": 8180 + }, + { + "epoch": 1.3224803697947154, + "grad_norm": 0.7653809189796448, + "learning_rate": 0.0001, + "loss": 70.0221, + "step": 8190 + }, + { + "epoch": 1.3240951938798167, + "grad_norm": 0.6777089238166809, + "learning_rate": 0.0001, + "loss": 70.614, + "step": 8200 + }, + { + "epoch": 1.325710017964918, + "grad_norm": 0.692739725112915, + "learning_rate": 0.0001, + "loss": 71.2311, + "step": 8210 + }, + { + "epoch": 1.3273248420500192, + "grad_norm": 0.7985519766807556, + "learning_rate": 0.0001, + "loss": 68.1982, + "step": 8220 + }, + { + "epoch": 1.3289396661351205, + "grad_norm": 0.7002689242362976, + "learning_rate": 0.0001, + "loss": 70.9181, + "step": 8230 + }, + { + "epoch": 1.3305544902202215, + "grad_norm": 0.7029191255569458, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 8240 + }, + { + "epoch": 1.3321693143053228, + "grad_norm": 0.7292912602424622, + "learning_rate": 0.0001, + "loss": 71.5386, + "step": 8250 + }, + { + "epoch": 1.333784138390424, + "grad_norm": 0.6141605973243713, + "learning_rate": 0.0001, + "loss": 72.3396, + "step": 8260 + }, + { + "epoch": 1.3353989624755254, + "grad_norm": 0.6486957669258118, + "learning_rate": 0.0001, + "loss": 69.7607, + "step": 8270 + }, + { + "epoch": 1.3370137865606266, + "grad_norm": 0.7813800573348999, + "learning_rate": 0.0001, + "loss": 68.8979, + "step": 8280 + }, + { + "epoch": 1.3386286106457277, + "grad_norm": 0.6805126070976257, + "learning_rate": 0.0001, + "loss": 70.2179, + "step": 8290 + }, + { + "epoch": 1.340243434730829, + "grad_norm": 0.7202772498130798, + "learning_rate": 0.0001, + "loss": 72.5008, + "step": 8300 + }, + { + "epoch": 1.3418582588159302, + "grad_norm": 0.6881810426712036, + "learning_rate": 0.0001, + "loss": 70.1987, + "step": 8310 + }, + { + "epoch": 1.3434730829010315, + "grad_norm": 0.6337536573410034, + "learning_rate": 0.0001, + "loss": 70.4073, + "step": 8320 + }, + { + "epoch": 1.3450879069861328, + "grad_norm": 0.7313241958618164, + "learning_rate": 0.0001, + "loss": 69.3598, + "step": 8330 + }, + { + "epoch": 1.3467027310712338, + "grad_norm": 0.635947585105896, + "learning_rate": 0.0001, + "loss": 69.5143, + "step": 8340 + }, + { + "epoch": 1.3483175551563351, + "grad_norm": 0.6219074726104736, + "learning_rate": 0.0001, + "loss": 69.9903, + "step": 8350 + }, + { + "epoch": 1.3499323792414364, + "grad_norm": 0.5928048491477966, + "learning_rate": 0.0001, + "loss": 70.4222, + "step": 8360 + }, + { + "epoch": 1.3515472033265377, + "grad_norm": 0.5927143096923828, + "learning_rate": 0.0001, + "loss": 69.8642, + "step": 8370 + }, + { + "epoch": 1.353162027411639, + "grad_norm": 0.7614993453025818, + "learning_rate": 0.0001, + "loss": 69.5787, + "step": 8380 + }, + { + "epoch": 1.35477685149674, + "grad_norm": 0.5488380193710327, + "learning_rate": 0.0001, + "loss": 69.0028, + "step": 8390 + }, + { + "epoch": 1.3563916755818413, + "grad_norm": 0.6219164133071899, + "learning_rate": 0.0001, + "loss": 69.4591, + "step": 8400 + }, + { + "epoch": 1.3580064996669425, + "grad_norm": 0.6940358877182007, + "learning_rate": 0.0001, + "loss": 71.3795, + "step": 8410 + }, + { + "epoch": 1.3596213237520438, + "grad_norm": 0.5919192433357239, + "learning_rate": 0.0001, + "loss": 71.2761, + "step": 8420 + }, + { + "epoch": 1.361236147837145, + "grad_norm": 1.0803378820419312, + "learning_rate": 0.0001, + "loss": 69.0046, + "step": 8430 + }, + { + "epoch": 1.3628509719222461, + "grad_norm": 0.7028629183769226, + "learning_rate": 0.0001, + "loss": 71.5996, + "step": 8440 + }, + { + "epoch": 1.3644657960073474, + "grad_norm": 0.6829854249954224, + "learning_rate": 0.0001, + "loss": 67.8831, + "step": 8450 + }, + { + "epoch": 1.3660806200924487, + "grad_norm": 0.6937337517738342, + "learning_rate": 0.0001, + "loss": 69.8321, + "step": 8460 + }, + { + "epoch": 1.36769544417755, + "grad_norm": 0.7725892066955566, + "learning_rate": 0.0001, + "loss": 71.0784, + "step": 8470 + }, + { + "epoch": 1.3693102682626512, + "grad_norm": 0.7171871066093445, + "learning_rate": 0.0001, + "loss": 69.0828, + "step": 8480 + }, + { + "epoch": 1.3709250923477523, + "grad_norm": 0.6605113744735718, + "learning_rate": 0.0001, + "loss": 70.9461, + "step": 8490 + }, + { + "epoch": 1.3725399164328536, + "grad_norm": 0.9005604982376099, + "learning_rate": 0.0001, + "loss": 71.474, + "step": 8500 + }, + { + "epoch": 1.3741547405179548, + "grad_norm": 0.7454419732093811, + "learning_rate": 0.0001, + "loss": 70.3113, + "step": 8510 + }, + { + "epoch": 1.375769564603056, + "grad_norm": 0.7574180960655212, + "learning_rate": 0.0001, + "loss": 70.9015, + "step": 8520 + }, + { + "epoch": 1.3773843886881574, + "grad_norm": 0.6525025367736816, + "learning_rate": 0.0001, + "loss": 72.3132, + "step": 8530 + }, + { + "epoch": 1.3789992127732584, + "grad_norm": 0.6548861861228943, + "learning_rate": 0.0001, + "loss": 68.6495, + "step": 8540 + }, + { + "epoch": 1.3806140368583597, + "grad_norm": 0.7881495952606201, + "learning_rate": 0.0001, + "loss": 70.1335, + "step": 8550 + }, + { + "epoch": 1.382228860943461, + "grad_norm": 0.619380533695221, + "learning_rate": 0.0001, + "loss": 71.7469, + "step": 8560 + }, + { + "epoch": 1.3838436850285623, + "grad_norm": 0.7637182474136353, + "learning_rate": 0.0001, + "loss": 69.4691, + "step": 8570 + }, + { + "epoch": 1.3854585091136635, + "grad_norm": 0.8711991906166077, + "learning_rate": 0.0001, + "loss": 70.8131, + "step": 8580 + }, + { + "epoch": 1.3870733331987646, + "grad_norm": 0.6550050973892212, + "learning_rate": 0.0001, + "loss": 70.0096, + "step": 8590 + }, + { + "epoch": 1.3886881572838659, + "grad_norm": 0.6310496926307678, + "learning_rate": 0.0001, + "loss": 70.4305, + "step": 8600 + }, + { + "epoch": 1.3903029813689671, + "grad_norm": 0.5780142545700073, + "learning_rate": 0.0001, + "loss": 68.9639, + "step": 8610 + }, + { + "epoch": 1.3919178054540684, + "grad_norm": 0.6526662111282349, + "learning_rate": 0.0001, + "loss": 69.1345, + "step": 8620 + }, + { + "epoch": 1.3935326295391697, + "grad_norm": 1.2579848766326904, + "learning_rate": 0.0001, + "loss": 71.6908, + "step": 8630 + }, + { + "epoch": 1.3951474536242707, + "grad_norm": 0.7702625393867493, + "learning_rate": 0.0001, + "loss": 69.1064, + "step": 8640 + }, + { + "epoch": 1.396762277709372, + "grad_norm": 0.7414085865020752, + "learning_rate": 0.0001, + "loss": 69.5106, + "step": 8650 + }, + { + "epoch": 1.3983771017944733, + "grad_norm": 0.666235089302063, + "learning_rate": 0.0001, + "loss": 67.938, + "step": 8660 + }, + { + "epoch": 1.3999919258795746, + "grad_norm": 0.7879519462585449, + "learning_rate": 0.0001, + "loss": 69.5422, + "step": 8670 + }, + { + "epoch": 1.4016067499646758, + "grad_norm": 0.6552250981330872, + "learning_rate": 0.0001, + "loss": 69.4161, + "step": 8680 + }, + { + "epoch": 1.4032215740497769, + "grad_norm": 0.7558363676071167, + "learning_rate": 0.0001, + "loss": 71.3052, + "step": 8690 + }, + { + "epoch": 1.4048363981348782, + "grad_norm": 0.677007794380188, + "learning_rate": 0.0001, + "loss": 70.0824, + "step": 8700 + }, + { + "epoch": 1.4064512222199794, + "grad_norm": 0.6416738629341125, + "learning_rate": 0.0001, + "loss": 68.3357, + "step": 8710 + }, + { + "epoch": 1.4080660463050807, + "grad_norm": 0.8067912459373474, + "learning_rate": 0.0001, + "loss": 69.8551, + "step": 8720 + }, + { + "epoch": 1.409680870390182, + "grad_norm": 0.6017255783081055, + "learning_rate": 0.0001, + "loss": 70.3936, + "step": 8730 + }, + { + "epoch": 1.411295694475283, + "grad_norm": 0.5473611354827881, + "learning_rate": 0.0001, + "loss": 69.1556, + "step": 8740 + }, + { + "epoch": 1.4129105185603843, + "grad_norm": 0.641984760761261, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 8750 + }, + { + "epoch": 1.4145253426454856, + "grad_norm": 0.7509932518005371, + "learning_rate": 0.0001, + "loss": 68.793, + "step": 8760 + }, + { + "epoch": 1.4161401667305868, + "grad_norm": 0.615459144115448, + "learning_rate": 0.0001, + "loss": 69.0787, + "step": 8770 + }, + { + "epoch": 1.4177549908156881, + "grad_norm": 0.625276505947113, + "learning_rate": 0.0001, + "loss": 68.5643, + "step": 8780 + }, + { + "epoch": 1.4193698149007892, + "grad_norm": 0.7294536828994751, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 8790 + }, + { + "epoch": 1.4209846389858904, + "grad_norm": 0.8188890814781189, + "learning_rate": 0.0001, + "loss": 70.7486, + "step": 8800 + }, + { + "epoch": 1.4225994630709917, + "grad_norm": 0.8662137389183044, + "learning_rate": 0.0001, + "loss": 69.0625, + "step": 8810 + }, + { + "epoch": 1.424214287156093, + "grad_norm": 0.7548071146011353, + "learning_rate": 0.0001, + "loss": 68.9828, + "step": 8820 + }, + { + "epoch": 1.4258291112411943, + "grad_norm": 0.7295185327529907, + "learning_rate": 0.0001, + "loss": 69.0606, + "step": 8830 + }, + { + "epoch": 1.4274439353262953, + "grad_norm": 0.7937467098236084, + "learning_rate": 0.0001, + "loss": 69.0135, + "step": 8840 + }, + { + "epoch": 1.4290587594113966, + "grad_norm": 0.5957842469215393, + "learning_rate": 0.0001, + "loss": 68.3992, + "step": 8850 + }, + { + "epoch": 1.4306735834964979, + "grad_norm": 0.786580502986908, + "learning_rate": 0.0001, + "loss": 69.4485, + "step": 8860 + }, + { + "epoch": 1.4322884075815991, + "grad_norm": 0.8539725542068481, + "learning_rate": 0.0001, + "loss": 67.1037, + "step": 8870 + }, + { + "epoch": 1.4339032316667004, + "grad_norm": 0.684476375579834, + "learning_rate": 0.0001, + "loss": 68.9814, + "step": 8880 + }, + { + "epoch": 1.4355180557518015, + "grad_norm": 0.6335768103599548, + "learning_rate": 0.0001, + "loss": 69.9716, + "step": 8890 + }, + { + "epoch": 1.4371328798369027, + "grad_norm": 0.9881808757781982, + "learning_rate": 0.0001, + "loss": 68.0338, + "step": 8900 + }, + { + "epoch": 1.438747703922004, + "grad_norm": 0.783983588218689, + "learning_rate": 0.0001, + "loss": 70.124, + "step": 8910 + }, + { + "epoch": 1.4403625280071053, + "grad_norm": 0.817820131778717, + "learning_rate": 0.0001, + "loss": 67.933, + "step": 8920 + }, + { + "epoch": 1.4419773520922066, + "grad_norm": 0.6900199055671692, + "learning_rate": 0.0001, + "loss": 69.8871, + "step": 8930 + }, + { + "epoch": 1.4435921761773076, + "grad_norm": 0.6702716946601868, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8940 + }, + { + "epoch": 1.445207000262409, + "grad_norm": 0.6699396967887878, + "learning_rate": 0.0001, + "loss": 67.8654, + "step": 8950 + }, + { + "epoch": 1.4468218243475102, + "grad_norm": 0.6388489603996277, + "learning_rate": 0.0001, + "loss": 69.5605, + "step": 8960 + }, + { + "epoch": 1.4484366484326114, + "grad_norm": 0.735910952091217, + "learning_rate": 0.0001, + "loss": 66.7865, + "step": 8970 + }, + { + "epoch": 1.4500514725177127, + "grad_norm": 0.6826426386833191, + "learning_rate": 0.0001, + "loss": 68.2896, + "step": 8980 + }, + { + "epoch": 1.4516662966028138, + "grad_norm": 0.8190213441848755, + "learning_rate": 0.0001, + "loss": 70.3524, + "step": 8990 + }, + { + "epoch": 1.453281120687915, + "grad_norm": 0.6774603724479675, + "learning_rate": 0.0001, + "loss": 68.9084, + "step": 9000 + }, + { + "epoch": 1.4548959447730163, + "grad_norm": 0.8953077793121338, + "learning_rate": 0.0001, + "loss": 69.228, + "step": 9010 + }, + { + "epoch": 1.4565107688581176, + "grad_norm": 1.8509501218795776, + "learning_rate": 0.0001, + "loss": 68.0847, + "step": 9020 + }, + { + "epoch": 1.4581255929432189, + "grad_norm": 0.671280026435852, + "learning_rate": 0.0001, + "loss": 70.0691, + "step": 9030 + }, + { + "epoch": 1.45974041702832, + "grad_norm": 0.7242400646209717, + "learning_rate": 0.0001, + "loss": 68.8001, + "step": 9040 + }, + { + "epoch": 1.4613552411134212, + "grad_norm": 0.6609033942222595, + "learning_rate": 0.0001, + "loss": 69.0398, + "step": 9050 + }, + { + "epoch": 1.4629700651985225, + "grad_norm": 0.6850711703300476, + "learning_rate": 0.0001, + "loss": 70.8849, + "step": 9060 + }, + { + "epoch": 1.4645848892836237, + "grad_norm": 0.7409449219703674, + "learning_rate": 0.0001, + "loss": 68.0411, + "step": 9070 + }, + { + "epoch": 1.466199713368725, + "grad_norm": 0.6832754611968994, + "learning_rate": 0.0001, + "loss": 69.1219, + "step": 9080 + }, + { + "epoch": 1.467814537453826, + "grad_norm": 0.6671826243400574, + "learning_rate": 0.0001, + "loss": 68.1225, + "step": 9090 + }, + { + "epoch": 1.4694293615389273, + "grad_norm": 0.6758706569671631, + "learning_rate": 0.0001, + "loss": 68.6981, + "step": 9100 + }, + { + "epoch": 1.4710441856240286, + "grad_norm": 0.7885150909423828, + "learning_rate": 0.0001, + "loss": 69.29, + "step": 9110 + }, + { + "epoch": 1.4726590097091299, + "grad_norm": 0.7636108994483948, + "learning_rate": 0.0001, + "loss": 67.3151, + "step": 9120 + }, + { + "epoch": 1.4742738337942312, + "grad_norm": 0.6518180966377258, + "learning_rate": 0.0001, + "loss": 68.6208, + "step": 9130 + }, + { + "epoch": 1.4758886578793322, + "grad_norm": 0.6715161204338074, + "learning_rate": 0.0001, + "loss": 70.5106, + "step": 9140 + }, + { + "epoch": 1.4775034819644335, + "grad_norm": 0.6342995762825012, + "learning_rate": 0.0001, + "loss": 70.3067, + "step": 9150 + }, + { + "epoch": 1.4791183060495348, + "grad_norm": 0.6773675084114075, + "learning_rate": 0.0001, + "loss": 69.2888, + "step": 9160 + }, + { + "epoch": 1.480733130134636, + "grad_norm": 0.6887578368186951, + "learning_rate": 0.0001, + "loss": 69.2788, + "step": 9170 + }, + { + "epoch": 1.4823479542197373, + "grad_norm": 0.6793885231018066, + "learning_rate": 0.0001, + "loss": 68.3523, + "step": 9180 + }, + { + "epoch": 1.4839627783048384, + "grad_norm": 0.7483563423156738, + "learning_rate": 0.0001, + "loss": 67.9991, + "step": 9190 + }, + { + "epoch": 1.4855776023899396, + "grad_norm": 0.7007381916046143, + "learning_rate": 0.0001, + "loss": 67.5335, + "step": 9200 + }, + { + "epoch": 1.487192426475041, + "grad_norm": 0.7565139532089233, + "learning_rate": 0.0001, + "loss": 69.1093, + "step": 9210 + }, + { + "epoch": 1.4888072505601422, + "grad_norm": 1.0212360620498657, + "learning_rate": 0.0001, + "loss": 68.5061, + "step": 9220 + }, + { + "epoch": 1.4904220746452435, + "grad_norm": 0.9198092818260193, + "learning_rate": 0.0001, + "loss": 67.9735, + "step": 9230 + }, + { + "epoch": 1.4920368987303445, + "grad_norm": 0.6228001713752747, + "learning_rate": 0.0001, + "loss": 69.2231, + "step": 9240 + }, + { + "epoch": 1.4936517228154458, + "grad_norm": 0.6618618965148926, + "learning_rate": 0.0001, + "loss": 68.0434, + "step": 9250 + }, + { + "epoch": 1.495266546900547, + "grad_norm": 0.6241064071655273, + "learning_rate": 0.0001, + "loss": 68.4004, + "step": 9260 + }, + { + "epoch": 1.4968813709856483, + "grad_norm": 0.7336916327476501, + "learning_rate": 0.0001, + "loss": 69.0389, + "step": 9270 + }, + { + "epoch": 1.4984961950707496, + "grad_norm": 0.718125581741333, + "learning_rate": 0.0001, + "loss": 68.4923, + "step": 9280 + }, + { + "epoch": 1.5001110191558507, + "grad_norm": 0.6694948673248291, + "learning_rate": 0.0001, + "loss": 68.1006, + "step": 9290 + }, + { + "epoch": 1.501725843240952, + "grad_norm": 0.5815121531486511, + "learning_rate": 0.0001, + "loss": 68.5943, + "step": 9300 + }, + { + "epoch": 1.5033406673260532, + "grad_norm": 1.4076049327850342, + "learning_rate": 0.0001, + "loss": 68.1251, + "step": 9310 + }, + { + "epoch": 1.5049554914111543, + "grad_norm": 0.7294307947158813, + "learning_rate": 0.0001, + "loss": 68.0539, + "step": 9320 + }, + { + "epoch": 1.5065703154962558, + "grad_norm": 0.627821683883667, + "learning_rate": 0.0001, + "loss": 68.6859, + "step": 9330 + }, + { + "epoch": 1.5081851395813568, + "grad_norm": 0.6683267951011658, + "learning_rate": 0.0001, + "loss": 68.2727, + "step": 9340 + }, + { + "epoch": 1.509799963666458, + "grad_norm": 0.7070158123970032, + "learning_rate": 0.0001, + "loss": 69.1171, + "step": 9350 + }, + { + "epoch": 1.5114147877515594, + "grad_norm": 0.6324070692062378, + "learning_rate": 0.0001, + "loss": 69.7498, + "step": 9360 + }, + { + "epoch": 1.5130296118366604, + "grad_norm": 0.9814477562904358, + "learning_rate": 0.0001, + "loss": 69.1426, + "step": 9370 + }, + { + "epoch": 1.514644435921762, + "grad_norm": 0.5883511304855347, + "learning_rate": 0.0001, + "loss": 68.5363, + "step": 9380 + }, + { + "epoch": 1.516259260006863, + "grad_norm": 0.8325281739234924, + "learning_rate": 0.0001, + "loss": 67.5951, + "step": 9390 + }, + { + "epoch": 1.5178740840919642, + "grad_norm": 0.7456872463226318, + "learning_rate": 0.0001, + "loss": 68.1876, + "step": 9400 + }, + { + "epoch": 1.5194889081770655, + "grad_norm": 0.7692755460739136, + "learning_rate": 0.0001, + "loss": 69.1677, + "step": 9410 + }, + { + "epoch": 1.5211037322621666, + "grad_norm": 0.7346193194389343, + "learning_rate": 0.0001, + "loss": 69.8608, + "step": 9420 + }, + { + "epoch": 1.522718556347268, + "grad_norm": 0.7658166289329529, + "learning_rate": 0.0001, + "loss": 68.8584, + "step": 9430 + }, + { + "epoch": 1.524333380432369, + "grad_norm": 0.7892371416091919, + "learning_rate": 0.0001, + "loss": 68.3882, + "step": 9440 + }, + { + "epoch": 1.5259482045174704, + "grad_norm": 0.5943968296051025, + "learning_rate": 0.0001, + "loss": 68.7935, + "step": 9450 + }, + { + "epoch": 1.5275630286025716, + "grad_norm": 0.6611922383308411, + "learning_rate": 0.0001, + "loss": 69.0865, + "step": 9460 + }, + { + "epoch": 1.5291778526876727, + "grad_norm": 0.6697778105735779, + "learning_rate": 0.0001, + "loss": 68.6131, + "step": 9470 + }, + { + "epoch": 1.5307926767727742, + "grad_norm": 0.862293541431427, + "learning_rate": 0.0001, + "loss": 71.0167, + "step": 9480 + }, + { + "epoch": 1.5324075008578752, + "grad_norm": 0.7098677754402161, + "learning_rate": 0.0001, + "loss": 67.7216, + "step": 9490 + }, + { + "epoch": 1.5340223249429765, + "grad_norm": 0.6578338742256165, + "learning_rate": 0.0001, + "loss": 69.872, + "step": 9500 + }, + { + "epoch": 1.5356371490280778, + "grad_norm": 0.7272375822067261, + "learning_rate": 0.0001, + "loss": 70.0971, + "step": 9510 + }, + { + "epoch": 1.5372519731131788, + "grad_norm": 0.6497064828872681, + "learning_rate": 0.0001, + "loss": 70.5921, + "step": 9520 + }, + { + "epoch": 1.5388667971982803, + "grad_norm": 0.6691288352012634, + "learning_rate": 0.0001, + "loss": 68.5213, + "step": 9530 + }, + { + "epoch": 1.5404816212833814, + "grad_norm": 0.9180259108543396, + "learning_rate": 0.0001, + "loss": 67.0699, + "step": 9540 + }, + { + "epoch": 1.5420964453684827, + "grad_norm": 0.7062082886695862, + "learning_rate": 0.0001, + "loss": 69.3965, + "step": 9550 + }, + { + "epoch": 1.543711269453584, + "grad_norm": 0.711613655090332, + "learning_rate": 0.0001, + "loss": 67.8276, + "step": 9560 + }, + { + "epoch": 1.545326093538685, + "grad_norm": 0.670032799243927, + "learning_rate": 0.0001, + "loss": 68.1687, + "step": 9570 + }, + { + "epoch": 1.5469409176237865, + "grad_norm": 0.7531955242156982, + "learning_rate": 0.0001, + "loss": 69.2144, + "step": 9580 + }, + { + "epoch": 1.5485557417088875, + "grad_norm": 0.6759967803955078, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 9590 + }, + { + "epoch": 1.5501705657939888, + "grad_norm": 0.6134868860244751, + "learning_rate": 0.0001, + "loss": 67.4348, + "step": 9600 + }, + { + "epoch": 1.55178538987909, + "grad_norm": 0.638785719871521, + "learning_rate": 0.0001, + "loss": 68.4635, + "step": 9610 + }, + { + "epoch": 1.5534002139641911, + "grad_norm": 0.7901169657707214, + "learning_rate": 0.0001, + "loss": 68.1079, + "step": 9620 + }, + { + "epoch": 1.5550150380492926, + "grad_norm": 0.6970444917678833, + "learning_rate": 0.0001, + "loss": 69.3883, + "step": 9630 + }, + { + "epoch": 1.5566298621343937, + "grad_norm": 0.6954355239868164, + "learning_rate": 0.0001, + "loss": 69.3841, + "step": 9640 + }, + { + "epoch": 1.558244686219495, + "grad_norm": 0.7097641825675964, + "learning_rate": 0.0001, + "loss": 68.3417, + "step": 9650 + }, + { + "epoch": 1.5598595103045962, + "grad_norm": 0.7082354426383972, + "learning_rate": 0.0001, + "loss": 69.9314, + "step": 9660 + }, + { + "epoch": 1.5614743343896973, + "grad_norm": 0.6570103764533997, + "learning_rate": 0.0001, + "loss": 69.7219, + "step": 9670 + }, + { + "epoch": 1.5630891584747988, + "grad_norm": 0.5705522894859314, + "learning_rate": 0.0001, + "loss": 67.5177, + "step": 9680 + }, + { + "epoch": 1.5647039825598998, + "grad_norm": 0.6992695331573486, + "learning_rate": 0.0001, + "loss": 67.6538, + "step": 9690 + }, + { + "epoch": 1.5663188066450011, + "grad_norm": 0.7523599863052368, + "learning_rate": 0.0001, + "loss": 66.869, + "step": 9700 + }, + { + "epoch": 1.5679336307301024, + "grad_norm": 0.6267555356025696, + "learning_rate": 0.0001, + "loss": 66.6805, + "step": 9710 + }, + { + "epoch": 1.5695484548152034, + "grad_norm": 0.6138116717338562, + "learning_rate": 0.0001, + "loss": 71.3217, + "step": 9720 + }, + { + "epoch": 1.571163278900305, + "grad_norm": 0.7408527731895447, + "learning_rate": 0.0001, + "loss": 69.2759, + "step": 9730 + }, + { + "epoch": 1.572778102985406, + "grad_norm": 0.813761830329895, + "learning_rate": 0.0001, + "loss": 70.4959, + "step": 9740 + }, + { + "epoch": 1.5743929270705073, + "grad_norm": 1.0363599061965942, + "learning_rate": 0.0001, + "loss": 68.1996, + "step": 9750 + }, + { + "epoch": 1.5760077511556085, + "grad_norm": 0.7773903012275696, + "learning_rate": 0.0001, + "loss": 67.9354, + "step": 9760 + }, + { + "epoch": 1.5776225752407096, + "grad_norm": 0.6465738415718079, + "learning_rate": 0.0001, + "loss": 69.5247, + "step": 9770 + }, + { + "epoch": 1.579237399325811, + "grad_norm": 0.8433385491371155, + "learning_rate": 0.0001, + "loss": 66.9448, + "step": 9780 + }, + { + "epoch": 1.5808522234109121, + "grad_norm": 0.8651545643806458, + "learning_rate": 0.0001, + "loss": 68.6994, + "step": 9790 + }, + { + "epoch": 1.5824670474960134, + "grad_norm": 0.6940448880195618, + "learning_rate": 0.0001, + "loss": 67.622, + "step": 9800 + }, + { + "epoch": 1.5840818715811147, + "grad_norm": 0.622756838798523, + "learning_rate": 0.0001, + "loss": 69.0994, + "step": 9810 + }, + { + "epoch": 1.5856966956662157, + "grad_norm": 0.6119604110717773, + "learning_rate": 0.0001, + "loss": 67.7682, + "step": 9820 + }, + { + "epoch": 1.5873115197513172, + "grad_norm": 0.5729681849479675, + "learning_rate": 0.0001, + "loss": 67.3113, + "step": 9830 + }, + { + "epoch": 1.5889263438364183, + "grad_norm": 0.7143844366073608, + "learning_rate": 0.0001, + "loss": 68.8128, + "step": 9840 + }, + { + "epoch": 1.5905411679215196, + "grad_norm": 0.9814873337745667, + "learning_rate": 0.0001, + "loss": 69.3099, + "step": 9850 + }, + { + "epoch": 1.5921559920066208, + "grad_norm": 0.8044945001602173, + "learning_rate": 0.0001, + "loss": 69.3939, + "step": 9860 + }, + { + "epoch": 1.5937708160917219, + "grad_norm": 0.8494839668273926, + "learning_rate": 0.0001, + "loss": 69.3889, + "step": 9870 + }, + { + "epoch": 1.5953856401768234, + "grad_norm": 0.6468870639801025, + "learning_rate": 0.0001, + "loss": 67.2047, + "step": 9880 + }, + { + "epoch": 1.5970004642619244, + "grad_norm": 0.6270474791526794, + "learning_rate": 0.0001, + "loss": 69.6813, + "step": 9890 + }, + { + "epoch": 1.5986152883470257, + "grad_norm": 0.7102645039558411, + "learning_rate": 0.0001, + "loss": 69.1546, + "step": 9900 + }, + { + "epoch": 1.600230112432127, + "grad_norm": 0.5929123759269714, + "learning_rate": 0.0001, + "loss": 67.2187, + "step": 9910 + }, + { + "epoch": 1.601844936517228, + "grad_norm": 0.6466200947761536, + "learning_rate": 0.0001, + "loss": 68.6876, + "step": 9920 + }, + { + "epoch": 1.6034597606023295, + "grad_norm": 0.6441522836685181, + "learning_rate": 0.0001, + "loss": 68.7955, + "step": 9930 + }, + { + "epoch": 1.6050745846874306, + "grad_norm": 0.7264062166213989, + "learning_rate": 0.0001, + "loss": 67.8876, + "step": 9940 + }, + { + "epoch": 1.6066894087725319, + "grad_norm": 0.6973199248313904, + "learning_rate": 0.0001, + "loss": 69.0904, + "step": 9950 + }, + { + "epoch": 1.6083042328576331, + "grad_norm": 0.582053005695343, + "learning_rate": 0.0001, + "loss": 69.3876, + "step": 9960 + }, + { + "epoch": 1.6099190569427342, + "grad_norm": 0.6431038975715637, + "learning_rate": 0.0001, + "loss": 67.4232, + "step": 9970 + }, + { + "epoch": 1.6115338810278357, + "grad_norm": 0.6313247084617615, + "learning_rate": 0.0001, + "loss": 68.3759, + "step": 9980 + }, + { + "epoch": 1.6131487051129367, + "grad_norm": 1.2003700733184814, + "learning_rate": 0.0001, + "loss": 68.3413, + "step": 9990 + }, + { + "epoch": 1.614763529198038, + "grad_norm": 1.0262068510055542, + "learning_rate": 0.0001, + "loss": 66.7308, + "step": 10000 + }, + { + "epoch": 1.6163783532831393, + "grad_norm": 0.5975973010063171, + "learning_rate": 0.0001, + "loss": 68.6893, + "step": 10010 + }, + { + "epoch": 1.6179931773682403, + "grad_norm": 0.6045264601707458, + "learning_rate": 0.0001, + "loss": 69.5241, + "step": 10020 + }, + { + "epoch": 1.6196080014533418, + "grad_norm": 0.654019832611084, + "learning_rate": 0.0001, + "loss": 66.1208, + "step": 10030 + }, + { + "epoch": 1.6212228255384429, + "grad_norm": 0.533981204032898, + "learning_rate": 0.0001, + "loss": 68.2129, + "step": 10040 + }, + { + "epoch": 1.6228376496235442, + "grad_norm": 1.184734582901001, + "learning_rate": 0.0001, + "loss": 70.0547, + "step": 10050 + }, + { + "epoch": 1.6244524737086454, + "grad_norm": 0.5836975574493408, + "learning_rate": 0.0001, + "loss": 68.1098, + "step": 10060 + }, + { + "epoch": 1.6260672977937465, + "grad_norm": 0.5856032967567444, + "learning_rate": 0.0001, + "loss": 67.483, + "step": 10070 + }, + { + "epoch": 1.627682121878848, + "grad_norm": 0.6735382080078125, + "learning_rate": 0.0001, + "loss": 68.8628, + "step": 10080 + }, + { + "epoch": 1.629296945963949, + "grad_norm": 0.5753204226493835, + "learning_rate": 0.0001, + "loss": 66.2777, + "step": 10090 + }, + { + "epoch": 1.6309117700490503, + "grad_norm": 0.8327603340148926, + "learning_rate": 0.0001, + "loss": 67.9309, + "step": 10100 + }, + { + "epoch": 1.6325265941341516, + "grad_norm": 0.6873740553855896, + "learning_rate": 0.0001, + "loss": 68.7627, + "step": 10110 + }, + { + "epoch": 1.6341414182192526, + "grad_norm": 0.7218474745750427, + "learning_rate": 0.0001, + "loss": 69.3637, + "step": 10120 + }, + { + "epoch": 1.6357562423043541, + "grad_norm": 0.8393228650093079, + "learning_rate": 0.0001, + "loss": 66.7686, + "step": 10130 + }, + { + "epoch": 1.6373710663894552, + "grad_norm": 0.6433869004249573, + "learning_rate": 0.0001, + "loss": 68.0896, + "step": 10140 + }, + { + "epoch": 1.6389858904745565, + "grad_norm": 0.7419982552528381, + "learning_rate": 0.0001, + "loss": 67.9707, + "step": 10150 + }, + { + "epoch": 1.6406007145596577, + "grad_norm": 0.5884562730789185, + "learning_rate": 0.0001, + "loss": 68.0287, + "step": 10160 + }, + { + "epoch": 1.6422155386447588, + "grad_norm": 0.729557454586029, + "learning_rate": 0.0001, + "loss": 69.942, + "step": 10170 + }, + { + "epoch": 1.6438303627298603, + "grad_norm": 0.6026669144630432, + "learning_rate": 0.0001, + "loss": 66.8192, + "step": 10180 + }, + { + "epoch": 1.6454451868149613, + "grad_norm": 0.6921732425689697, + "learning_rate": 0.0001, + "loss": 66.7087, + "step": 10190 + }, + { + "epoch": 1.6470600109000626, + "grad_norm": 0.755172848701477, + "learning_rate": 0.0001, + "loss": 70.799, + "step": 10200 + }, + { + "epoch": 1.6486748349851639, + "grad_norm": 0.7318310737609863, + "learning_rate": 0.0001, + "loss": 67.6957, + "step": 10210 + }, + { + "epoch": 1.650289659070265, + "grad_norm": 0.6915041208267212, + "learning_rate": 0.0001, + "loss": 69.3388, + "step": 10220 + }, + { + "epoch": 1.6519044831553664, + "grad_norm": 0.7173444628715515, + "learning_rate": 0.0001, + "loss": 68.3612, + "step": 10230 + }, + { + "epoch": 1.6535193072404675, + "grad_norm": 0.7485487461090088, + "learning_rate": 0.0001, + "loss": 68.5366, + "step": 10240 + }, + { + "epoch": 1.6551341313255687, + "grad_norm": 0.5838190913200378, + "learning_rate": 0.0001, + "loss": 68.3841, + "step": 10250 + }, + { + "epoch": 1.65674895541067, + "grad_norm": 0.5908819437026978, + "learning_rate": 0.0001, + "loss": 67.8741, + "step": 10260 + }, + { + "epoch": 1.658363779495771, + "grad_norm": 0.7475612163543701, + "learning_rate": 0.0001, + "loss": 69.0636, + "step": 10270 + }, + { + "epoch": 1.6599786035808726, + "grad_norm": 0.5451382994651794, + "learning_rate": 0.0001, + "loss": 68.6695, + "step": 10280 + }, + { + "epoch": 1.6615934276659736, + "grad_norm": 0.6277157068252563, + "learning_rate": 0.0001, + "loss": 67.5432, + "step": 10290 + }, + { + "epoch": 1.663208251751075, + "grad_norm": 0.8144198060035706, + "learning_rate": 0.0001, + "loss": 66.7892, + "step": 10300 + }, + { + "epoch": 1.6648230758361762, + "grad_norm": 0.7400529980659485, + "learning_rate": 0.0001, + "loss": 66.2653, + "step": 10310 + }, + { + "epoch": 1.6664378999212772, + "grad_norm": 0.74524986743927, + "learning_rate": 0.0001, + "loss": 67.8252, + "step": 10320 + }, + { + "epoch": 1.6680527240063787, + "grad_norm": 0.5554162263870239, + "learning_rate": 0.0001, + "loss": 69.7704, + "step": 10330 + }, + { + "epoch": 1.6696675480914798, + "grad_norm": 0.8194338083267212, + "learning_rate": 0.0001, + "loss": 69.4668, + "step": 10340 + }, + { + "epoch": 1.671282372176581, + "grad_norm": 0.7947783470153809, + "learning_rate": 0.0001, + "loss": 64.9276, + "step": 10350 + }, + { + "epoch": 1.6728971962616823, + "grad_norm": 0.6145230531692505, + "learning_rate": 0.0001, + "loss": 68.1815, + "step": 10360 + }, + { + "epoch": 1.6745120203467834, + "grad_norm": 0.6293516755104065, + "learning_rate": 0.0001, + "loss": 67.8994, + "step": 10370 + }, + { + "epoch": 1.6761268444318849, + "grad_norm": 0.6160871386528015, + "learning_rate": 0.0001, + "loss": 66.7463, + "step": 10380 + }, + { + "epoch": 1.677741668516986, + "grad_norm": 0.6925204992294312, + "learning_rate": 0.0001, + "loss": 68.5261, + "step": 10390 + }, + { + "epoch": 1.6793564926020872, + "grad_norm": 0.6892788410186768, + "learning_rate": 0.0001, + "loss": 66.8928, + "step": 10400 + }, + { + "epoch": 1.6809713166871885, + "grad_norm": 0.9066945910453796, + "learning_rate": 0.0001, + "loss": 66.2556, + "step": 10410 + }, + { + "epoch": 1.6825861407722895, + "grad_norm": 0.5857477784156799, + "learning_rate": 0.0001, + "loss": 67.1866, + "step": 10420 + }, + { + "epoch": 1.684200964857391, + "grad_norm": 0.7989841103553772, + "learning_rate": 0.0001, + "loss": 67.7762, + "step": 10430 + }, + { + "epoch": 1.685815788942492, + "grad_norm": 0.559428870677948, + "learning_rate": 0.0001, + "loss": 67.5628, + "step": 10440 + }, + { + "epoch": 1.6874306130275933, + "grad_norm": 0.6718077659606934, + "learning_rate": 0.0001, + "loss": 69.2877, + "step": 10450 + }, + { + "epoch": 1.6890454371126946, + "grad_norm": 0.6952136754989624, + "learning_rate": 0.0001, + "loss": 67.2734, + "step": 10460 + }, + { + "epoch": 1.6906602611977957, + "grad_norm": 0.6079897880554199, + "learning_rate": 0.0001, + "loss": 67.4677, + "step": 10470 + }, + { + "epoch": 1.692275085282897, + "grad_norm": 0.6139684915542603, + "learning_rate": 0.0001, + "loss": 69.2464, + "step": 10480 + }, + { + "epoch": 1.6938899093679982, + "grad_norm": 0.7009351253509521, + "learning_rate": 0.0001, + "loss": 67.5424, + "step": 10490 + }, + { + "epoch": 1.6955047334530995, + "grad_norm": 0.8567383885383606, + "learning_rate": 0.0001, + "loss": 65.6573, + "step": 10500 + }, + { + "epoch": 1.6971195575382008, + "grad_norm": 0.6903256177902222, + "learning_rate": 0.0001, + "loss": 69.2218, + "step": 10510 + }, + { + "epoch": 1.6987343816233018, + "grad_norm": 0.5958806872367859, + "learning_rate": 0.0001, + "loss": 67.3718, + "step": 10520 + }, + { + "epoch": 1.700349205708403, + "grad_norm": 0.5983722805976868, + "learning_rate": 0.0001, + "loss": 68.7944, + "step": 10530 + }, + { + "epoch": 1.7019640297935044, + "grad_norm": 0.8119397759437561, + "learning_rate": 0.0001, + "loss": 66.7936, + "step": 10540 + }, + { + "epoch": 1.7035788538786056, + "grad_norm": 0.6383484601974487, + "learning_rate": 0.0001, + "loss": 67.8636, + "step": 10550 + }, + { + "epoch": 1.705193677963707, + "grad_norm": 0.6574084162712097, + "learning_rate": 0.0001, + "loss": 67.26, + "step": 10560 + }, + { + "epoch": 1.706808502048808, + "grad_norm": 0.6716108918190002, + "learning_rate": 0.0001, + "loss": 66.485, + "step": 10570 + }, + { + "epoch": 1.7084233261339092, + "grad_norm": 0.6998292803764343, + "learning_rate": 0.0001, + "loss": 66.4201, + "step": 10580 + }, + { + "epoch": 1.7100381502190105, + "grad_norm": 0.7300788760185242, + "learning_rate": 0.0001, + "loss": 67.1446, + "step": 10590 + }, + { + "epoch": 1.7116529743041118, + "grad_norm": 0.7066853642463684, + "learning_rate": 0.0001, + "loss": 70.8954, + "step": 10600 + }, + { + "epoch": 1.713267798389213, + "grad_norm": 0.6933701038360596, + "learning_rate": 0.0001, + "loss": 67.8473, + "step": 10610 + }, + { + "epoch": 1.7148826224743141, + "grad_norm": 0.6885779500007629, + "learning_rate": 0.0001, + "loss": 65.3276, + "step": 10620 + }, + { + "epoch": 1.7164974465594154, + "grad_norm": 0.6542202830314636, + "learning_rate": 0.0001, + "loss": 69.2596, + "step": 10630 + }, + { + "epoch": 1.7181122706445167, + "grad_norm": 0.5719691514968872, + "learning_rate": 0.0001, + "loss": 69.1907, + "step": 10640 + }, + { + "epoch": 1.719727094729618, + "grad_norm": 0.6596891283988953, + "learning_rate": 0.0001, + "loss": 69.0778, + "step": 10650 + }, + { + "epoch": 1.7213419188147192, + "grad_norm": 0.6860799193382263, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 10660 + }, + { + "epoch": 1.7229567428998203, + "grad_norm": 0.7912564277648926, + "learning_rate": 0.0001, + "loss": 68.2091, + "step": 10670 + }, + { + "epoch": 1.7245715669849215, + "grad_norm": 0.6673903465270996, + "learning_rate": 0.0001, + "loss": 68.24, + "step": 10680 + }, + { + "epoch": 1.7261863910700228, + "grad_norm": 0.6243769526481628, + "learning_rate": 0.0001, + "loss": 68.6396, + "step": 10690 + }, + { + "epoch": 1.727801215155124, + "grad_norm": 0.6622587442398071, + "learning_rate": 0.0001, + "loss": 67.9406, + "step": 10700 + }, + { + "epoch": 1.7294160392402254, + "grad_norm": 0.6808138489723206, + "learning_rate": 0.0001, + "loss": 69.1403, + "step": 10710 + }, + { + "epoch": 1.7310308633253264, + "grad_norm": 0.8498415350914001, + "learning_rate": 0.0001, + "loss": 70.1438, + "step": 10720 + }, + { + "epoch": 1.7326456874104277, + "grad_norm": 1.0091742277145386, + "learning_rate": 0.0001, + "loss": 67.0196, + "step": 10730 + }, + { + "epoch": 1.734260511495529, + "grad_norm": 0.6997767686843872, + "learning_rate": 0.0001, + "loss": 67.0307, + "step": 10740 + }, + { + "epoch": 1.7358753355806302, + "grad_norm": 0.6618793606758118, + "learning_rate": 0.0001, + "loss": 67.906, + "step": 10750 + }, + { + "epoch": 1.7374901596657315, + "grad_norm": 0.7082014679908752, + "learning_rate": 0.0001, + "loss": 68.0665, + "step": 10760 + }, + { + "epoch": 1.7391049837508326, + "grad_norm": 0.8494391441345215, + "learning_rate": 0.0001, + "loss": 68.4202, + "step": 10770 + }, + { + "epoch": 1.7407198078359338, + "grad_norm": 0.6247998476028442, + "learning_rate": 0.0001, + "loss": 66.4399, + "step": 10780 + }, + { + "epoch": 1.742334631921035, + "grad_norm": 0.8239936828613281, + "learning_rate": 0.0001, + "loss": 68.4374, + "step": 10790 + }, + { + "epoch": 1.7439494560061364, + "grad_norm": 0.6734293103218079, + "learning_rate": 0.0001, + "loss": 66.3411, + "step": 10800 + }, + { + "epoch": 1.7455642800912377, + "grad_norm": 0.7066450119018555, + "learning_rate": 0.0001, + "loss": 67.7102, + "step": 10810 + }, + { + "epoch": 1.7471791041763387, + "grad_norm": 0.7440647482872009, + "learning_rate": 0.0001, + "loss": 67.7221, + "step": 10820 + }, + { + "epoch": 1.74879392826144, + "grad_norm": 0.7771329283714294, + "learning_rate": 0.0001, + "loss": 67.5907, + "step": 10830 + }, + { + "epoch": 1.7504087523465413, + "grad_norm": 1.2484230995178223, + "learning_rate": 0.0001, + "loss": 66.3242, + "step": 10840 + }, + { + "epoch": 1.7520235764316425, + "grad_norm": 0.6946772933006287, + "learning_rate": 0.0001, + "loss": 67.8741, + "step": 10850 + }, + { + "epoch": 1.7536384005167438, + "grad_norm": 0.7676035761833191, + "learning_rate": 0.0001, + "loss": 67.3776, + "step": 10860 + }, + { + "epoch": 1.7552532246018449, + "grad_norm": 0.5534589290618896, + "learning_rate": 0.0001, + "loss": 67.6656, + "step": 10870 + }, + { + "epoch": 1.7568680486869461, + "grad_norm": 1.2745810747146606, + "learning_rate": 0.0001, + "loss": 68.5959, + "step": 10880 + }, + { + "epoch": 1.7584828727720474, + "grad_norm": 0.6241582036018372, + "learning_rate": 0.0001, + "loss": 67.7142, + "step": 10890 + }, + { + "epoch": 1.7600976968571487, + "grad_norm": 0.9308313131332397, + "learning_rate": 0.0001, + "loss": 67.0322, + "step": 10900 + }, + { + "epoch": 1.76171252094225, + "grad_norm": 0.5639570951461792, + "learning_rate": 0.0001, + "loss": 67.928, + "step": 10910 + }, + { + "epoch": 1.763327345027351, + "grad_norm": 0.6692782044410706, + "learning_rate": 0.0001, + "loss": 67.625, + "step": 10920 + }, + { + "epoch": 1.7649421691124523, + "grad_norm": 0.7322304248809814, + "learning_rate": 0.0001, + "loss": 66.2372, + "step": 10930 + }, + { + "epoch": 1.7665569931975535, + "grad_norm": 0.6275238394737244, + "learning_rate": 0.0001, + "loss": 67.7628, + "step": 10940 + }, + { + "epoch": 1.7681718172826548, + "grad_norm": 0.6832380890846252, + "learning_rate": 0.0001, + "loss": 67.6587, + "step": 10950 + }, + { + "epoch": 1.769786641367756, + "grad_norm": 0.6934792399406433, + "learning_rate": 0.0001, + "loss": 67.1411, + "step": 10960 + }, + { + "epoch": 1.7714014654528571, + "grad_norm": 0.7031943202018738, + "learning_rate": 0.0001, + "loss": 69.2734, + "step": 10970 + }, + { + "epoch": 1.7730162895379584, + "grad_norm": 0.91269451379776, + "learning_rate": 0.0001, + "loss": 68.1353, + "step": 10980 + }, + { + "epoch": 1.7746311136230597, + "grad_norm": 0.8927266001701355, + "learning_rate": 0.0001, + "loss": 68.003, + "step": 10990 + }, + { + "epoch": 1.776245937708161, + "grad_norm": 0.9115116596221924, + "learning_rate": 0.0001, + "loss": 69.1506, + "step": 11000 + }, + { + "epoch": 1.7778607617932622, + "grad_norm": 0.6055391430854797, + "learning_rate": 0.0001, + "loss": 68.9785, + "step": 11010 + }, + { + "epoch": 1.7794755858783633, + "grad_norm": 0.872809112071991, + "learning_rate": 0.0001, + "loss": 66.373, + "step": 11020 + }, + { + "epoch": 1.7810904099634646, + "grad_norm": 0.5838716626167297, + "learning_rate": 0.0001, + "loss": 69.507, + "step": 11030 + }, + { + "epoch": 1.7827052340485658, + "grad_norm": 0.6108883619308472, + "learning_rate": 0.0001, + "loss": 67.3743, + "step": 11040 + }, + { + "epoch": 1.784320058133667, + "grad_norm": 0.5457114577293396, + "learning_rate": 0.0001, + "loss": 66.1342, + "step": 11050 + }, + { + "epoch": 1.7859348822187684, + "grad_norm": 0.7696211338043213, + "learning_rate": 0.0001, + "loss": 67.9779, + "step": 11060 + }, + { + "epoch": 1.7875497063038694, + "grad_norm": 0.5795412659645081, + "learning_rate": 0.0001, + "loss": 67.1763, + "step": 11070 + }, + { + "epoch": 1.7891645303889707, + "grad_norm": 0.7401878833770752, + "learning_rate": 0.0001, + "loss": 67.4972, + "step": 11080 + }, + { + "epoch": 1.790779354474072, + "grad_norm": 0.666051983833313, + "learning_rate": 0.0001, + "loss": 68.7621, + "step": 11090 + }, + { + "epoch": 1.792394178559173, + "grad_norm": 0.9650185108184814, + "learning_rate": 0.0001, + "loss": 66.7721, + "step": 11100 + }, + { + "epoch": 1.7940090026442745, + "grad_norm": 0.6026908159255981, + "learning_rate": 0.0001, + "loss": 66.2993, + "step": 11110 + }, + { + "epoch": 1.7956238267293756, + "grad_norm": 0.6560576558113098, + "learning_rate": 0.0001, + "loss": 67.8543, + "step": 11120 + }, + { + "epoch": 1.7972386508144769, + "grad_norm": 0.9523794054985046, + "learning_rate": 0.0001, + "loss": 66.7881, + "step": 11130 + }, + { + "epoch": 1.7988534748995781, + "grad_norm": 0.7241806983947754, + "learning_rate": 0.0001, + "loss": 68.1071, + "step": 11140 + }, + { + "epoch": 1.8004682989846792, + "grad_norm": 0.639866292476654, + "learning_rate": 0.0001, + "loss": 68.992, + "step": 11150 + }, + { + "epoch": 1.8020831230697807, + "grad_norm": 0.6050712466239929, + "learning_rate": 0.0001, + "loss": 67.7993, + "step": 11160 + }, + { + "epoch": 1.8036979471548817, + "grad_norm": 0.5777723789215088, + "learning_rate": 0.0001, + "loss": 66.9515, + "step": 11170 + }, + { + "epoch": 1.805312771239983, + "grad_norm": 0.6153426766395569, + "learning_rate": 0.0001, + "loss": 68.3413, + "step": 11180 + }, + { + "epoch": 1.8069275953250843, + "grad_norm": 0.6118552088737488, + "learning_rate": 0.0001, + "loss": 67.7853, + "step": 11190 + }, + { + "epoch": 1.8085424194101853, + "grad_norm": 0.6520341038703918, + "learning_rate": 0.0001, + "loss": 67.3511, + "step": 11200 + }, + { + "epoch": 1.8101572434952868, + "grad_norm": 0.6189568638801575, + "learning_rate": 0.0001, + "loss": 69.28, + "step": 11210 + }, + { + "epoch": 1.811772067580388, + "grad_norm": 0.602411150932312, + "learning_rate": 0.0001, + "loss": 67.5133, + "step": 11220 + }, + { + "epoch": 1.8133868916654892, + "grad_norm": 0.7359414100646973, + "learning_rate": 0.0001, + "loss": 68.631, + "step": 11230 + }, + { + "epoch": 1.8150017157505904, + "grad_norm": 0.6216995120048523, + "learning_rate": 0.0001, + "loss": 68.3588, + "step": 11240 + }, + { + "epoch": 1.8166165398356915, + "grad_norm": 0.7683486938476562, + "learning_rate": 0.0001, + "loss": 67.1434, + "step": 11250 + }, + { + "epoch": 1.818231363920793, + "grad_norm": 0.5565283894538879, + "learning_rate": 0.0001, + "loss": 66.9995, + "step": 11260 + }, + { + "epoch": 1.819846188005894, + "grad_norm": 0.5623223781585693, + "learning_rate": 0.0001, + "loss": 68.1316, + "step": 11270 + }, + { + "epoch": 1.8214610120909953, + "grad_norm": 0.532821774482727, + "learning_rate": 0.0001, + "loss": 69.1903, + "step": 11280 + }, + { + "epoch": 1.8230758361760966, + "grad_norm": 0.6297196745872498, + "learning_rate": 0.0001, + "loss": 68.4965, + "step": 11290 + }, + { + "epoch": 1.8246906602611976, + "grad_norm": 0.839862048625946, + "learning_rate": 0.0001, + "loss": 67.1782, + "step": 11300 + }, + { + "epoch": 1.8263054843462991, + "grad_norm": 0.7021651864051819, + "learning_rate": 0.0001, + "loss": 67.4355, + "step": 11310 + }, + { + "epoch": 1.8279203084314002, + "grad_norm": 0.676034152507782, + "learning_rate": 0.0001, + "loss": 68.3721, + "step": 11320 + }, + { + "epoch": 1.8295351325165015, + "grad_norm": 0.6621078252792358, + "learning_rate": 0.0001, + "loss": 67.3255, + "step": 11330 + }, + { + "epoch": 1.8311499566016027, + "grad_norm": 0.5902897119522095, + "learning_rate": 0.0001, + "loss": 68.0641, + "step": 11340 + }, + { + "epoch": 1.8327647806867038, + "grad_norm": 0.6334371566772461, + "learning_rate": 0.0001, + "loss": 68.2682, + "step": 11350 + }, + { + "epoch": 1.8343796047718053, + "grad_norm": 0.6753348708152771, + "learning_rate": 0.0001, + "loss": 66.6749, + "step": 11360 + }, + { + "epoch": 1.8359944288569063, + "grad_norm": 0.7221750617027283, + "learning_rate": 0.0001, + "loss": 68.4142, + "step": 11370 + }, + { + "epoch": 1.8376092529420076, + "grad_norm": 0.7476344108581543, + "learning_rate": 0.0001, + "loss": 67.1416, + "step": 11380 + }, + { + "epoch": 1.8392240770271089, + "grad_norm": 0.5927479863166809, + "learning_rate": 0.0001, + "loss": 67.6368, + "step": 11390 + }, + { + "epoch": 1.84083890111221, + "grad_norm": 0.6753717660903931, + "learning_rate": 0.0001, + "loss": 67.1026, + "step": 11400 + }, + { + "epoch": 1.8424537251973114, + "grad_norm": 0.6500268578529358, + "learning_rate": 0.0001, + "loss": 68.7537, + "step": 11410 + }, + { + "epoch": 1.8440685492824125, + "grad_norm": 0.5930709838867188, + "learning_rate": 0.0001, + "loss": 65.7429, + "step": 11420 + }, + { + "epoch": 1.8456833733675138, + "grad_norm": 0.6467777490615845, + "learning_rate": 0.0001, + "loss": 68.1333, + "step": 11430 + }, + { + "epoch": 1.847298197452615, + "grad_norm": 0.6935669183731079, + "learning_rate": 0.0001, + "loss": 66.7728, + "step": 11440 + }, + { + "epoch": 1.848913021537716, + "grad_norm": 0.5838116407394409, + "learning_rate": 0.0001, + "loss": 66.6005, + "step": 11450 + }, + { + "epoch": 1.8505278456228176, + "grad_norm": 0.8621205687522888, + "learning_rate": 0.0001, + "loss": 66.0859, + "step": 11460 + }, + { + "epoch": 1.8521426697079186, + "grad_norm": 0.7119213342666626, + "learning_rate": 0.0001, + "loss": 66.6198, + "step": 11470 + }, + { + "epoch": 1.85375749379302, + "grad_norm": 0.6861911416053772, + "learning_rate": 0.0001, + "loss": 67.2275, + "step": 11480 + }, + { + "epoch": 1.8553723178781212, + "grad_norm": 0.5653937458992004, + "learning_rate": 0.0001, + "loss": 67.827, + "step": 11490 + }, + { + "epoch": 1.8569871419632222, + "grad_norm": 0.6458482146263123, + "learning_rate": 0.0001, + "loss": 66.3906, + "step": 11500 + }, + { + "epoch": 1.8586019660483237, + "grad_norm": 0.6701959371566772, + "learning_rate": 0.0001, + "loss": 67.88, + "step": 11510 + }, + { + "epoch": 1.8602167901334248, + "grad_norm": 0.6598711609840393, + "learning_rate": 0.0001, + "loss": 66.8373, + "step": 11520 + }, + { + "epoch": 1.861831614218526, + "grad_norm": 0.6095236539840698, + "learning_rate": 0.0001, + "loss": 67.6478, + "step": 11530 + }, + { + "epoch": 1.8634464383036273, + "grad_norm": 0.8192022442817688, + "learning_rate": 0.0001, + "loss": 67.7343, + "step": 11540 + }, + { + "epoch": 1.8650612623887284, + "grad_norm": 0.5788753628730774, + "learning_rate": 0.0001, + "loss": 66.7909, + "step": 11550 + }, + { + "epoch": 1.8666760864738299, + "grad_norm": 0.5383989214897156, + "learning_rate": 0.0001, + "loss": 68.5349, + "step": 11560 + }, + { + "epoch": 1.868290910558931, + "grad_norm": 0.6224480271339417, + "learning_rate": 0.0001, + "loss": 68.1938, + "step": 11570 + }, + { + "epoch": 1.8699057346440322, + "grad_norm": 0.5852788686752319, + "learning_rate": 0.0001, + "loss": 66.4678, + "step": 11580 + }, + { + "epoch": 1.8715205587291335, + "grad_norm": 0.730378270149231, + "learning_rate": 0.0001, + "loss": 67.57, + "step": 11590 + }, + { + "epoch": 1.8731353828142345, + "grad_norm": 0.6197090744972229, + "learning_rate": 0.0001, + "loss": 65.7801, + "step": 11600 + }, + { + "epoch": 1.874750206899336, + "grad_norm": 0.518974781036377, + "learning_rate": 0.0001, + "loss": 68.9308, + "step": 11610 + }, + { + "epoch": 1.876365030984437, + "grad_norm": 0.6601742506027222, + "learning_rate": 0.0001, + "loss": 66.821, + "step": 11620 + }, + { + "epoch": 1.8779798550695384, + "grad_norm": 0.7409974932670593, + "learning_rate": 0.0001, + "loss": 66.6883, + "step": 11630 + }, + { + "epoch": 1.8795946791546396, + "grad_norm": 0.6372189521789551, + "learning_rate": 0.0001, + "loss": 67.2109, + "step": 11640 + }, + { + "epoch": 1.8812095032397407, + "grad_norm": 0.6107574701309204, + "learning_rate": 0.0001, + "loss": 67.029, + "step": 11650 + }, + { + "epoch": 1.8828243273248422, + "grad_norm": 0.6543708443641663, + "learning_rate": 0.0001, + "loss": 64.2133, + "step": 11660 + }, + { + "epoch": 1.8844391514099432, + "grad_norm": 0.5161535739898682, + "learning_rate": 0.0001, + "loss": 66.7091, + "step": 11670 + }, + { + "epoch": 1.8860539754950445, + "grad_norm": 0.6089748740196228, + "learning_rate": 0.0001, + "loss": 69.2073, + "step": 11680 + }, + { + "epoch": 1.8876687995801458, + "grad_norm": 0.6930866837501526, + "learning_rate": 0.0001, + "loss": 68.7227, + "step": 11690 + }, + { + "epoch": 1.8892836236652468, + "grad_norm": 0.6563196182250977, + "learning_rate": 0.0001, + "loss": 68.27, + "step": 11700 + }, + { + "epoch": 1.8908984477503483, + "grad_norm": 0.6173767447471619, + "learning_rate": 0.0001, + "loss": 68.5535, + "step": 11710 + }, + { + "epoch": 1.8925132718354494, + "grad_norm": 0.5498561263084412, + "learning_rate": 0.0001, + "loss": 67.7681, + "step": 11720 + }, + { + "epoch": 1.8941280959205506, + "grad_norm": 0.584891676902771, + "learning_rate": 0.0001, + "loss": 68.1709, + "step": 11730 + }, + { + "epoch": 1.895742920005652, + "grad_norm": 0.7491183280944824, + "learning_rate": 0.0001, + "loss": 65.7435, + "step": 11740 + }, + { + "epoch": 1.897357744090753, + "grad_norm": 0.6519315242767334, + "learning_rate": 0.0001, + "loss": 67.5436, + "step": 11750 + }, + { + "epoch": 1.8989725681758545, + "grad_norm": 0.5740950107574463, + "learning_rate": 0.0001, + "loss": 66.3092, + "step": 11760 + }, + { + "epoch": 1.9005873922609555, + "grad_norm": 0.7978914380073547, + "learning_rate": 0.0001, + "loss": 66.8598, + "step": 11770 + }, + { + "epoch": 1.9022022163460568, + "grad_norm": 0.5536732077598572, + "learning_rate": 0.0001, + "loss": 68.1609, + "step": 11780 + }, + { + "epoch": 1.903817040431158, + "grad_norm": 0.6358754634857178, + "learning_rate": 0.0001, + "loss": 68.2948, + "step": 11790 + }, + { + "epoch": 1.9054318645162591, + "grad_norm": 0.8400535583496094, + "learning_rate": 0.0001, + "loss": 67.7722, + "step": 11800 + }, + { + "epoch": 1.9070466886013606, + "grad_norm": 0.8256928324699402, + "learning_rate": 0.0001, + "loss": 65.8444, + "step": 11810 + }, + { + "epoch": 1.9086615126864617, + "grad_norm": 0.9160122871398926, + "learning_rate": 0.0001, + "loss": 67.4363, + "step": 11820 + }, + { + "epoch": 1.910276336771563, + "grad_norm": 0.6507587432861328, + "learning_rate": 0.0001, + "loss": 65.9515, + "step": 11830 + }, + { + "epoch": 1.9118911608566642, + "grad_norm": 0.5940394401550293, + "learning_rate": 0.0001, + "loss": 65.8083, + "step": 11840 + }, + { + "epoch": 1.9135059849417653, + "grad_norm": 0.7403995990753174, + "learning_rate": 0.0001, + "loss": 66.7136, + "step": 11850 + }, + { + "epoch": 1.9151208090268668, + "grad_norm": 0.6039106845855713, + "learning_rate": 0.0001, + "loss": 67.6913, + "step": 11860 + }, + { + "epoch": 1.9167356331119678, + "grad_norm": 0.9714781641960144, + "learning_rate": 0.0001, + "loss": 67.9658, + "step": 11870 + }, + { + "epoch": 1.918350457197069, + "grad_norm": 0.6880549192428589, + "learning_rate": 0.0001, + "loss": 68.4463, + "step": 11880 + }, + { + "epoch": 1.9199652812821704, + "grad_norm": 0.5718733668327332, + "learning_rate": 0.0001, + "loss": 66.9625, + "step": 11890 + }, + { + "epoch": 1.9215801053672714, + "grad_norm": 0.6208164691925049, + "learning_rate": 0.0001, + "loss": 65.9462, + "step": 11900 + }, + { + "epoch": 1.923194929452373, + "grad_norm": 0.5957098007202148, + "learning_rate": 0.0001, + "loss": 66.9398, + "step": 11910 + }, + { + "epoch": 1.924809753537474, + "grad_norm": 0.6295427680015564, + "learning_rate": 0.0001, + "loss": 66.2246, + "step": 11920 + }, + { + "epoch": 1.9264245776225752, + "grad_norm": 0.8903796672821045, + "learning_rate": 0.0001, + "loss": 66.2999, + "step": 11930 + }, + { + "epoch": 1.9280394017076765, + "grad_norm": 0.6410874128341675, + "learning_rate": 0.0001, + "loss": 67.6857, + "step": 11940 + }, + { + "epoch": 1.9296542257927776, + "grad_norm": 0.5800307393074036, + "learning_rate": 0.0001, + "loss": 66.0943, + "step": 11950 + }, + { + "epoch": 1.931269049877879, + "grad_norm": 0.6834051609039307, + "learning_rate": 0.0001, + "loss": 67.6365, + "step": 11960 + }, + { + "epoch": 1.9328838739629801, + "grad_norm": 0.6112056970596313, + "learning_rate": 0.0001, + "loss": 66.8131, + "step": 11970 + }, + { + "epoch": 1.9344986980480814, + "grad_norm": 0.6657601594924927, + "learning_rate": 0.0001, + "loss": 65.7584, + "step": 11980 + }, + { + "epoch": 1.9361135221331827, + "grad_norm": 0.7570573091506958, + "learning_rate": 0.0001, + "loss": 68.6683, + "step": 11990 + }, + { + "epoch": 1.9377283462182837, + "grad_norm": 0.6231463551521301, + "learning_rate": 0.0001, + "loss": 67.71, + "step": 12000 + }, + { + "epoch": 1.9393431703033852, + "grad_norm": 0.5517798662185669, + "learning_rate": 0.0001, + "loss": 65.5666, + "step": 12010 + }, + { + "epoch": 1.9409579943884863, + "grad_norm": 0.6614659428596497, + "learning_rate": 0.0001, + "loss": 65.4867, + "step": 12020 + }, + { + "epoch": 1.9425728184735875, + "grad_norm": 1.027906060218811, + "learning_rate": 0.0001, + "loss": 65.2863, + "step": 12030 + }, + { + "epoch": 1.9441876425586888, + "grad_norm": 0.5714905261993408, + "learning_rate": 0.0001, + "loss": 67.52, + "step": 12040 + }, + { + "epoch": 1.9458024666437899, + "grad_norm": 0.6580681204795837, + "learning_rate": 0.0001, + "loss": 68.3794, + "step": 12050 + }, + { + "epoch": 1.9474172907288914, + "grad_norm": 0.5728104710578918, + "learning_rate": 0.0001, + "loss": 65.6871, + "step": 12060 + }, + { + "epoch": 1.9490321148139924, + "grad_norm": 0.6828856468200684, + "learning_rate": 0.0001, + "loss": 66.6735, + "step": 12070 + }, + { + "epoch": 1.9506469388990937, + "grad_norm": 0.7448258996009827, + "learning_rate": 0.0001, + "loss": 67.4387, + "step": 12080 + }, + { + "epoch": 1.952261762984195, + "grad_norm": 0.5058867931365967, + "learning_rate": 0.0001, + "loss": 67.3999, + "step": 12090 + }, + { + "epoch": 1.953876587069296, + "grad_norm": 0.5884380340576172, + "learning_rate": 0.0001, + "loss": 66.6189, + "step": 12100 + }, + { + "epoch": 1.9554914111543975, + "grad_norm": 0.7010582685470581, + "learning_rate": 0.0001, + "loss": 66.4048, + "step": 12110 + }, + { + "epoch": 1.9571062352394986, + "grad_norm": 0.556446373462677, + "learning_rate": 0.0001, + "loss": 67.7398, + "step": 12120 + }, + { + "epoch": 1.9587210593245998, + "grad_norm": 0.7856518626213074, + "learning_rate": 0.0001, + "loss": 64.4189, + "step": 12130 + }, + { + "epoch": 1.960335883409701, + "grad_norm": 0.573811948299408, + "learning_rate": 0.0001, + "loss": 66.081, + "step": 12140 + }, + { + "epoch": 1.9619507074948022, + "grad_norm": 0.6113094687461853, + "learning_rate": 0.0001, + "loss": 65.6262, + "step": 12150 + }, + { + "epoch": 1.9635655315799037, + "grad_norm": 0.7158921360969543, + "learning_rate": 0.0001, + "loss": 66.646, + "step": 12160 + }, + { + "epoch": 1.9651803556650047, + "grad_norm": 0.6789208054542542, + "learning_rate": 0.0001, + "loss": 66.9576, + "step": 12170 + }, + { + "epoch": 1.966795179750106, + "grad_norm": 0.6550324559211731, + "learning_rate": 0.0001, + "loss": 67.2544, + "step": 12180 + }, + { + "epoch": 1.9684100038352073, + "grad_norm": 0.6580352187156677, + "learning_rate": 0.0001, + "loss": 67.6755, + "step": 12190 + }, + { + "epoch": 1.9700248279203083, + "grad_norm": 0.7927120327949524, + "learning_rate": 0.0001, + "loss": 66.9819, + "step": 12200 + }, + { + "epoch": 1.9716396520054098, + "grad_norm": 0.5637071132659912, + "learning_rate": 0.0001, + "loss": 67.9308, + "step": 12210 + }, + { + "epoch": 1.9732544760905109, + "grad_norm": 0.5903081297874451, + "learning_rate": 0.0001, + "loss": 67.0279, + "step": 12220 + }, + { + "epoch": 1.9748693001756121, + "grad_norm": 0.7173757553100586, + "learning_rate": 0.0001, + "loss": 66.1328, + "step": 12230 + }, + { + "epoch": 1.9764841242607134, + "grad_norm": 0.7677658796310425, + "learning_rate": 0.0001, + "loss": 66.8729, + "step": 12240 + }, + { + "epoch": 1.9780989483458145, + "grad_norm": 0.5733856558799744, + "learning_rate": 0.0001, + "loss": 65.7204, + "step": 12250 + }, + { + "epoch": 1.979713772430916, + "grad_norm": 0.575779378414154, + "learning_rate": 0.0001, + "loss": 65.082, + "step": 12260 + }, + { + "epoch": 1.981328596516017, + "grad_norm": 0.7983589768409729, + "learning_rate": 0.0001, + "loss": 66.7954, + "step": 12270 + }, + { + "epoch": 1.9829434206011183, + "grad_norm": 0.6657546758651733, + "learning_rate": 0.0001, + "loss": 66.1977, + "step": 12280 + }, + { + "epoch": 1.9845582446862196, + "grad_norm": 0.723057746887207, + "learning_rate": 0.0001, + "loss": 65.2823, + "step": 12290 + }, + { + "epoch": 1.9861730687713206, + "grad_norm": 0.7241561412811279, + "learning_rate": 0.0001, + "loss": 66.5179, + "step": 12300 + }, + { + "epoch": 1.987787892856422, + "grad_norm": 0.7907382249832153, + "learning_rate": 0.0001, + "loss": 67.1835, + "step": 12310 + }, + { + "epoch": 1.9894027169415232, + "grad_norm": 1.0993635654449463, + "learning_rate": 0.0001, + "loss": 66.2844, + "step": 12320 + }, + { + "epoch": 1.9910175410266244, + "grad_norm": 0.6327719688415527, + "learning_rate": 0.0001, + "loss": 64.3111, + "step": 12330 + }, + { + "epoch": 1.9926323651117257, + "grad_norm": 0.5955413579940796, + "learning_rate": 0.0001, + "loss": 67.3991, + "step": 12340 + }, + { + "epoch": 1.9942471891968268, + "grad_norm": 0.7130835056304932, + "learning_rate": 0.0001, + "loss": 67.2216, + "step": 12350 + }, + { + "epoch": 1.9958620132819282, + "grad_norm": 0.5579924583435059, + "learning_rate": 0.0001, + "loss": 65.8894, + "step": 12360 + }, + { + "epoch": 1.9974768373670293, + "grad_norm": 0.7395238876342773, + "learning_rate": 0.0001, + "loss": 66.5182, + "step": 12370 + }, + { + "epoch": 1.9990916614521306, + "grad_norm": 0.6070617437362671, + "learning_rate": 0.0001, + "loss": 67.0429, + "step": 12380 + }, + { + "epoch": 2.0006459296340404, + "grad_norm": 0.6658774018287659, + "learning_rate": 0.0001, + "loss": 65.1403, + "step": 12390 + }, + { + "epoch": 2.002260753719142, + "grad_norm": 0.7703005075454712, + "learning_rate": 0.0001, + "loss": 66.0792, + "step": 12400 + }, + { + "epoch": 2.003875577804243, + "grad_norm": 0.6887365579605103, + "learning_rate": 0.0001, + "loss": 67.5743, + "step": 12410 + }, + { + "epoch": 2.005490401889344, + "grad_norm": 0.6181573271751404, + "learning_rate": 0.0001, + "loss": 65.7725, + "step": 12420 + }, + { + "epoch": 2.0071052259744455, + "grad_norm": 0.6926079988479614, + "learning_rate": 0.0001, + "loss": 66.3818, + "step": 12430 + }, + { + "epoch": 2.0087200500595466, + "grad_norm": 0.9466767311096191, + "learning_rate": 0.0001, + "loss": 66.8674, + "step": 12440 + }, + { + "epoch": 2.010334874144648, + "grad_norm": 0.693327784538269, + "learning_rate": 0.0001, + "loss": 67.1573, + "step": 12450 + }, + { + "epoch": 2.011949698229749, + "grad_norm": 0.6354495286941528, + "learning_rate": 0.0001, + "loss": 67.6646, + "step": 12460 + }, + { + "epoch": 2.01356452231485, + "grad_norm": 0.6671343445777893, + "learning_rate": 0.0001, + "loss": 66.9442, + "step": 12470 + }, + { + "epoch": 2.0151793463999517, + "grad_norm": 0.5987095236778259, + "learning_rate": 0.0001, + "loss": 68.265, + "step": 12480 + }, + { + "epoch": 2.0167941704850527, + "grad_norm": 0.8267726302146912, + "learning_rate": 0.0001, + "loss": 67.7709, + "step": 12490 + }, + { + "epoch": 2.018408994570154, + "grad_norm": 0.5237421989440918, + "learning_rate": 0.0001, + "loss": 64.8426, + "step": 12500 + }, + { + "epoch": 2.0200238186552553, + "grad_norm": 0.5602227449417114, + "learning_rate": 0.0001, + "loss": 65.4287, + "step": 12510 + }, + { + "epoch": 2.0216386427403563, + "grad_norm": 0.6129289269447327, + "learning_rate": 0.0001, + "loss": 65.9583, + "step": 12520 + }, + { + "epoch": 2.023253466825458, + "grad_norm": 0.6047223210334778, + "learning_rate": 0.0001, + "loss": 66.9257, + "step": 12530 + }, + { + "epoch": 2.024868290910559, + "grad_norm": 0.6117433905601501, + "learning_rate": 0.0001, + "loss": 63.0344, + "step": 12540 + }, + { + "epoch": 2.0264831149956604, + "grad_norm": 0.6688846945762634, + "learning_rate": 0.0001, + "loss": 66.2295, + "step": 12550 + }, + { + "epoch": 2.0280979390807614, + "grad_norm": 0.779574453830719, + "learning_rate": 0.0001, + "loss": 67.066, + "step": 12560 + }, + { + "epoch": 2.0297127631658625, + "grad_norm": 0.624839186668396, + "learning_rate": 0.0001, + "loss": 66.7905, + "step": 12570 + }, + { + "epoch": 2.031327587250964, + "grad_norm": 0.5970683693885803, + "learning_rate": 0.0001, + "loss": 67.6997, + "step": 12580 + }, + { + "epoch": 2.032942411336065, + "grad_norm": 0.6005929708480835, + "learning_rate": 0.0001, + "loss": 66.717, + "step": 12590 + }, + { + "epoch": 2.0345572354211665, + "grad_norm": 0.6358321905136108, + "learning_rate": 0.0001, + "loss": 67.8494, + "step": 12600 + }, + { + "epoch": 2.0361720595062676, + "grad_norm": 0.8642273545265198, + "learning_rate": 0.0001, + "loss": 67.1791, + "step": 12610 + }, + { + "epoch": 2.0377868835913686, + "grad_norm": 0.587282121181488, + "learning_rate": 0.0001, + "loss": 66.8169, + "step": 12620 + }, + { + "epoch": 2.03940170767647, + "grad_norm": 0.5977926850318909, + "learning_rate": 0.0001, + "loss": 66.8393, + "step": 12630 + }, + { + "epoch": 2.041016531761571, + "grad_norm": 0.5654817223548889, + "learning_rate": 0.0001, + "loss": 68.8243, + "step": 12640 + }, + { + "epoch": 2.0426313558466727, + "grad_norm": 0.5778902173042297, + "learning_rate": 0.0001, + "loss": 66.8893, + "step": 12650 + }, + { + "epoch": 2.0442461799317737, + "grad_norm": 0.6533665657043457, + "learning_rate": 0.0001, + "loss": 66.6174, + "step": 12660 + }, + { + "epoch": 2.0458610040168748, + "grad_norm": 0.5945654511451721, + "learning_rate": 0.0001, + "loss": 67.8193, + "step": 12670 + }, + { + "epoch": 2.0474758281019763, + "grad_norm": 0.5898523926734924, + "learning_rate": 0.0001, + "loss": 65.6194, + "step": 12680 + }, + { + "epoch": 2.0490906521870773, + "grad_norm": 0.5497958660125732, + "learning_rate": 0.0001, + "loss": 66.4571, + "step": 12690 + }, + { + "epoch": 2.050705476272179, + "grad_norm": 0.6195752024650574, + "learning_rate": 0.0001, + "loss": 66.0555, + "step": 12700 + }, + { + "epoch": 2.05232030035728, + "grad_norm": 0.6400334239006042, + "learning_rate": 0.0001, + "loss": 66.7153, + "step": 12710 + }, + { + "epoch": 2.053935124442381, + "grad_norm": 0.7953216433525085, + "learning_rate": 0.0001, + "loss": 66.5822, + "step": 12720 + }, + { + "epoch": 2.0555499485274824, + "grad_norm": 0.8052420616149902, + "learning_rate": 0.0001, + "loss": 66.4395, + "step": 12730 + }, + { + "epoch": 2.0571647726125835, + "grad_norm": 0.567957878112793, + "learning_rate": 0.0001, + "loss": 65.7762, + "step": 12740 + }, + { + "epoch": 2.058779596697685, + "grad_norm": 0.6517317891120911, + "learning_rate": 0.0001, + "loss": 66.7526, + "step": 12750 + }, + { + "epoch": 2.060394420782786, + "grad_norm": 0.6032442450523376, + "learning_rate": 0.0001, + "loss": 64.2725, + "step": 12760 + }, + { + "epoch": 2.062009244867887, + "grad_norm": 0.6162487268447876, + "learning_rate": 0.0001, + "loss": 65.4797, + "step": 12770 + }, + { + "epoch": 2.0636240689529886, + "grad_norm": 0.7512190937995911, + "learning_rate": 0.0001, + "loss": 65.9602, + "step": 12780 + }, + { + "epoch": 2.0652388930380896, + "grad_norm": 0.7601141929626465, + "learning_rate": 0.0001, + "loss": 66.0806, + "step": 12790 + }, + { + "epoch": 2.066853717123191, + "grad_norm": 0.5737994909286499, + "learning_rate": 0.0001, + "loss": 66.8124, + "step": 12800 + }, + { + "epoch": 2.068468541208292, + "grad_norm": 0.6629387736320496, + "learning_rate": 0.0001, + "loss": 65.8944, + "step": 12810 + }, + { + "epoch": 2.070083365293393, + "grad_norm": 0.6374203562736511, + "learning_rate": 0.0001, + "loss": 67.1589, + "step": 12820 + }, + { + "epoch": 2.0716981893784947, + "grad_norm": 0.6732942461967468, + "learning_rate": 0.0001, + "loss": 67.1901, + "step": 12830 + }, + { + "epoch": 2.0733130134635958, + "grad_norm": 0.6219559907913208, + "learning_rate": 0.0001, + "loss": 66.613, + "step": 12840 + }, + { + "epoch": 2.0749278375486973, + "grad_norm": 0.591238796710968, + "learning_rate": 0.0001, + "loss": 67.6111, + "step": 12850 + }, + { + "epoch": 2.0765426616337983, + "grad_norm": 0.6472163200378418, + "learning_rate": 0.0001, + "loss": 66.305, + "step": 12860 + }, + { + "epoch": 2.0781574857188994, + "grad_norm": 0.6349777579307556, + "learning_rate": 0.0001, + "loss": 65.2865, + "step": 12870 + }, + { + "epoch": 2.079772309804001, + "grad_norm": 0.9257296323776245, + "learning_rate": 0.0001, + "loss": 67.6326, + "step": 12880 + }, + { + "epoch": 2.081387133889102, + "grad_norm": 0.5744728446006775, + "learning_rate": 0.0001, + "loss": 65.7023, + "step": 12890 + }, + { + "epoch": 2.0830019579742034, + "grad_norm": 0.6725465059280396, + "learning_rate": 0.0001, + "loss": 65.1233, + "step": 12900 + }, + { + "epoch": 2.0846167820593045, + "grad_norm": 0.6141199469566345, + "learning_rate": 0.0001, + "loss": 65.5593, + "step": 12910 + }, + { + "epoch": 2.0862316061444055, + "grad_norm": 0.6364370584487915, + "learning_rate": 0.0001, + "loss": 65.3899, + "step": 12920 + }, + { + "epoch": 2.087846430229507, + "grad_norm": 0.5948928594589233, + "learning_rate": 0.0001, + "loss": 63.5976, + "step": 12930 + }, + { + "epoch": 2.089461254314608, + "grad_norm": 0.630434513092041, + "learning_rate": 0.0001, + "loss": 66.764, + "step": 12940 + }, + { + "epoch": 2.091076078399709, + "grad_norm": 0.6178410649299622, + "learning_rate": 0.0001, + "loss": 66.3948, + "step": 12950 + }, + { + "epoch": 2.0926909024848106, + "grad_norm": 0.7695903778076172, + "learning_rate": 0.0001, + "loss": 68.2218, + "step": 12960 + }, + { + "epoch": 2.0943057265699117, + "grad_norm": 0.6712848544120789, + "learning_rate": 0.0001, + "loss": 66.8801, + "step": 12970 + }, + { + "epoch": 2.095920550655013, + "grad_norm": 0.7133443355560303, + "learning_rate": 0.0001, + "loss": 65.343, + "step": 12980 + }, + { + "epoch": 2.097535374740114, + "grad_norm": 0.6836514472961426, + "learning_rate": 0.0001, + "loss": 64.4993, + "step": 12990 + }, + { + "epoch": 2.0991501988252157, + "grad_norm": 0.6734977960586548, + "learning_rate": 0.0001, + "loss": 67.6078, + "step": 13000 + }, + { + "epoch": 2.1007650229103167, + "grad_norm": 0.590630054473877, + "learning_rate": 0.0001, + "loss": 65.4567, + "step": 13010 + }, + { + "epoch": 2.102379846995418, + "grad_norm": 0.7174671292304993, + "learning_rate": 0.0001, + "loss": 66.2743, + "step": 13020 + }, + { + "epoch": 2.1039946710805193, + "grad_norm": 0.5266520977020264, + "learning_rate": 0.0001, + "loss": 65.0063, + "step": 13030 + }, + { + "epoch": 2.1056094951656203, + "grad_norm": 0.550865113735199, + "learning_rate": 0.0001, + "loss": 67.0641, + "step": 13040 + }, + { + "epoch": 2.1072243192507214, + "grad_norm": 0.7976025938987732, + "learning_rate": 0.0001, + "loss": 67.0135, + "step": 13050 + }, + { + "epoch": 2.108839143335823, + "grad_norm": 0.7592310905456543, + "learning_rate": 0.0001, + "loss": 65.9875, + "step": 13060 + }, + { + "epoch": 2.110453967420924, + "grad_norm": 0.7613953948020935, + "learning_rate": 0.0001, + "loss": 66.0958, + "step": 13070 + }, + { + "epoch": 2.1120687915060254, + "grad_norm": 0.6951309442520142, + "learning_rate": 0.0001, + "loss": 65.949, + "step": 13080 + }, + { + "epoch": 2.1136836155911265, + "grad_norm": 0.6439896821975708, + "learning_rate": 0.0001, + "loss": 66.4056, + "step": 13090 + }, + { + "epoch": 2.115298439676228, + "grad_norm": 0.6922518610954285, + "learning_rate": 0.0001, + "loss": 67.537, + "step": 13100 + }, + { + "epoch": 2.116913263761329, + "grad_norm": 0.5330061316490173, + "learning_rate": 0.0001, + "loss": 67.0438, + "step": 13110 + }, + { + "epoch": 2.11852808784643, + "grad_norm": 0.6507022380828857, + "learning_rate": 0.0001, + "loss": 66.9908, + "step": 13120 + }, + { + "epoch": 2.1201429119315316, + "grad_norm": 0.994196891784668, + "learning_rate": 0.0001, + "loss": 63.9818, + "step": 13130 + }, + { + "epoch": 2.1217577360166326, + "grad_norm": 0.7431663870811462, + "learning_rate": 0.0001, + "loss": 65.8259, + "step": 13140 + }, + { + "epoch": 2.1233725601017337, + "grad_norm": 0.5852991938591003, + "learning_rate": 0.0001, + "loss": 65.918, + "step": 13150 + }, + { + "epoch": 2.124987384186835, + "grad_norm": 0.551194429397583, + "learning_rate": 0.0001, + "loss": 64.9907, + "step": 13160 + }, + { + "epoch": 2.1266022082719362, + "grad_norm": 0.6882205605506897, + "learning_rate": 0.0001, + "loss": 66.7266, + "step": 13170 + }, + { + "epoch": 2.1282170323570377, + "grad_norm": 0.8906047940254211, + "learning_rate": 0.0001, + "loss": 65.3441, + "step": 13180 + }, + { + "epoch": 2.129831856442139, + "grad_norm": 0.5340781211853027, + "learning_rate": 0.0001, + "loss": 65.6772, + "step": 13190 + }, + { + "epoch": 2.1314466805272403, + "grad_norm": 0.7657933831214905, + "learning_rate": 0.0001, + "loss": 65.3888, + "step": 13200 + }, + { + "epoch": 2.1330615046123413, + "grad_norm": 1.2086169719696045, + "learning_rate": 0.0001, + "loss": 65.7214, + "step": 13210 + }, + { + "epoch": 2.1346763286974424, + "grad_norm": 0.6794751286506653, + "learning_rate": 0.0001, + "loss": 66.6761, + "step": 13220 + }, + { + "epoch": 2.136291152782544, + "grad_norm": 0.5939100384712219, + "learning_rate": 0.0001, + "loss": 64.1858, + "step": 13230 + }, + { + "epoch": 2.137905976867645, + "grad_norm": 0.581163763999939, + "learning_rate": 0.0001, + "loss": 65.9456, + "step": 13240 + }, + { + "epoch": 2.139520800952746, + "grad_norm": 0.676103949546814, + "learning_rate": 0.0001, + "loss": 65.1919, + "step": 13250 + }, + { + "epoch": 2.1411356250378475, + "grad_norm": 0.6052015423774719, + "learning_rate": 0.0001, + "loss": 67.9872, + "step": 13260 + }, + { + "epoch": 2.1427504491229485, + "grad_norm": 0.5938939452171326, + "learning_rate": 0.0001, + "loss": 64.7395, + "step": 13270 + }, + { + "epoch": 2.14436527320805, + "grad_norm": 0.563115656375885, + "learning_rate": 0.0001, + "loss": 63.9757, + "step": 13280 + }, + { + "epoch": 2.145980097293151, + "grad_norm": 0.5397975444793701, + "learning_rate": 0.0001, + "loss": 65.6596, + "step": 13290 + }, + { + "epoch": 2.1475949213782526, + "grad_norm": 0.6397805213928223, + "learning_rate": 0.0001, + "loss": 67.0618, + "step": 13300 + }, + { + "epoch": 2.1492097454633536, + "grad_norm": 0.678053081035614, + "learning_rate": 0.0001, + "loss": 65.8194, + "step": 13310 + }, + { + "epoch": 2.1508245695484547, + "grad_norm": 0.6067008972167969, + "learning_rate": 0.0001, + "loss": 65.9035, + "step": 13320 + }, + { + "epoch": 2.152439393633556, + "grad_norm": 0.6560337543487549, + "learning_rate": 0.0001, + "loss": 64.9907, + "step": 13330 + }, + { + "epoch": 2.1540542177186572, + "grad_norm": 0.5570183396339417, + "learning_rate": 0.0001, + "loss": 64.5903, + "step": 13340 + }, + { + "epoch": 2.1556690418037583, + "grad_norm": 0.6721981167793274, + "learning_rate": 0.0001, + "loss": 66.5618, + "step": 13350 + }, + { + "epoch": 2.15728386588886, + "grad_norm": 0.7251304388046265, + "learning_rate": 0.0001, + "loss": 66.724, + "step": 13360 + }, + { + "epoch": 2.158898689973961, + "grad_norm": 0.5928503274917603, + "learning_rate": 0.0001, + "loss": 66.6991, + "step": 13370 + }, + { + "epoch": 2.1605135140590623, + "grad_norm": 0.7684288620948792, + "learning_rate": 0.0001, + "loss": 66.4064, + "step": 13380 + }, + { + "epoch": 2.1621283381441634, + "grad_norm": 0.5499702095985413, + "learning_rate": 0.0001, + "loss": 66.7671, + "step": 13390 + }, + { + "epoch": 2.163743162229265, + "grad_norm": 0.5267577767372131, + "learning_rate": 0.0001, + "loss": 65.8112, + "step": 13400 + }, + { + "epoch": 2.165357986314366, + "grad_norm": 0.6214171648025513, + "learning_rate": 0.0001, + "loss": 65.851, + "step": 13410 + }, + { + "epoch": 2.166972810399467, + "grad_norm": 0.5952336192131042, + "learning_rate": 0.0001, + "loss": 65.4166, + "step": 13420 + }, + { + "epoch": 2.1685876344845685, + "grad_norm": 1.3454428911209106, + "learning_rate": 0.0001, + "loss": 64.7546, + "step": 13430 + }, + { + "epoch": 2.1702024585696695, + "grad_norm": 0.6586198210716248, + "learning_rate": 0.0001, + "loss": 65.6818, + "step": 13440 + }, + { + "epoch": 2.1718172826547706, + "grad_norm": 0.8028526306152344, + "learning_rate": 0.0001, + "loss": 67.2677, + "step": 13450 + }, + { + "epoch": 2.173432106739872, + "grad_norm": 0.5949039459228516, + "learning_rate": 0.0001, + "loss": 66.994, + "step": 13460 + }, + { + "epoch": 2.175046930824973, + "grad_norm": 0.6666626930236816, + "learning_rate": 0.0001, + "loss": 66.6509, + "step": 13470 + }, + { + "epoch": 2.1766617549100746, + "grad_norm": 0.5985739827156067, + "learning_rate": 0.0001, + "loss": 65.9211, + "step": 13480 + }, + { + "epoch": 2.1782765789951757, + "grad_norm": 0.6771738529205322, + "learning_rate": 0.0001, + "loss": 65.3477, + "step": 13490 + }, + { + "epoch": 2.1798914030802767, + "grad_norm": 0.6147419214248657, + "learning_rate": 0.0001, + "loss": 64.4142, + "step": 13500 + }, + { + "epoch": 2.1815062271653782, + "grad_norm": 0.6774442791938782, + "learning_rate": 0.0001, + "loss": 64.5831, + "step": 13510 + }, + { + "epoch": 2.1831210512504793, + "grad_norm": 0.5778859853744507, + "learning_rate": 0.0001, + "loss": 64.4449, + "step": 13520 + }, + { + "epoch": 2.184735875335581, + "grad_norm": 0.5697338581085205, + "learning_rate": 0.0001, + "loss": 67.1627, + "step": 13530 + }, + { + "epoch": 2.186350699420682, + "grad_norm": 1.2444179058074951, + "learning_rate": 0.0001, + "loss": 64.7959, + "step": 13540 + }, + { + "epoch": 2.187965523505783, + "grad_norm": 0.6489098072052002, + "learning_rate": 0.0001, + "loss": 65.9926, + "step": 13550 + }, + { + "epoch": 2.1895803475908844, + "grad_norm": 0.6871664524078369, + "learning_rate": 0.0001, + "loss": 67.681, + "step": 13560 + }, + { + "epoch": 2.1911951716759854, + "grad_norm": 0.5581620931625366, + "learning_rate": 0.0001, + "loss": 65.6707, + "step": 13570 + }, + { + "epoch": 2.192809995761087, + "grad_norm": 0.522402286529541, + "learning_rate": 0.0001, + "loss": 65.9014, + "step": 13580 + }, + { + "epoch": 2.194424819846188, + "grad_norm": 0.6149793267250061, + "learning_rate": 0.0001, + "loss": 64.6348, + "step": 13590 + }, + { + "epoch": 2.196039643931289, + "grad_norm": 0.5616180896759033, + "learning_rate": 0.0001, + "loss": 66.3635, + "step": 13600 + }, + { + "epoch": 2.1976544680163905, + "grad_norm": 1.024154543876648, + "learning_rate": 0.0001, + "loss": 66.7156, + "step": 13610 + }, + { + "epoch": 2.1992692921014916, + "grad_norm": 1.035205602645874, + "learning_rate": 0.0001, + "loss": 65.996, + "step": 13620 + }, + { + "epoch": 2.200884116186593, + "grad_norm": 0.5354496240615845, + "learning_rate": 0.0001, + "loss": 66.6107, + "step": 13630 + }, + { + "epoch": 2.202498940271694, + "grad_norm": 0.5648348331451416, + "learning_rate": 0.0001, + "loss": 66.473, + "step": 13640 + }, + { + "epoch": 2.204113764356795, + "grad_norm": 0.6247662901878357, + "learning_rate": 0.0001, + "loss": 66.576, + "step": 13650 + }, + { + "epoch": 2.2057285884418967, + "grad_norm": 0.5871408581733704, + "learning_rate": 0.0001, + "loss": 66.6254, + "step": 13660 + }, + { + "epoch": 2.2073434125269977, + "grad_norm": 0.6394441723823547, + "learning_rate": 0.0001, + "loss": 66.2924, + "step": 13670 + }, + { + "epoch": 2.2089582366120992, + "grad_norm": 0.6017318367958069, + "learning_rate": 0.0001, + "loss": 66.6313, + "step": 13680 + }, + { + "epoch": 2.2105730606972003, + "grad_norm": 0.5696585774421692, + "learning_rate": 0.0001, + "loss": 65.9198, + "step": 13690 + }, + { + "epoch": 2.2121878847823013, + "grad_norm": 0.5966867804527283, + "learning_rate": 0.0001, + "loss": 67.507, + "step": 13700 + }, + { + "epoch": 2.213802708867403, + "grad_norm": 0.6163555979728699, + "learning_rate": 0.0001, + "loss": 67.2444, + "step": 13710 + }, + { + "epoch": 2.215417532952504, + "grad_norm": 0.5670737624168396, + "learning_rate": 0.0001, + "loss": 66.5747, + "step": 13720 + }, + { + "epoch": 2.2170323570376054, + "grad_norm": 0.6289104223251343, + "learning_rate": 0.0001, + "loss": 65.6289, + "step": 13730 + }, + { + "epoch": 2.2186471811227064, + "grad_norm": 0.5649837255477905, + "learning_rate": 0.0001, + "loss": 66.0953, + "step": 13740 + }, + { + "epoch": 2.2202620052078075, + "grad_norm": 0.6066703200340271, + "learning_rate": 0.0001, + "loss": 65.9425, + "step": 13750 + }, + { + "epoch": 2.221876829292909, + "grad_norm": 0.6202117204666138, + "learning_rate": 0.0001, + "loss": 65.3268, + "step": 13760 + }, + { + "epoch": 2.22349165337801, + "grad_norm": 0.6883425116539001, + "learning_rate": 0.0001, + "loss": 64.1026, + "step": 13770 + }, + { + "epoch": 2.2251064774631115, + "grad_norm": 0.5564731955528259, + "learning_rate": 0.0001, + "loss": 65.1203, + "step": 13780 + }, + { + "epoch": 2.2267213015482126, + "grad_norm": 0.48229852318763733, + "learning_rate": 0.0001, + "loss": 65.2423, + "step": 13790 + }, + { + "epoch": 2.2283361256333136, + "grad_norm": 0.549240231513977, + "learning_rate": 0.0001, + "loss": 68.3811, + "step": 13800 + }, + { + "epoch": 2.229950949718415, + "grad_norm": 0.7217127084732056, + "learning_rate": 0.0001, + "loss": 64.9966, + "step": 13810 + }, + { + "epoch": 2.231565773803516, + "grad_norm": 0.8491377830505371, + "learning_rate": 0.0001, + "loss": 65.6468, + "step": 13820 + }, + { + "epoch": 2.2331805978886177, + "grad_norm": 0.5460603833198547, + "learning_rate": 0.0001, + "loss": 65.192, + "step": 13830 + }, + { + "epoch": 2.2347954219737187, + "grad_norm": 0.5784180164337158, + "learning_rate": 0.0001, + "loss": 67.3488, + "step": 13840 + }, + { + "epoch": 2.2364102460588198, + "grad_norm": 0.7575398087501526, + "learning_rate": 0.0001, + "loss": 65.6969, + "step": 13850 + }, + { + "epoch": 2.2380250701439213, + "grad_norm": 0.5111517310142517, + "learning_rate": 0.0001, + "loss": 66.482, + "step": 13860 + }, + { + "epoch": 2.2396398942290223, + "grad_norm": 0.6383832693099976, + "learning_rate": 0.0001, + "loss": 67.4894, + "step": 13870 + }, + { + "epoch": 2.241254718314124, + "grad_norm": 0.5922625064849854, + "learning_rate": 0.0001, + "loss": 66.1718, + "step": 13880 + }, + { + "epoch": 2.242869542399225, + "grad_norm": 5.395338535308838, + "learning_rate": 0.0001, + "loss": 64.7385, + "step": 13890 + }, + { + "epoch": 2.244484366484326, + "grad_norm": 1.4203481674194336, + "learning_rate": 0.0001, + "loss": 64.7676, + "step": 13900 + }, + { + "epoch": 2.2460991905694274, + "grad_norm": 0.6692256331443787, + "learning_rate": 0.0001, + "loss": 65.856, + "step": 13910 + }, + { + "epoch": 2.2477140146545285, + "grad_norm": 0.5868324041366577, + "learning_rate": 0.0001, + "loss": 66.1823, + "step": 13920 + }, + { + "epoch": 2.24932883873963, + "grad_norm": 0.6828433275222778, + "learning_rate": 0.0001, + "loss": 65.7601, + "step": 13930 + }, + { + "epoch": 2.250943662824731, + "grad_norm": 0.5108211040496826, + "learning_rate": 0.0001, + "loss": 63.413, + "step": 13940 + }, + { + "epoch": 2.252558486909832, + "grad_norm": 0.5538185238838196, + "learning_rate": 0.0001, + "loss": 64.9088, + "step": 13950 + }, + { + "epoch": 2.2541733109949336, + "grad_norm": 0.7211924195289612, + "learning_rate": 0.0001, + "loss": 66.0293, + "step": 13960 + }, + { + "epoch": 2.2557881350800346, + "grad_norm": 0.5632925033569336, + "learning_rate": 0.0001, + "loss": 64.8261, + "step": 13970 + }, + { + "epoch": 2.257402959165136, + "grad_norm": 0.574968159198761, + "learning_rate": 0.0001, + "loss": 65.2667, + "step": 13980 + }, + { + "epoch": 2.259017783250237, + "grad_norm": 0.5919519066810608, + "learning_rate": 0.0001, + "loss": 66.2643, + "step": 13990 + }, + { + "epoch": 2.260632607335338, + "grad_norm": 0.6732460260391235, + "learning_rate": 0.0001, + "loss": 64.7524, + "step": 14000 + }, + { + "epoch": 2.2622474314204397, + "grad_norm": 0.5927128791809082, + "learning_rate": 0.0001, + "loss": 65.653, + "step": 14010 + }, + { + "epoch": 2.2638622555055408, + "grad_norm": 0.5900641083717346, + "learning_rate": 0.0001, + "loss": 65.3171, + "step": 14020 + }, + { + "epoch": 2.2654770795906423, + "grad_norm": 0.5498414635658264, + "learning_rate": 0.0001, + "loss": 66.8886, + "step": 14030 + }, + { + "epoch": 2.2670919036757433, + "grad_norm": 0.7032901048660278, + "learning_rate": 0.0001, + "loss": 65.9949, + "step": 14040 + }, + { + "epoch": 2.2687067277608444, + "grad_norm": 0.6699029207229614, + "learning_rate": 0.0001, + "loss": 67.2374, + "step": 14050 + }, + { + "epoch": 2.270321551845946, + "grad_norm": 0.5847286581993103, + "learning_rate": 0.0001, + "loss": 63.996, + "step": 14060 + }, + { + "epoch": 2.271936375931047, + "grad_norm": 0.6173244714736938, + "learning_rate": 0.0001, + "loss": 66.927, + "step": 14070 + }, + { + "epoch": 2.2735512000161484, + "grad_norm": 0.6063929200172424, + "learning_rate": 0.0001, + "loss": 64.1213, + "step": 14080 + }, + { + "epoch": 2.2751660241012495, + "grad_norm": 0.5927698016166687, + "learning_rate": 0.0001, + "loss": 65.5379, + "step": 14090 + }, + { + "epoch": 2.2767808481863505, + "grad_norm": 0.6632462739944458, + "learning_rate": 0.0001, + "loss": 66.4461, + "step": 14100 + }, + { + "epoch": 2.278395672271452, + "grad_norm": 0.6032512187957764, + "learning_rate": 0.0001, + "loss": 65.3191, + "step": 14110 + }, + { + "epoch": 2.280010496356553, + "grad_norm": 0.5799509882926941, + "learning_rate": 0.0001, + "loss": 64.7406, + "step": 14120 + }, + { + "epoch": 2.2816253204416546, + "grad_norm": 0.6965101361274719, + "learning_rate": 0.0001, + "loss": 65.2813, + "step": 14130 + }, + { + "epoch": 2.2832401445267556, + "grad_norm": 0.6447696089744568, + "learning_rate": 0.0001, + "loss": 65.6274, + "step": 14140 + }, + { + "epoch": 2.2848549686118567, + "grad_norm": 0.7639263272285461, + "learning_rate": 0.0001, + "loss": 64.9972, + "step": 14150 + }, + { + "epoch": 2.286469792696958, + "grad_norm": 0.6040478944778442, + "learning_rate": 0.0001, + "loss": 66.1419, + "step": 14160 + }, + { + "epoch": 2.288084616782059, + "grad_norm": 0.6466631889343262, + "learning_rate": 0.0001, + "loss": 66.0022, + "step": 14170 + }, + { + "epoch": 2.2896994408671607, + "grad_norm": 0.6795923709869385, + "learning_rate": 0.0001, + "loss": 65.7562, + "step": 14180 + }, + { + "epoch": 2.2913142649522618, + "grad_norm": 0.5632458329200745, + "learning_rate": 0.0001, + "loss": 64.7196, + "step": 14190 + }, + { + "epoch": 2.292929089037363, + "grad_norm": 1.2942395210266113, + "learning_rate": 0.0001, + "loss": 65.3416, + "step": 14200 + }, + { + "epoch": 2.2945439131224643, + "grad_norm": 0.728187084197998, + "learning_rate": 0.0001, + "loss": 65.3743, + "step": 14210 + }, + { + "epoch": 2.2961587372075654, + "grad_norm": 0.5735196471214294, + "learning_rate": 0.0001, + "loss": 67.1841, + "step": 14220 + }, + { + "epoch": 2.297773561292667, + "grad_norm": 0.5804634690284729, + "learning_rate": 0.0001, + "loss": 65.856, + "step": 14230 + }, + { + "epoch": 2.299388385377768, + "grad_norm": 0.5995981693267822, + "learning_rate": 0.0001, + "loss": 65.7268, + "step": 14240 + }, + { + "epoch": 2.301003209462869, + "grad_norm": 0.4901711940765381, + "learning_rate": 0.0001, + "loss": 66.5355, + "step": 14250 + }, + { + "epoch": 2.3026180335479705, + "grad_norm": 0.6467465162277222, + "learning_rate": 0.0001, + "loss": 66.0457, + "step": 14260 + }, + { + "epoch": 2.3042328576330715, + "grad_norm": 0.6831316351890564, + "learning_rate": 0.0001, + "loss": 65.7363, + "step": 14270 + }, + { + "epoch": 2.305847681718173, + "grad_norm": 0.5308393836021423, + "learning_rate": 0.0001, + "loss": 64.5911, + "step": 14280 + }, + { + "epoch": 2.307462505803274, + "grad_norm": 0.6398120522499084, + "learning_rate": 0.0001, + "loss": 63.9705, + "step": 14290 + }, + { + "epoch": 2.309077329888375, + "grad_norm": 0.5657719969749451, + "learning_rate": 0.0001, + "loss": 65.9251, + "step": 14300 + }, + { + "epoch": 2.3106921539734766, + "grad_norm": 0.5824252367019653, + "learning_rate": 0.0001, + "loss": 64.7748, + "step": 14310 + }, + { + "epoch": 2.3123069780585777, + "grad_norm": 0.6470819115638733, + "learning_rate": 0.0001, + "loss": 65.2014, + "step": 14320 + }, + { + "epoch": 2.313921802143679, + "grad_norm": 0.643173336982727, + "learning_rate": 0.0001, + "loss": 65.9344, + "step": 14330 + }, + { + "epoch": 2.31553662622878, + "grad_norm": 0.6090084314346313, + "learning_rate": 0.0001, + "loss": 66.0823, + "step": 14340 + }, + { + "epoch": 2.3171514503138813, + "grad_norm": 0.627765953540802, + "learning_rate": 0.0001, + "loss": 66.2703, + "step": 14350 + }, + { + "epoch": 2.3187662743989828, + "grad_norm": 0.6484197974205017, + "learning_rate": 0.0001, + "loss": 65.928, + "step": 14360 + }, + { + "epoch": 2.320381098484084, + "grad_norm": 0.653978168964386, + "learning_rate": 0.0001, + "loss": 67.0193, + "step": 14370 + }, + { + "epoch": 2.3219959225691853, + "grad_norm": 0.6991199851036072, + "learning_rate": 0.0001, + "loss": 66.1559, + "step": 14380 + }, + { + "epoch": 2.3236107466542864, + "grad_norm": 0.938056468963623, + "learning_rate": 0.0001, + "loss": 66.6212, + "step": 14390 + }, + { + "epoch": 2.3252255707393874, + "grad_norm": 0.6154502630233765, + "learning_rate": 0.0001, + "loss": 64.5868, + "step": 14400 + }, + { + "epoch": 2.326840394824489, + "grad_norm": 0.6453447341918945, + "learning_rate": 0.0001, + "loss": 67.5199, + "step": 14410 + }, + { + "epoch": 2.32845521890959, + "grad_norm": 0.6678239107131958, + "learning_rate": 0.0001, + "loss": 65.3518, + "step": 14420 + }, + { + "epoch": 2.3300700429946914, + "grad_norm": 0.6608372330665588, + "learning_rate": 0.0001, + "loss": 65.1985, + "step": 14430 + }, + { + "epoch": 2.3316848670797925, + "grad_norm": 0.625511646270752, + "learning_rate": 0.0001, + "loss": 66.9224, + "step": 14440 + }, + { + "epoch": 2.3332996911648936, + "grad_norm": 0.6678557991981506, + "learning_rate": 0.0001, + "loss": 65.8618, + "step": 14450 + }, + { + "epoch": 2.334914515249995, + "grad_norm": 0.6184708476066589, + "learning_rate": 0.0001, + "loss": 66.5419, + "step": 14460 + }, + { + "epoch": 2.336529339335096, + "grad_norm": 0.6668255925178528, + "learning_rate": 0.0001, + "loss": 64.0567, + "step": 14470 + }, + { + "epoch": 2.3381441634201976, + "grad_norm": 0.5756039023399353, + "learning_rate": 0.0001, + "loss": 63.043, + "step": 14480 + }, + { + "epoch": 2.3397589875052986, + "grad_norm": 0.6090635657310486, + "learning_rate": 0.0001, + "loss": 65.8793, + "step": 14490 + }, + { + "epoch": 2.3413738115903997, + "grad_norm": 0.8366266489028931, + "learning_rate": 0.0001, + "loss": 65.1995, + "step": 14500 + }, + { + "epoch": 2.342988635675501, + "grad_norm": 0.6535093188285828, + "learning_rate": 0.0001, + "loss": 63.3831, + "step": 14510 + }, + { + "epoch": 2.3446034597606022, + "grad_norm": 0.9697316884994507, + "learning_rate": 0.0001, + "loss": 65.1754, + "step": 14520 + }, + { + "epoch": 2.3462182838457037, + "grad_norm": 0.7946458458900452, + "learning_rate": 0.0001, + "loss": 65.5404, + "step": 14530 + }, + { + "epoch": 2.347833107930805, + "grad_norm": 0.5621834397315979, + "learning_rate": 0.0001, + "loss": 65.8532, + "step": 14540 + }, + { + "epoch": 2.349447932015906, + "grad_norm": 0.6668163537979126, + "learning_rate": 0.0001, + "loss": 65.0884, + "step": 14550 + }, + { + "epoch": 2.3510627561010073, + "grad_norm": 0.5969575047492981, + "learning_rate": 0.0001, + "loss": 66.5821, + "step": 14560 + }, + { + "epoch": 2.3526775801861084, + "grad_norm": 0.6905517578125, + "learning_rate": 0.0001, + "loss": 65.0928, + "step": 14570 + }, + { + "epoch": 2.35429240427121, + "grad_norm": 0.5888230204582214, + "learning_rate": 0.0001, + "loss": 67.4596, + "step": 14580 + }, + { + "epoch": 2.355907228356311, + "grad_norm": 0.5115624666213989, + "learning_rate": 0.0001, + "loss": 65.771, + "step": 14590 + }, + { + "epoch": 2.357522052441412, + "grad_norm": 0.7888309359550476, + "learning_rate": 0.0001, + "loss": 63.5326, + "step": 14600 + }, + { + "epoch": 2.3591368765265135, + "grad_norm": 0.6653981804847717, + "learning_rate": 0.0001, + "loss": 64.8015, + "step": 14610 + }, + { + "epoch": 2.3607517006116145, + "grad_norm": 0.7898159027099609, + "learning_rate": 0.0001, + "loss": 66.398, + "step": 14620 + }, + { + "epoch": 2.362366524696716, + "grad_norm": 1.02628755569458, + "learning_rate": 0.0001, + "loss": 67.6004, + "step": 14630 + }, + { + "epoch": 2.363981348781817, + "grad_norm": 0.5777941942214966, + "learning_rate": 0.0001, + "loss": 64.9402, + "step": 14640 + }, + { + "epoch": 2.365596172866918, + "grad_norm": 0.7004761695861816, + "learning_rate": 0.0001, + "loss": 64.9344, + "step": 14650 + }, + { + "epoch": 2.3672109969520196, + "grad_norm": 0.6230147480964661, + "learning_rate": 0.0001, + "loss": 64.8412, + "step": 14660 + }, + { + "epoch": 2.3688258210371207, + "grad_norm": 0.7234131097793579, + "learning_rate": 0.0001, + "loss": 65.4454, + "step": 14670 + }, + { + "epoch": 2.370440645122222, + "grad_norm": 0.7054212093353271, + "learning_rate": 0.0001, + "loss": 62.2294, + "step": 14680 + }, + { + "epoch": 2.3720554692073232, + "grad_norm": 0.5377123355865479, + "learning_rate": 0.0001, + "loss": 63.1186, + "step": 14690 + }, + { + "epoch": 2.3736702932924243, + "grad_norm": 0.5648056864738464, + "learning_rate": 0.0001, + "loss": 66.1893, + "step": 14700 + }, + { + "epoch": 2.375285117377526, + "grad_norm": 0.6017388105392456, + "learning_rate": 0.0001, + "loss": 64.8003, + "step": 14710 + }, + { + "epoch": 2.376899941462627, + "grad_norm": 0.592069149017334, + "learning_rate": 0.0001, + "loss": 64.7885, + "step": 14720 + }, + { + "epoch": 2.3785147655477283, + "grad_norm": 0.5448320508003235, + "learning_rate": 0.0001, + "loss": 64.3022, + "step": 14730 + }, + { + "epoch": 2.3801295896328294, + "grad_norm": 0.7200132012367249, + "learning_rate": 0.0001, + "loss": 65.3915, + "step": 14740 + }, + { + "epoch": 2.3817444137179304, + "grad_norm": 0.6497626304626465, + "learning_rate": 0.0001, + "loss": 63.9914, + "step": 14750 + }, + { + "epoch": 2.383359237803032, + "grad_norm": 0.7847583293914795, + "learning_rate": 0.0001, + "loss": 65.7746, + "step": 14760 + }, + { + "epoch": 2.384974061888133, + "grad_norm": 0.6991321444511414, + "learning_rate": 0.0001, + "loss": 65.0386, + "step": 14770 + }, + { + "epoch": 2.386588885973234, + "grad_norm": 0.5715296268463135, + "learning_rate": 0.0001, + "loss": 64.2633, + "step": 14780 + }, + { + "epoch": 2.3882037100583355, + "grad_norm": 0.5266057848930359, + "learning_rate": 0.0001, + "loss": 63.0002, + "step": 14790 + }, + { + "epoch": 2.3898185341434366, + "grad_norm": 0.7040116786956787, + "learning_rate": 0.0001, + "loss": 66.2895, + "step": 14800 + }, + { + "epoch": 2.391433358228538, + "grad_norm": 0.6963793635368347, + "learning_rate": 0.0001, + "loss": 64.7487, + "step": 14810 + }, + { + "epoch": 2.393048182313639, + "grad_norm": 0.5361505746841431, + "learning_rate": 0.0001, + "loss": 66.6834, + "step": 14820 + }, + { + "epoch": 2.3946630063987406, + "grad_norm": 0.8016838431358337, + "learning_rate": 0.0001, + "loss": 64.3359, + "step": 14830 + }, + { + "epoch": 2.3962778304838417, + "grad_norm": 0.5636866092681885, + "learning_rate": 0.0001, + "loss": 65.2468, + "step": 14840 + }, + { + "epoch": 2.3978926545689427, + "grad_norm": 0.6926373839378357, + "learning_rate": 0.0001, + "loss": 66.3353, + "step": 14850 + }, + { + "epoch": 2.3995074786540442, + "grad_norm": 0.6361585855484009, + "learning_rate": 0.0001, + "loss": 64.8466, + "step": 14860 + }, + { + "epoch": 2.4011223027391453, + "grad_norm": 0.6232298612594604, + "learning_rate": 0.0001, + "loss": 65.1253, + "step": 14870 + }, + { + "epoch": 2.4027371268242463, + "grad_norm": 0.5644697546958923, + "learning_rate": 0.0001, + "loss": 67.1739, + "step": 14880 + }, + { + "epoch": 2.404351950909348, + "grad_norm": 0.5655877590179443, + "learning_rate": 0.0001, + "loss": 65.1823, + "step": 14890 + }, + { + "epoch": 2.405966774994449, + "grad_norm": 0.8255041837692261, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 14900 + }, + { + "epoch": 2.4075815990795504, + "grad_norm": 0.856689989566803, + "learning_rate": 0.0001, + "loss": 64.4978, + "step": 14910 + }, + { + "epoch": 2.4091964231646514, + "grad_norm": 0.6280230283737183, + "learning_rate": 0.0001, + "loss": 66.3989, + "step": 14920 + }, + { + "epoch": 2.410811247249753, + "grad_norm": 0.5739511251449585, + "learning_rate": 0.0001, + "loss": 64.6365, + "step": 14930 + }, + { + "epoch": 2.412426071334854, + "grad_norm": 0.6244319677352905, + "learning_rate": 0.0001, + "loss": 65.5078, + "step": 14940 + }, + { + "epoch": 2.414040895419955, + "grad_norm": 0.6932142376899719, + "learning_rate": 0.0001, + "loss": 65.1869, + "step": 14950 + }, + { + "epoch": 2.4156557195050565, + "grad_norm": 0.5393000245094299, + "learning_rate": 0.0001, + "loss": 65.553, + "step": 14960 + }, + { + "epoch": 2.4172705435901576, + "grad_norm": 0.6555871963500977, + "learning_rate": 0.0001, + "loss": 66.625, + "step": 14970 + }, + { + "epoch": 2.4188853676752586, + "grad_norm": 0.6515706181526184, + "learning_rate": 0.0001, + "loss": 64.876, + "step": 14980 + }, + { + "epoch": 2.42050019176036, + "grad_norm": 0.6380309462547302, + "learning_rate": 0.0001, + "loss": 63.1801, + "step": 14990 + }, + { + "epoch": 2.422115015845461, + "grad_norm": 0.8313520550727844, + "learning_rate": 0.0001, + "loss": 66.2692, + "step": 15000 + }, + { + "epoch": 2.4237298399305627, + "grad_norm": 0.6654364466667175, + "learning_rate": 0.0001, + "loss": 65.4955, + "step": 15010 + }, + { + "epoch": 2.4253446640156637, + "grad_norm": 0.6382327675819397, + "learning_rate": 0.0001, + "loss": 65.0912, + "step": 15020 + }, + { + "epoch": 2.4269594881007652, + "grad_norm": 0.5685209035873413, + "learning_rate": 0.0001, + "loss": 64.8058, + "step": 15030 + }, + { + "epoch": 2.4285743121858663, + "grad_norm": 0.6129642724990845, + "learning_rate": 0.0001, + "loss": 65.8475, + "step": 15040 + }, + { + "epoch": 2.4301891362709673, + "grad_norm": 0.6892847418785095, + "learning_rate": 0.0001, + "loss": 64.7247, + "step": 15050 + }, + { + "epoch": 2.431803960356069, + "grad_norm": 0.6307751536369324, + "learning_rate": 0.0001, + "loss": 64.0042, + "step": 15060 + }, + { + "epoch": 2.43341878444117, + "grad_norm": 0.5506501197814941, + "learning_rate": 0.0001, + "loss": 64.5555, + "step": 15070 + }, + { + "epoch": 2.435033608526271, + "grad_norm": 0.5580377578735352, + "learning_rate": 0.0001, + "loss": 65.0344, + "step": 15080 + }, + { + "epoch": 2.4366484326113724, + "grad_norm": 0.5712753534317017, + "learning_rate": 0.0001, + "loss": 66.6778, + "step": 15090 + }, + { + "epoch": 2.4382632566964735, + "grad_norm": 0.7153586745262146, + "learning_rate": 0.0001, + "loss": 65.893, + "step": 15100 + }, + { + "epoch": 2.439878080781575, + "grad_norm": 0.6946969628334045, + "learning_rate": 0.0001, + "loss": 64.1815, + "step": 15110 + }, + { + "epoch": 2.441492904866676, + "grad_norm": 0.608729898929596, + "learning_rate": 0.0001, + "loss": 64.8054, + "step": 15120 + }, + { + "epoch": 2.4431077289517775, + "grad_norm": 0.6703302264213562, + "learning_rate": 0.0001, + "loss": 65.3736, + "step": 15130 + }, + { + "epoch": 2.4447225530368786, + "grad_norm": 0.736193835735321, + "learning_rate": 0.0001, + "loss": 66.7051, + "step": 15140 + }, + { + "epoch": 2.4463373771219796, + "grad_norm": 0.48381003737449646, + "learning_rate": 0.0001, + "loss": 64.3723, + "step": 15150 + }, + { + "epoch": 2.447952201207081, + "grad_norm": 0.7487664222717285, + "learning_rate": 0.0001, + "loss": 64.4967, + "step": 15160 + }, + { + "epoch": 2.449567025292182, + "grad_norm": 0.7033581733703613, + "learning_rate": 0.0001, + "loss": 66.2185, + "step": 15170 + }, + { + "epoch": 2.4511818493772832, + "grad_norm": 0.5489983558654785, + "learning_rate": 0.0001, + "loss": 64.9698, + "step": 15180 + }, + { + "epoch": 2.4527966734623847, + "grad_norm": 0.6467112898826599, + "learning_rate": 0.0001, + "loss": 64.5987, + "step": 15190 + }, + { + "epoch": 2.4544114975474858, + "grad_norm": 0.6499149799346924, + "learning_rate": 0.0001, + "loss": 65.757, + "step": 15200 + }, + { + "epoch": 2.4560263216325873, + "grad_norm": 0.7622562050819397, + "learning_rate": 0.0001, + "loss": 66.4642, + "step": 15210 + }, + { + "epoch": 2.4576411457176883, + "grad_norm": 0.8985865712165833, + "learning_rate": 0.0001, + "loss": 64.5804, + "step": 15220 + }, + { + "epoch": 2.45925596980279, + "grad_norm": 0.6116635203361511, + "learning_rate": 0.0001, + "loss": 64.8022, + "step": 15230 + }, + { + "epoch": 2.460870793887891, + "grad_norm": 0.6313971281051636, + "learning_rate": 0.0001, + "loss": 63.4837, + "step": 15240 + }, + { + "epoch": 2.462485617972992, + "grad_norm": 0.7002763152122498, + "learning_rate": 0.0001, + "loss": 64.7201, + "step": 15250 + }, + { + "epoch": 2.4641004420580934, + "grad_norm": 0.6030148863792419, + "learning_rate": 0.0001, + "loss": 65.0987, + "step": 15260 + }, + { + "epoch": 2.4657152661431945, + "grad_norm": 0.5838733911514282, + "learning_rate": 0.0001, + "loss": 64.9182, + "step": 15270 + }, + { + "epoch": 2.4673300902282955, + "grad_norm": 1.1749393939971924, + "learning_rate": 0.0001, + "loss": 63.4387, + "step": 15280 + }, + { + "epoch": 2.468944914313397, + "grad_norm": 0.5215787291526794, + "learning_rate": 0.0001, + "loss": 64.7365, + "step": 15290 + }, + { + "epoch": 2.470559738398498, + "grad_norm": 0.5576916933059692, + "learning_rate": 0.0001, + "loss": 65.644, + "step": 15300 + }, + { + "epoch": 2.4721745624835996, + "grad_norm": 0.6760591864585876, + "learning_rate": 0.0001, + "loss": 64.729, + "step": 15310 + }, + { + "epoch": 2.4737893865687006, + "grad_norm": 0.7162330150604248, + "learning_rate": 0.0001, + "loss": 66.8289, + "step": 15320 + }, + { + "epoch": 2.475404210653802, + "grad_norm": 0.5687427520751953, + "learning_rate": 0.0001, + "loss": 63.9522, + "step": 15330 + }, + { + "epoch": 2.477019034738903, + "grad_norm": 0.5663572549819946, + "learning_rate": 0.0001, + "loss": 65.7151, + "step": 15340 + }, + { + "epoch": 2.478633858824004, + "grad_norm": 0.6018327474594116, + "learning_rate": 0.0001, + "loss": 64.6599, + "step": 15350 + }, + { + "epoch": 2.4802486829091057, + "grad_norm": 0.6908984184265137, + "learning_rate": 0.0001, + "loss": 65.3083, + "step": 15360 + }, + { + "epoch": 2.4818635069942068, + "grad_norm": 0.6582157015800476, + "learning_rate": 0.0001, + "loss": 64.6219, + "step": 15370 + }, + { + "epoch": 2.483478331079308, + "grad_norm": 0.7800308465957642, + "learning_rate": 0.0001, + "loss": 66.6917, + "step": 15380 + }, + { + "epoch": 2.4850931551644093, + "grad_norm": 0.5621150732040405, + "learning_rate": 0.0001, + "loss": 66.6579, + "step": 15390 + }, + { + "epoch": 2.4867079792495104, + "grad_norm": 0.6318013668060303, + "learning_rate": 0.0001, + "loss": 64.8888, + "step": 15400 + }, + { + "epoch": 2.488322803334612, + "grad_norm": 0.522668719291687, + "learning_rate": 0.0001, + "loss": 64.6468, + "step": 15410 + }, + { + "epoch": 2.489937627419713, + "grad_norm": 0.5940443873405457, + "learning_rate": 0.0001, + "loss": 65.0119, + "step": 15420 + }, + { + "epoch": 2.4915524515048144, + "grad_norm": 0.5932700037956238, + "learning_rate": 0.0001, + "loss": 66.2562, + "step": 15430 + }, + { + "epoch": 2.4931672755899155, + "grad_norm": 0.5753180980682373, + "learning_rate": 0.0001, + "loss": 65.6223, + "step": 15440 + }, + { + "epoch": 2.4947820996750165, + "grad_norm": 1.0136449337005615, + "learning_rate": 0.0001, + "loss": 65.4779, + "step": 15450 + }, + { + "epoch": 2.496396923760118, + "grad_norm": 0.6616164445877075, + "learning_rate": 0.0001, + "loss": 65.2368, + "step": 15460 + }, + { + "epoch": 2.498011747845219, + "grad_norm": 0.584385871887207, + "learning_rate": 0.0001, + "loss": 63.5195, + "step": 15470 + }, + { + "epoch": 2.49962657193032, + "grad_norm": 0.6892259120941162, + "learning_rate": 0.0001, + "loss": 63.3364, + "step": 15480 + }, + { + "epoch": 2.5012413960154216, + "grad_norm": 0.594386637210846, + "learning_rate": 0.0001, + "loss": 65.062, + "step": 15490 + }, + { + "epoch": 2.5028562201005227, + "grad_norm": 0.6604944467544556, + "learning_rate": 0.0001, + "loss": 65.3744, + "step": 15500 + }, + { + "epoch": 2.504471044185624, + "grad_norm": 0.5402690768241882, + "learning_rate": 0.0001, + "loss": 65.0583, + "step": 15510 + }, + { + "epoch": 2.506085868270725, + "grad_norm": 0.4574640989303589, + "learning_rate": 0.0001, + "loss": 64.3828, + "step": 15520 + }, + { + "epoch": 2.5077006923558267, + "grad_norm": 0.62662672996521, + "learning_rate": 0.0001, + "loss": 62.774, + "step": 15530 + }, + { + "epoch": 2.5093155164409278, + "grad_norm": 1.1336137056350708, + "learning_rate": 0.0001, + "loss": 63.9481, + "step": 15540 + }, + { + "epoch": 2.510930340526029, + "grad_norm": 0.7171993255615234, + "learning_rate": 0.0001, + "loss": 63.2949, + "step": 15550 + }, + { + "epoch": 2.5125451646111303, + "grad_norm": 0.6186808943748474, + "learning_rate": 0.0001, + "loss": 65.3805, + "step": 15560 + }, + { + "epoch": 2.5141599886962314, + "grad_norm": 0.5631883144378662, + "learning_rate": 0.0001, + "loss": 65.9464, + "step": 15570 + }, + { + "epoch": 2.5157748127813324, + "grad_norm": 0.6015043258666992, + "learning_rate": 0.0001, + "loss": 66.5938, + "step": 15580 + }, + { + "epoch": 2.517389636866434, + "grad_norm": 0.63025963306427, + "learning_rate": 0.0001, + "loss": 65.6977, + "step": 15590 + }, + { + "epoch": 2.519004460951535, + "grad_norm": 0.6715611815452576, + "learning_rate": 0.0001, + "loss": 66.1286, + "step": 15600 + }, + { + "epoch": 2.5206192850366365, + "grad_norm": 0.6191041469573975, + "learning_rate": 0.0001, + "loss": 64.257, + "step": 15610 + }, + { + "epoch": 2.5222341091217375, + "grad_norm": 0.5771703720092773, + "learning_rate": 0.0001, + "loss": 64.1293, + "step": 15620 + }, + { + "epoch": 2.523848933206839, + "grad_norm": 0.6421854496002197, + "learning_rate": 0.0001, + "loss": 64.27, + "step": 15630 + }, + { + "epoch": 2.52546375729194, + "grad_norm": 0.5963718295097351, + "learning_rate": 0.0001, + "loss": 66.5889, + "step": 15640 + }, + { + "epoch": 2.527078581377041, + "grad_norm": 0.5750302672386169, + "learning_rate": 0.0001, + "loss": 64.4224, + "step": 15650 + }, + { + "epoch": 2.5286934054621426, + "grad_norm": 0.558082640171051, + "learning_rate": 0.0001, + "loss": 64.3853, + "step": 15660 + }, + { + "epoch": 2.5303082295472437, + "grad_norm": 0.6801859736442566, + "learning_rate": 0.0001, + "loss": 65.5297, + "step": 15670 + }, + { + "epoch": 2.5319230536323447, + "grad_norm": 0.7200396656990051, + "learning_rate": 0.0001, + "loss": 66.7467, + "step": 15680 + }, + { + "epoch": 2.533537877717446, + "grad_norm": 0.5736697316169739, + "learning_rate": 0.0001, + "loss": 62.2177, + "step": 15690 + }, + { + "epoch": 2.5351527018025473, + "grad_norm": 1.6533524990081787, + "learning_rate": 0.0001, + "loss": 65.245, + "step": 15700 + }, + { + "epoch": 2.5367675258876488, + "grad_norm": 0.578233003616333, + "learning_rate": 0.0001, + "loss": 67.0796, + "step": 15710 + }, + { + "epoch": 2.53838234997275, + "grad_norm": 0.5405007600784302, + "learning_rate": 0.0001, + "loss": 65.4651, + "step": 15720 + }, + { + "epoch": 2.5399971740578513, + "grad_norm": 0.65571528673172, + "learning_rate": 0.0001, + "loss": 65.8385, + "step": 15730 + }, + { + "epoch": 2.5416119981429524, + "grad_norm": 0.5576966404914856, + "learning_rate": 0.0001, + "loss": 64.3478, + "step": 15740 + }, + { + "epoch": 2.5432268222280534, + "grad_norm": 0.7222445011138916, + "learning_rate": 0.0001, + "loss": 65.8258, + "step": 15750 + }, + { + "epoch": 2.544841646313155, + "grad_norm": 0.6464501619338989, + "learning_rate": 0.0001, + "loss": 63.535, + "step": 15760 + }, + { + "epoch": 2.546456470398256, + "grad_norm": 0.5374952554702759, + "learning_rate": 0.0001, + "loss": 63.7492, + "step": 15770 + }, + { + "epoch": 2.548071294483357, + "grad_norm": 0.6572895646095276, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 15780 + }, + { + "epoch": 2.5496861185684585, + "grad_norm": 0.578153133392334, + "learning_rate": 0.0001, + "loss": 64.311, + "step": 15790 + }, + { + "epoch": 2.5513009426535596, + "grad_norm": 0.6136942505836487, + "learning_rate": 0.0001, + "loss": 64.6734, + "step": 15800 + }, + { + "epoch": 2.552915766738661, + "grad_norm": 0.8686643242835999, + "learning_rate": 0.0001, + "loss": 64.5421, + "step": 15810 + }, + { + "epoch": 2.554530590823762, + "grad_norm": 0.8858036994934082, + "learning_rate": 0.0001, + "loss": 66.358, + "step": 15820 + }, + { + "epoch": 2.5561454149088636, + "grad_norm": 0.6373804807662964, + "learning_rate": 0.0001, + "loss": 63.7582, + "step": 15830 + }, + { + "epoch": 2.5577602389939647, + "grad_norm": 1.5556457042694092, + "learning_rate": 0.0001, + "loss": 63.8109, + "step": 15840 + }, + { + "epoch": 2.5593750630790657, + "grad_norm": 0.5945340394973755, + "learning_rate": 0.0001, + "loss": 64.7864, + "step": 15850 + }, + { + "epoch": 2.560989887164167, + "grad_norm": 0.6172908544540405, + "learning_rate": 0.0001, + "loss": 62.7738, + "step": 15860 + }, + { + "epoch": 2.5626047112492683, + "grad_norm": 0.6360666155815125, + "learning_rate": 0.0001, + "loss": 63.4814, + "step": 15870 + }, + { + "epoch": 2.5642195353343693, + "grad_norm": 0.7986658811569214, + "learning_rate": 0.0001, + "loss": 64.3178, + "step": 15880 + }, + { + "epoch": 2.565834359419471, + "grad_norm": 0.5716906785964966, + "learning_rate": 0.0001, + "loss": 64.371, + "step": 15890 + }, + { + "epoch": 2.567449183504572, + "grad_norm": 0.5859067440032959, + "learning_rate": 0.0001, + "loss": 64.0359, + "step": 15900 + }, + { + "epoch": 2.5690640075896733, + "grad_norm": 0.6288742423057556, + "learning_rate": 0.0001, + "loss": 64.4613, + "step": 15910 + }, + { + "epoch": 2.5706788316747744, + "grad_norm": 0.9052578210830688, + "learning_rate": 0.0001, + "loss": 66.3599, + "step": 15920 + }, + { + "epoch": 2.572293655759876, + "grad_norm": 0.5532401204109192, + "learning_rate": 0.0001, + "loss": 64.1753, + "step": 15930 + }, + { + "epoch": 2.573908479844977, + "grad_norm": 0.5749264359474182, + "learning_rate": 0.0001, + "loss": 63.4118, + "step": 15940 + }, + { + "epoch": 2.575523303930078, + "grad_norm": 2.064448356628418, + "learning_rate": 0.0001, + "loss": 63.2922, + "step": 15950 + }, + { + "epoch": 2.5771381280151795, + "grad_norm": 0.6260086297988892, + "learning_rate": 0.0001, + "loss": 65.2183, + "step": 15960 + }, + { + "epoch": 2.5787529521002805, + "grad_norm": 0.5910785794258118, + "learning_rate": 0.0001, + "loss": 63.3134, + "step": 15970 + }, + { + "epoch": 2.5803677761853816, + "grad_norm": 0.5972382426261902, + "learning_rate": 0.0001, + "loss": 64.3669, + "step": 15980 + }, + { + "epoch": 2.581982600270483, + "grad_norm": 0.5171772837638855, + "learning_rate": 0.0001, + "loss": 64.3819, + "step": 15990 + }, + { + "epoch": 2.583597424355584, + "grad_norm": 0.5792955160140991, + "learning_rate": 0.0001, + "loss": 65.8632, + "step": 16000 + }, + { + "epoch": 2.5852122484406856, + "grad_norm": 0.6603825092315674, + "learning_rate": 0.0001, + "loss": 63.7743, + "step": 16010 + }, + { + "epoch": 2.5868270725257867, + "grad_norm": 0.6616955399513245, + "learning_rate": 0.0001, + "loss": 64.5524, + "step": 16020 + }, + { + "epoch": 2.588441896610888, + "grad_norm": 0.6547766327857971, + "learning_rate": 0.0001, + "loss": 63.9554, + "step": 16030 + }, + { + "epoch": 2.5900567206959892, + "grad_norm": 0.6487435698509216, + "learning_rate": 0.0001, + "loss": 65.8675, + "step": 16040 + }, + { + "epoch": 2.5916715447810903, + "grad_norm": 0.7175359129905701, + "learning_rate": 0.0001, + "loss": 64.2535, + "step": 16050 + }, + { + "epoch": 2.593286368866192, + "grad_norm": 0.6271935701370239, + "learning_rate": 0.0001, + "loss": 65.0441, + "step": 16060 + }, + { + "epoch": 2.594901192951293, + "grad_norm": 0.5246796607971191, + "learning_rate": 0.0001, + "loss": 62.7952, + "step": 16070 + }, + { + "epoch": 2.596516017036394, + "grad_norm": 0.5847776532173157, + "learning_rate": 0.0001, + "loss": 65.7221, + "step": 16080 + }, + { + "epoch": 2.5981308411214954, + "grad_norm": 0.6071584224700928, + "learning_rate": 0.0001, + "loss": 65.3868, + "step": 16090 + }, + { + "epoch": 2.5997456652065964, + "grad_norm": 0.8454984426498413, + "learning_rate": 0.0001, + "loss": 62.8597, + "step": 16100 + }, + { + "epoch": 2.601360489291698, + "grad_norm": 0.7428585290908813, + "learning_rate": 0.0001, + "loss": 66.3603, + "step": 16110 + }, + { + "epoch": 2.602975313376799, + "grad_norm": 0.586359977722168, + "learning_rate": 0.0001, + "loss": 64.1264, + "step": 16120 + }, + { + "epoch": 2.6045901374619005, + "grad_norm": 0.6131327152252197, + "learning_rate": 0.0001, + "loss": 65.4991, + "step": 16130 + }, + { + "epoch": 2.6062049615470015, + "grad_norm": 0.5977164506912231, + "learning_rate": 0.0001, + "loss": 64.4486, + "step": 16140 + }, + { + "epoch": 2.6078197856321026, + "grad_norm": 0.5891970992088318, + "learning_rate": 0.0001, + "loss": 63.7493, + "step": 16150 + }, + { + "epoch": 2.609434609717204, + "grad_norm": 0.5650322437286377, + "learning_rate": 0.0001, + "loss": 63.5063, + "step": 16160 + }, + { + "epoch": 2.611049433802305, + "grad_norm": 0.6460058689117432, + "learning_rate": 0.0001, + "loss": 63.4877, + "step": 16170 + }, + { + "epoch": 2.612664257887406, + "grad_norm": 1.0753989219665527, + "learning_rate": 0.0001, + "loss": 63.8523, + "step": 16180 + }, + { + "epoch": 2.6142790819725077, + "grad_norm": 0.5751987099647522, + "learning_rate": 0.0001, + "loss": 66.5687, + "step": 16190 + }, + { + "epoch": 2.6158939060576087, + "grad_norm": 0.7001827955245972, + "learning_rate": 0.0001, + "loss": 65.8639, + "step": 16200 + }, + { + "epoch": 2.6175087301427102, + "grad_norm": 0.6699106693267822, + "learning_rate": 0.0001, + "loss": 64.5044, + "step": 16210 + }, + { + "epoch": 2.6191235542278113, + "grad_norm": 0.5610349774360657, + "learning_rate": 0.0001, + "loss": 64.5379, + "step": 16220 + }, + { + "epoch": 2.620738378312913, + "grad_norm": 0.5153414607048035, + "learning_rate": 0.0001, + "loss": 66.7831, + "step": 16230 + }, + { + "epoch": 2.622353202398014, + "grad_norm": 0.5742719769477844, + "learning_rate": 0.0001, + "loss": 64.6758, + "step": 16240 + }, + { + "epoch": 2.623968026483115, + "grad_norm": 0.5706104636192322, + "learning_rate": 0.0001, + "loss": 65.3165, + "step": 16250 + }, + { + "epoch": 2.6255828505682164, + "grad_norm": 0.6260847449302673, + "learning_rate": 0.0001, + "loss": 64.3578, + "step": 16260 + }, + { + "epoch": 2.6271976746533174, + "grad_norm": 0.5445175766944885, + "learning_rate": 0.0001, + "loss": 63.3265, + "step": 16270 + }, + { + "epoch": 2.6288124987384185, + "grad_norm": 0.7492052316665649, + "learning_rate": 0.0001, + "loss": 62.7638, + "step": 16280 + }, + { + "epoch": 2.63042732282352, + "grad_norm": 0.6429327130317688, + "learning_rate": 0.0001, + "loss": 64.2921, + "step": 16290 + }, + { + "epoch": 2.632042146908621, + "grad_norm": 0.6327224969863892, + "learning_rate": 0.0001, + "loss": 65.3948, + "step": 16300 + }, + { + "epoch": 2.6336569709937225, + "grad_norm": 0.5778248906135559, + "learning_rate": 0.0001, + "loss": 63.8617, + "step": 16310 + }, + { + "epoch": 2.6352717950788236, + "grad_norm": 0.5628714561462402, + "learning_rate": 0.0001, + "loss": 62.5165, + "step": 16320 + }, + { + "epoch": 2.636886619163925, + "grad_norm": 0.6409966945648193, + "learning_rate": 0.0001, + "loss": 64.2843, + "step": 16330 + }, + { + "epoch": 2.638501443249026, + "grad_norm": 0.7780113816261292, + "learning_rate": 0.0001, + "loss": 64.7801, + "step": 16340 + }, + { + "epoch": 2.640116267334127, + "grad_norm": 0.5766817331314087, + "learning_rate": 0.0001, + "loss": 65.0183, + "step": 16350 + }, + { + "epoch": 2.6417310914192287, + "grad_norm": 0.6962740421295166, + "learning_rate": 0.0001, + "loss": 65.9575, + "step": 16360 + }, + { + "epoch": 2.6433459155043297, + "grad_norm": 0.5021215081214905, + "learning_rate": 0.0001, + "loss": 66.8226, + "step": 16370 + }, + { + "epoch": 2.644960739589431, + "grad_norm": 0.6883698105812073, + "learning_rate": 0.0001, + "loss": 63.6297, + "step": 16380 + }, + { + "epoch": 2.6465755636745323, + "grad_norm": 0.7089389562606812, + "learning_rate": 0.0001, + "loss": 65.1165, + "step": 16390 + }, + { + "epoch": 2.6481903877596333, + "grad_norm": 0.5431233644485474, + "learning_rate": 0.0001, + "loss": 63.7242, + "step": 16400 + }, + { + "epoch": 2.6498052118447344, + "grad_norm": 0.6602722406387329, + "learning_rate": 0.0001, + "loss": 64.497, + "step": 16410 + }, + { + "epoch": 2.651420035929836, + "grad_norm": 0.5621533393859863, + "learning_rate": 0.0001, + "loss": 63.9001, + "step": 16420 + }, + { + "epoch": 2.6530348600149374, + "grad_norm": 0.612298846244812, + "learning_rate": 0.0001, + "loss": 65.2207, + "step": 16430 + }, + { + "epoch": 2.6546496841000384, + "grad_norm": 0.6222931146621704, + "learning_rate": 0.0001, + "loss": 64.5573, + "step": 16440 + }, + { + "epoch": 2.6562645081851395, + "grad_norm": 0.5502400994300842, + "learning_rate": 0.0001, + "loss": 63.9544, + "step": 16450 + }, + { + "epoch": 2.657879332270241, + "grad_norm": 0.5200631022453308, + "learning_rate": 0.0001, + "loss": 62.4098, + "step": 16460 + }, + { + "epoch": 2.659494156355342, + "grad_norm": 0.6516156196594238, + "learning_rate": 0.0001, + "loss": 63.4309, + "step": 16470 + }, + { + "epoch": 2.661108980440443, + "grad_norm": 0.5822672247886658, + "learning_rate": 0.0001, + "loss": 64.9704, + "step": 16480 + }, + { + "epoch": 2.6627238045255446, + "grad_norm": 0.7026363611221313, + "learning_rate": 0.0001, + "loss": 64.8368, + "step": 16490 + }, + { + "epoch": 2.6643386286106456, + "grad_norm": 0.5978866219520569, + "learning_rate": 0.0001, + "loss": 63.8391, + "step": 16500 + }, + { + "epoch": 2.6659534526957467, + "grad_norm": 0.6636821627616882, + "learning_rate": 0.0001, + "loss": 64.5218, + "step": 16510 + }, + { + "epoch": 2.667568276780848, + "grad_norm": 0.8163989782333374, + "learning_rate": 0.0001, + "loss": 64.7803, + "step": 16520 + }, + { + "epoch": 2.6691831008659497, + "grad_norm": 0.6390615701675415, + "learning_rate": 0.0001, + "loss": 65.8387, + "step": 16530 + }, + { + "epoch": 2.6707979249510507, + "grad_norm": 0.714682400226593, + "learning_rate": 0.0001, + "loss": 65.3721, + "step": 16540 + }, + { + "epoch": 2.672412749036152, + "grad_norm": 0.6131526827812195, + "learning_rate": 0.0001, + "loss": 65.3495, + "step": 16550 + }, + { + "epoch": 2.6740275731212533, + "grad_norm": 0.5448771715164185, + "learning_rate": 0.0001, + "loss": 63.439, + "step": 16560 + }, + { + "epoch": 2.6756423972063543, + "grad_norm": 0.6785935163497925, + "learning_rate": 0.0001, + "loss": 65.5655, + "step": 16570 + }, + { + "epoch": 2.6772572212914554, + "grad_norm": 0.6915177702903748, + "learning_rate": 0.0001, + "loss": 62.5622, + "step": 16580 + }, + { + "epoch": 2.678872045376557, + "grad_norm": 0.5327939987182617, + "learning_rate": 0.0001, + "loss": 62.3226, + "step": 16590 + }, + { + "epoch": 2.680486869461658, + "grad_norm": 0.6650441884994507, + "learning_rate": 0.0001, + "loss": 66.2329, + "step": 16600 + }, + { + "epoch": 2.682101693546759, + "grad_norm": 0.7371799945831299, + "learning_rate": 0.0001, + "loss": 64.45, + "step": 16610 + }, + { + "epoch": 2.6837165176318605, + "grad_norm": 0.6186871528625488, + "learning_rate": 0.0001, + "loss": 64.1364, + "step": 16620 + }, + { + "epoch": 2.685331341716962, + "grad_norm": 0.6977059245109558, + "learning_rate": 0.0001, + "loss": 62.1233, + "step": 16630 + }, + { + "epoch": 2.686946165802063, + "grad_norm": 0.580351710319519, + "learning_rate": 0.0001, + "loss": 64.0906, + "step": 16640 + }, + { + "epoch": 2.688560989887164, + "grad_norm": 0.6118156313896179, + "learning_rate": 0.0001, + "loss": 64.2748, + "step": 16650 + }, + { + "epoch": 2.6901758139722656, + "grad_norm": 0.5368832945823669, + "learning_rate": 0.0001, + "loss": 63.7136, + "step": 16660 + }, + { + "epoch": 2.6917906380573666, + "grad_norm": 0.6193193793296814, + "learning_rate": 0.0001, + "loss": 64.2116, + "step": 16670 + }, + { + "epoch": 2.6934054621424677, + "grad_norm": 0.6088812947273254, + "learning_rate": 0.0001, + "loss": 64.1221, + "step": 16680 + }, + { + "epoch": 2.695020286227569, + "grad_norm": 0.6514917612075806, + "learning_rate": 0.0001, + "loss": 63.764, + "step": 16690 + }, + { + "epoch": 2.6966351103126702, + "grad_norm": 0.5403677821159363, + "learning_rate": 0.0001, + "loss": 64.7421, + "step": 16700 + }, + { + "epoch": 2.6982499343977713, + "grad_norm": 0.7827529311180115, + "learning_rate": 0.0001, + "loss": 63.5076, + "step": 16710 + }, + { + "epoch": 2.6998647584828728, + "grad_norm": 0.6968997716903687, + "learning_rate": 0.0001, + "loss": 64.4448, + "step": 16720 + }, + { + "epoch": 2.7014795825679743, + "grad_norm": 0.6300367116928101, + "learning_rate": 0.0001, + "loss": 64.0245, + "step": 16730 + }, + { + "epoch": 2.7030944066530753, + "grad_norm": 0.6817230582237244, + "learning_rate": 0.0001, + "loss": 64.2338, + "step": 16740 + }, + { + "epoch": 2.7047092307381764, + "grad_norm": 0.6073988080024719, + "learning_rate": 0.0001, + "loss": 64.748, + "step": 16750 + }, + { + "epoch": 2.706324054823278, + "grad_norm": 0.5220157504081726, + "learning_rate": 0.0001, + "loss": 65.8302, + "step": 16760 + }, + { + "epoch": 2.707938878908379, + "grad_norm": 0.536228597164154, + "learning_rate": 0.0001, + "loss": 63.4033, + "step": 16770 + }, + { + "epoch": 2.70955370299348, + "grad_norm": 0.7102165818214417, + "learning_rate": 0.0001, + "loss": 64.862, + "step": 16780 + }, + { + "epoch": 2.7111685270785815, + "grad_norm": 0.6942351460456848, + "learning_rate": 0.0001, + "loss": 63.8927, + "step": 16790 + }, + { + "epoch": 2.7127833511636825, + "grad_norm": 0.7046493887901306, + "learning_rate": 0.0001, + "loss": 63.8942, + "step": 16800 + }, + { + "epoch": 2.7143981752487836, + "grad_norm": 0.5580130815505981, + "learning_rate": 0.0001, + "loss": 64.369, + "step": 16810 + }, + { + "epoch": 2.716012999333885, + "grad_norm": 0.7228151559829712, + "learning_rate": 0.0001, + "loss": 64.4892, + "step": 16820 + }, + { + "epoch": 2.7176278234189866, + "grad_norm": 0.5859993100166321, + "learning_rate": 0.0001, + "loss": 64.895, + "step": 16830 + }, + { + "epoch": 2.7192426475040876, + "grad_norm": 0.5286171436309814, + "learning_rate": 0.0001, + "loss": 64.4283, + "step": 16840 + }, + { + "epoch": 2.7208574715891887, + "grad_norm": 0.5749659538269043, + "learning_rate": 0.0001, + "loss": 65.7189, + "step": 16850 + }, + { + "epoch": 2.72247229567429, + "grad_norm": 0.8945223689079285, + "learning_rate": 0.0001, + "loss": 64.7374, + "step": 16860 + }, + { + "epoch": 2.724087119759391, + "grad_norm": 0.5596129298210144, + "learning_rate": 0.0001, + "loss": 64.7539, + "step": 16870 + }, + { + "epoch": 2.7257019438444923, + "grad_norm": 0.6479132771492004, + "learning_rate": 0.0001, + "loss": 63.6447, + "step": 16880 + }, + { + "epoch": 2.7273167679295938, + "grad_norm": 0.8423380255699158, + "learning_rate": 0.0001, + "loss": 63.7605, + "step": 16890 + }, + { + "epoch": 2.728931592014695, + "grad_norm": 1.0722684860229492, + "learning_rate": 0.0001, + "loss": 65.054, + "step": 16900 + }, + { + "epoch": 2.730546416099796, + "grad_norm": 0.5196233987808228, + "learning_rate": 0.0001, + "loss": 64.7511, + "step": 16910 + }, + { + "epoch": 2.7321612401848974, + "grad_norm": 0.7080458998680115, + "learning_rate": 0.0001, + "loss": 65.1224, + "step": 16920 + }, + { + "epoch": 2.733776064269999, + "grad_norm": 0.5816940665245056, + "learning_rate": 0.0001, + "loss": 64.8532, + "step": 16930 + }, + { + "epoch": 2.7353908883551, + "grad_norm": 0.5616808533668518, + "learning_rate": 0.0001, + "loss": 62.4221, + "step": 16940 + }, + { + "epoch": 2.737005712440201, + "grad_norm": 0.4750169515609741, + "learning_rate": 0.0001, + "loss": 60.3851, + "step": 16950 + }, + { + "epoch": 2.7386205365253025, + "grad_norm": 0.5957995653152466, + "learning_rate": 0.0001, + "loss": 62.907, + "step": 16960 + }, + { + "epoch": 2.7402353606104035, + "grad_norm": 0.60072261095047, + "learning_rate": 0.0001, + "loss": 66.446, + "step": 16970 + }, + { + "epoch": 2.7418501846955046, + "grad_norm": 0.5808581113815308, + "learning_rate": 0.0001, + "loss": 62.8852, + "step": 16980 + }, + { + "epoch": 2.743465008780606, + "grad_norm": 0.5243962407112122, + "learning_rate": 0.0001, + "loss": 64.3737, + "step": 16990 + }, + { + "epoch": 2.745079832865707, + "grad_norm": 0.6608695387840271, + "learning_rate": 0.0001, + "loss": 65.9526, + "step": 17000 + }, + { + "epoch": 2.746694656950808, + "grad_norm": 0.5109400153160095, + "learning_rate": 0.0001, + "loss": 63.5495, + "step": 17010 + }, + { + "epoch": 2.7483094810359097, + "grad_norm": 0.5007188320159912, + "learning_rate": 0.0001, + "loss": 64.3408, + "step": 17020 + }, + { + "epoch": 2.7499243051210107, + "grad_norm": 0.7058080434799194, + "learning_rate": 0.0001, + "loss": 63.1929, + "step": 17030 + }, + { + "epoch": 2.751539129206112, + "grad_norm": 0.5310391783714294, + "learning_rate": 0.0001, + "loss": 64.9107, + "step": 17040 + }, + { + "epoch": 2.7531539532912133, + "grad_norm": 0.5303440093994141, + "learning_rate": 0.0001, + "loss": 64.9961, + "step": 17050 + }, + { + "epoch": 2.7547687773763148, + "grad_norm": 0.5952566266059875, + "learning_rate": 0.0001, + "loss": 64.7917, + "step": 17060 + }, + { + "epoch": 2.756383601461416, + "grad_norm": 0.5863012075424194, + "learning_rate": 0.0001, + "loss": 65.2902, + "step": 17070 + }, + { + "epoch": 2.757998425546517, + "grad_norm": 0.8943840265274048, + "learning_rate": 0.0001, + "loss": 64.9076, + "step": 17080 + }, + { + "epoch": 2.7596132496316184, + "grad_norm": 0.7045338153839111, + "learning_rate": 0.0001, + "loss": 63.2516, + "step": 17090 + }, + { + "epoch": 2.7612280737167194, + "grad_norm": 0.8543638586997986, + "learning_rate": 0.0001, + "loss": 64.5437, + "step": 17100 + }, + { + "epoch": 2.7628428978018205, + "grad_norm": 0.6041001081466675, + "learning_rate": 0.0001, + "loss": 64.6194, + "step": 17110 + }, + { + "epoch": 2.764457721886922, + "grad_norm": 0.7917356491088867, + "learning_rate": 0.0001, + "loss": 64.5536, + "step": 17120 + }, + { + "epoch": 2.766072545972023, + "grad_norm": 0.6502277851104736, + "learning_rate": 0.0001, + "loss": 64.603, + "step": 17130 + }, + { + "epoch": 2.7676873700571245, + "grad_norm": 0.6309056282043457, + "learning_rate": 0.0001, + "loss": 64.757, + "step": 17140 + }, + { + "epoch": 2.7693021941422256, + "grad_norm": 0.6861969828605652, + "learning_rate": 0.0001, + "loss": 64.3539, + "step": 17150 + }, + { + "epoch": 2.770917018227327, + "grad_norm": 0.7183892130851746, + "learning_rate": 0.0001, + "loss": 63.9602, + "step": 17160 + }, + { + "epoch": 2.772531842312428, + "grad_norm": 0.5584074854850769, + "learning_rate": 0.0001, + "loss": 63.5869, + "step": 17170 + }, + { + "epoch": 2.774146666397529, + "grad_norm": 0.6466682553291321, + "learning_rate": 0.0001, + "loss": 62.8788, + "step": 17180 + }, + { + "epoch": 2.7757614904826307, + "grad_norm": 0.5255836844444275, + "learning_rate": 0.0001, + "loss": 62.0409, + "step": 17190 + }, + { + "epoch": 2.7773763145677317, + "grad_norm": 0.5032522678375244, + "learning_rate": 0.0001, + "loss": 64.8646, + "step": 17200 + }, + { + "epoch": 2.7789911386528328, + "grad_norm": 0.6183506846427917, + "learning_rate": 0.0001, + "loss": 63.2399, + "step": 17210 + }, + { + "epoch": 2.7806059627379343, + "grad_norm": 0.788912296295166, + "learning_rate": 0.0001, + "loss": 63.245, + "step": 17220 + }, + { + "epoch": 2.7822207868230353, + "grad_norm": 0.6142173409461975, + "learning_rate": 0.0001, + "loss": 65.9609, + "step": 17230 + }, + { + "epoch": 2.783835610908137, + "grad_norm": 0.5913170576095581, + "learning_rate": 0.0001, + "loss": 62.1876, + "step": 17240 + }, + { + "epoch": 2.785450434993238, + "grad_norm": 0.6448681354522705, + "learning_rate": 0.0001, + "loss": 63.1455, + "step": 17250 + }, + { + "epoch": 2.7870652590783394, + "grad_norm": 0.5871738195419312, + "learning_rate": 0.0001, + "loss": 63.6158, + "step": 17260 + }, + { + "epoch": 2.7886800831634404, + "grad_norm": 0.5945441722869873, + "learning_rate": 0.0001, + "loss": 63.7733, + "step": 17270 + }, + { + "epoch": 2.7902949072485415, + "grad_norm": 0.5735594630241394, + "learning_rate": 0.0001, + "loss": 64.6803, + "step": 17280 + }, + { + "epoch": 2.791909731333643, + "grad_norm": 0.6030424237251282, + "learning_rate": 0.0001, + "loss": 64.5242, + "step": 17290 + }, + { + "epoch": 2.793524555418744, + "grad_norm": 0.6867285966873169, + "learning_rate": 0.0001, + "loss": 64.8689, + "step": 17300 + }, + { + "epoch": 2.795139379503845, + "grad_norm": 0.5623561143875122, + "learning_rate": 0.0001, + "loss": 63.6502, + "step": 17310 + }, + { + "epoch": 2.7967542035889466, + "grad_norm": 0.6703289151191711, + "learning_rate": 0.0001, + "loss": 63.3221, + "step": 17320 + }, + { + "epoch": 2.7983690276740476, + "grad_norm": 0.6562789082527161, + "learning_rate": 0.0001, + "loss": 63.9181, + "step": 17330 + }, + { + "epoch": 2.799983851759149, + "grad_norm": 0.729101836681366, + "learning_rate": 0.0001, + "loss": 65.6384, + "step": 17340 + }, + { + "epoch": 2.80159867584425, + "grad_norm": 0.6310204267501831, + "learning_rate": 0.0001, + "loss": 63.5784, + "step": 17350 + }, + { + "epoch": 2.8032134999293516, + "grad_norm": 0.6090773344039917, + "learning_rate": 0.0001, + "loss": 64.3578, + "step": 17360 + }, + { + "epoch": 2.8048283240144527, + "grad_norm": 0.6332791447639465, + "learning_rate": 0.0001, + "loss": 66.4391, + "step": 17370 + }, + { + "epoch": 2.8064431480995538, + "grad_norm": 0.6627421379089355, + "learning_rate": 0.0001, + "loss": 65.8978, + "step": 17380 + }, + { + "epoch": 2.8080579721846552, + "grad_norm": 0.4875560402870178, + "learning_rate": 0.0001, + "loss": 64.1927, + "step": 17390 + }, + { + "epoch": 2.8096727962697563, + "grad_norm": 0.5367199778556824, + "learning_rate": 0.0001, + "loss": 65.9923, + "step": 17400 + }, + { + "epoch": 2.8112876203548574, + "grad_norm": 0.7587525844573975, + "learning_rate": 0.0001, + "loss": 64.5313, + "step": 17410 + }, + { + "epoch": 2.812902444439959, + "grad_norm": 0.5619229674339294, + "learning_rate": 0.0001, + "loss": 65.6483, + "step": 17420 + }, + { + "epoch": 2.81451726852506, + "grad_norm": 0.6196359992027283, + "learning_rate": 0.0001, + "loss": 63.1428, + "step": 17430 + }, + { + "epoch": 2.8161320926101614, + "grad_norm": 0.7516114115715027, + "learning_rate": 0.0001, + "loss": 63.6484, + "step": 17440 + }, + { + "epoch": 2.8177469166952624, + "grad_norm": 0.5730687379837036, + "learning_rate": 0.0001, + "loss": 63.0208, + "step": 17450 + }, + { + "epoch": 2.819361740780364, + "grad_norm": 0.5426228046417236, + "learning_rate": 0.0001, + "loss": 64.9543, + "step": 17460 + }, + { + "epoch": 2.820976564865465, + "grad_norm": 0.6501995921134949, + "learning_rate": 0.0001, + "loss": 64.8474, + "step": 17470 + }, + { + "epoch": 2.822591388950566, + "grad_norm": 0.7911072373390198, + "learning_rate": 0.0001, + "loss": 64.6111, + "step": 17480 + }, + { + "epoch": 2.8242062130356675, + "grad_norm": 0.5159533023834229, + "learning_rate": 0.0001, + "loss": 64.7333, + "step": 17490 + }, + { + "epoch": 2.8258210371207686, + "grad_norm": 0.6600466966629028, + "learning_rate": 0.0001, + "loss": 63.9058, + "step": 17500 + }, + { + "epoch": 2.8274358612058696, + "grad_norm": 0.9110124707221985, + "learning_rate": 0.0001, + "loss": 62.7915, + "step": 17510 + }, + { + "epoch": 2.829050685290971, + "grad_norm": 0.6339550018310547, + "learning_rate": 0.0001, + "loss": 62.6357, + "step": 17520 + }, + { + "epoch": 2.830665509376072, + "grad_norm": 0.5881263613700867, + "learning_rate": 0.0001, + "loss": 65.1601, + "step": 17530 + }, + { + "epoch": 2.8322803334611737, + "grad_norm": 0.5641292333602905, + "learning_rate": 0.0001, + "loss": 64.0657, + "step": 17540 + }, + { + "epoch": 2.8338951575462747, + "grad_norm": 0.5120996236801147, + "learning_rate": 0.0001, + "loss": 65.4236, + "step": 17550 + }, + { + "epoch": 2.8355099816313762, + "grad_norm": 0.5632758736610413, + "learning_rate": 0.0001, + "loss": 63.6252, + "step": 17560 + }, + { + "epoch": 2.8371248057164773, + "grad_norm": 0.507538378238678, + "learning_rate": 0.0001, + "loss": 63.9376, + "step": 17570 + }, + { + "epoch": 2.8387396298015783, + "grad_norm": 0.6714527010917664, + "learning_rate": 0.0001, + "loss": 63.7478, + "step": 17580 + }, + { + "epoch": 2.84035445388668, + "grad_norm": 0.5168405175209045, + "learning_rate": 0.0001, + "loss": 63.2652, + "step": 17590 + }, + { + "epoch": 2.841969277971781, + "grad_norm": 0.597294270992279, + "learning_rate": 0.0001, + "loss": 61.7012, + "step": 17600 + }, + { + "epoch": 2.843584102056882, + "grad_norm": 0.5981683135032654, + "learning_rate": 0.0001, + "loss": 63.5361, + "step": 17610 + }, + { + "epoch": 2.8451989261419834, + "grad_norm": 1.3800987005233765, + "learning_rate": 0.0001, + "loss": 64.9525, + "step": 17620 + }, + { + "epoch": 2.8468137502270845, + "grad_norm": 0.7254458069801331, + "learning_rate": 0.0001, + "loss": 65.9079, + "step": 17630 + }, + { + "epoch": 2.848428574312186, + "grad_norm": 0.7237294316291809, + "learning_rate": 0.0001, + "loss": 63.8467, + "step": 17640 + }, + { + "epoch": 2.850043398397287, + "grad_norm": 0.6137713193893433, + "learning_rate": 0.0001, + "loss": 64.088, + "step": 17650 + }, + { + "epoch": 2.8516582224823885, + "grad_norm": 0.5131928324699402, + "learning_rate": 0.0001, + "loss": 66.4916, + "step": 17660 + }, + { + "epoch": 2.8532730465674896, + "grad_norm": 0.6695879697799683, + "learning_rate": 0.0001, + "loss": 64.9182, + "step": 17670 + }, + { + "epoch": 2.8548878706525906, + "grad_norm": 0.9055695533752441, + "learning_rate": 0.0001, + "loss": 64.84, + "step": 17680 + }, + { + "epoch": 2.856502694737692, + "grad_norm": 0.9126661419868469, + "learning_rate": 0.0001, + "loss": 63.657, + "step": 17690 + }, + { + "epoch": 2.858117518822793, + "grad_norm": 0.8321142792701721, + "learning_rate": 0.0001, + "loss": 62.86, + "step": 17700 + }, + { + "epoch": 2.8597323429078942, + "grad_norm": 0.6571202278137207, + "learning_rate": 0.0001, + "loss": 66.1002, + "step": 17710 + }, + { + "epoch": 2.8613471669929957, + "grad_norm": 0.7111368775367737, + "learning_rate": 0.0001, + "loss": 64.3513, + "step": 17720 + }, + { + "epoch": 2.862961991078097, + "grad_norm": 0.5026522874832153, + "learning_rate": 0.0001, + "loss": 61.8481, + "step": 17730 + }, + { + "epoch": 2.8645768151631983, + "grad_norm": 0.5820951461791992, + "learning_rate": 0.0001, + "loss": 65.2891, + "step": 17740 + }, + { + "epoch": 2.8661916392482993, + "grad_norm": 0.5339917540550232, + "learning_rate": 0.0001, + "loss": 66.4793, + "step": 17750 + }, + { + "epoch": 2.867806463333401, + "grad_norm": 0.7611919045448303, + "learning_rate": 0.0001, + "loss": 63.3336, + "step": 17760 + }, + { + "epoch": 2.869421287418502, + "grad_norm": 0.7337769865989685, + "learning_rate": 0.0001, + "loss": 64.0139, + "step": 17770 + }, + { + "epoch": 2.871036111503603, + "grad_norm": 0.631750226020813, + "learning_rate": 0.0001, + "loss": 64.7222, + "step": 17780 + }, + { + "epoch": 2.8726509355887044, + "grad_norm": 0.6601754426956177, + "learning_rate": 0.0001, + "loss": 63.5139, + "step": 17790 + }, + { + "epoch": 2.8742657596738055, + "grad_norm": 0.567677915096283, + "learning_rate": 0.0001, + "loss": 64.9689, + "step": 17800 + }, + { + "epoch": 2.8758805837589065, + "grad_norm": 0.6223359704017639, + "learning_rate": 0.0001, + "loss": 64.0291, + "step": 17810 + }, + { + "epoch": 2.877495407844008, + "grad_norm": 0.5666190981864929, + "learning_rate": 0.0001, + "loss": 65.1292, + "step": 17820 + }, + { + "epoch": 2.879110231929109, + "grad_norm": 0.5870898365974426, + "learning_rate": 0.0001, + "loss": 62.9442, + "step": 17830 + }, + { + "epoch": 2.8807250560142106, + "grad_norm": 0.7579465508460999, + "learning_rate": 0.0001, + "loss": 63.997, + "step": 17840 + }, + { + "epoch": 2.8823398800993116, + "grad_norm": 0.674678385257721, + "learning_rate": 0.0001, + "loss": 62.7068, + "step": 17850 + }, + { + "epoch": 2.883954704184413, + "grad_norm": 0.7183076739311218, + "learning_rate": 0.0001, + "loss": 63.2744, + "step": 17860 + }, + { + "epoch": 2.885569528269514, + "grad_norm": 0.8441048264503479, + "learning_rate": 0.0001, + "loss": 64.587, + "step": 17870 + }, + { + "epoch": 2.8871843523546152, + "grad_norm": 0.5780750513076782, + "learning_rate": 0.0001, + "loss": 63.2929, + "step": 17880 + }, + { + "epoch": 2.8887991764397167, + "grad_norm": 0.5931091904640198, + "learning_rate": 0.0001, + "loss": 63.0084, + "step": 17890 + }, + { + "epoch": 2.890414000524818, + "grad_norm": 0.5232470631599426, + "learning_rate": 0.0001, + "loss": 64.4947, + "step": 17900 + }, + { + "epoch": 2.892028824609919, + "grad_norm": 0.5742111206054688, + "learning_rate": 0.0001, + "loss": 63.544, + "step": 17910 + }, + { + "epoch": 2.8936436486950203, + "grad_norm": 0.531182587146759, + "learning_rate": 0.0001, + "loss": 64.5583, + "step": 17920 + }, + { + "epoch": 2.8952584727801214, + "grad_norm": 0.5353103876113892, + "learning_rate": 0.0001, + "loss": 61.9807, + "step": 17930 + }, + { + "epoch": 2.896873296865223, + "grad_norm": 0.53264981508255, + "learning_rate": 0.0001, + "loss": 62.3831, + "step": 17940 + }, + { + "epoch": 2.898488120950324, + "grad_norm": 0.5475999712944031, + "learning_rate": 0.0001, + "loss": 64.9383, + "step": 17950 + }, + { + "epoch": 2.9001029450354254, + "grad_norm": 0.876865804195404, + "learning_rate": 0.0001, + "loss": 64.955, + "step": 17960 + }, + { + "epoch": 2.9017177691205265, + "grad_norm": 0.5121579170227051, + "learning_rate": 0.0001, + "loss": 62.8306, + "step": 17970 + }, + { + "epoch": 2.9033325932056275, + "grad_norm": 0.6591141819953918, + "learning_rate": 0.0001, + "loss": 61.6768, + "step": 17980 + }, + { + "epoch": 2.904947417290729, + "grad_norm": 0.5192371010780334, + "learning_rate": 0.0001, + "loss": 63.5609, + "step": 17990 + }, + { + "epoch": 2.90656224137583, + "grad_norm": 0.5635988116264343, + "learning_rate": 0.0001, + "loss": 61.4911, + "step": 18000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.2040103027111625e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6a151f546555c78ebc502dd7b703b665f0672d6f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35235df8408202e91954a585851d27187bafaf4e1d75c4e63c51fbf721136ff5 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-18000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/latest b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/latest new file mode 100644 index 0000000000000000000000000000000000000000..2a79fdc19587e6bc9de060e90633f3a151b04516 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/latest @@ -0,0 +1 @@ +global_step2000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..56367a3acbbef4a70ff30da0ebc9973973f53a5e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e78824bc7c646eeeb3a3bdf32daea02b31d1ab00c5ca5719aa09e9fc87bfbbf +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..d0d8142a10754deb275cc17ce3b6a04b3e2f3791 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d5bb3be44ff89f6aa0da9aaf7d96160ef241c86ee22c23ad28ea99797946aa2 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..0fe285198c024c34286a28c525291dbcb452fedf --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4d14a2905705aac1bae8338343627736224ef7586935f1c1b5ea8edd35972d7 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..87ec73b27fca77edce8520cc0f4aee151145a106 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c4400a95c39b8e5bf0600670436c61a9e40a938c2b90d2fb9a74ce7f10662b9 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..ee3005335d6573cfbc966337b8d14c306a3f1b6e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fd7cfdfa98f577e3019f4e0bfd7e5af0037fbe0e445892d6a510b008b20b897 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..f64565dac20f22d0f3f2339b039b1ecd3ee02478 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aae4dd6255977dc6fe0c583507d4f3bf1aac543ffaef503607f552bab870923c +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..5a0bfd5f08ba6d0a7cb6330a286f3f26d98a0745 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:587149279e0adedd80119cc41c87b3ebe9d9bb6f3b66161ddd200680351d7bb2 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..416667f06594705625897695cb37a28c8e38bf5a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95eca9a4c43f1508a01b33a5ce298ef5fae31b3317563d25ef5e36eefcf7da82 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..06f635f06e5e1f18789cfa9a24e438c4d00a3df1 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fce0029eaa1c4d3fa508cc11330ab6ac8b0f5ad5af2575203b09f4f920c80b90 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..059097d31013bcb853c1f29bd20164bc1deb61d8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6a4e6ea22558ae7a3d868dc31596451ac2fbd9303bffcf03fa9584c5490e3ca +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7e2241c61e6d67231db83d3ea7b022f3ff7fb4fe --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/trainer_state.json @@ -0,0 +1,1434 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.3229648170202459, + "eval_steps": 2000, + "global_step": 2000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0016148240851012292, + "grad_norm": 43.90101623535156, + "learning_rate": 0.0001, + "loss": 307.2084, + "step": 10 + }, + { + "epoch": 0.0032296481702024584, + "grad_norm": 19.106313705444336, + "learning_rate": 0.0001, + "loss": 320.2794, + "step": 20 + }, + { + "epoch": 0.004844472255303688, + "grad_norm": 5.771847248077393, + "learning_rate": 0.0001, + "loss": 247.4977, + "step": 30 + }, + { + "epoch": 0.006459296340404917, + "grad_norm": 2.630659580230713, + "learning_rate": 0.0001, + "loss": 229.9593, + "step": 40 + }, + { + "epoch": 0.008074120425506147, + "grad_norm": 2.487499475479126, + "learning_rate": 0.0001, + "loss": 228.65, + "step": 50 + }, + { + "epoch": 0.009688944510607376, + "grad_norm": 2.078263521194458, + "learning_rate": 0.0001, + "loss": 227.648, + "step": 60 + }, + { + "epoch": 0.011303768595708604, + "grad_norm": 3.105417490005493, + "learning_rate": 0.0001, + "loss": 227.9345, + "step": 70 + }, + { + "epoch": 0.012918592680809833, + "grad_norm": 2.2145848274230957, + "learning_rate": 0.0001, + "loss": 227.3662, + "step": 80 + }, + { + "epoch": 0.014533416765911064, + "grad_norm": 1.9644615650177002, + "learning_rate": 0.0001, + "loss": 226.9145, + "step": 90 + }, + { + "epoch": 0.016148240851012293, + "grad_norm": 1.867382287979126, + "learning_rate": 0.0001, + "loss": 227.5703, + "step": 100 + }, + { + "epoch": 0.017763064936113522, + "grad_norm": 1.4613710641860962, + "learning_rate": 0.0001, + "loss": 227.1107, + "step": 110 + }, + { + "epoch": 0.01937788902121475, + "grad_norm": 2.8953869342803955, + "learning_rate": 0.0001, + "loss": 226.8185, + "step": 120 + }, + { + "epoch": 0.02099271310631598, + "grad_norm": 1.8446482419967651, + "learning_rate": 0.0001, + "loss": 226.9624, + "step": 130 + }, + { + "epoch": 0.02260753719141721, + "grad_norm": 1.3048362731933594, + "learning_rate": 0.0001, + "loss": 226.8404, + "step": 140 + }, + { + "epoch": 0.024222361276518438, + "grad_norm": 1.7100074291229248, + "learning_rate": 0.0001, + "loss": 226.1071, + "step": 150 + }, + { + "epoch": 0.025837185361619667, + "grad_norm": 1.7875330448150635, + "learning_rate": 0.0001, + "loss": 226.0764, + "step": 160 + }, + { + "epoch": 0.0274520094467209, + "grad_norm": 1.7503769397735596, + "learning_rate": 0.0001, + "loss": 226.0772, + "step": 170 + }, + { + "epoch": 0.029066833531822128, + "grad_norm": 2.7083983421325684, + "learning_rate": 0.0001, + "loss": 224.7667, + "step": 180 + }, + { + "epoch": 0.030681657616923357, + "grad_norm": 1.8192371129989624, + "learning_rate": 0.0001, + "loss": 223.6494, + "step": 190 + }, + { + "epoch": 0.032296481702024586, + "grad_norm": 3.210358142852783, + "learning_rate": 0.0001, + "loss": 223.3545, + "step": 200 + }, + { + "epoch": 0.03391130578712581, + "grad_norm": 5.819669246673584, + "learning_rate": 0.0001, + "loss": 222.1732, + "step": 210 + }, + { + "epoch": 0.035526129872227044, + "grad_norm": 4.185793399810791, + "learning_rate": 0.0001, + "loss": 221.6866, + "step": 220 + }, + { + "epoch": 0.03714095395732828, + "grad_norm": 2.7514216899871826, + "learning_rate": 0.0001, + "loss": 221.0769, + "step": 230 + }, + { + "epoch": 0.0387557780424295, + "grad_norm": 1.5478988885879517, + "learning_rate": 0.0001, + "loss": 219.7232, + "step": 240 + }, + { + "epoch": 0.040370602127530734, + "grad_norm": 1.4958223104476929, + "learning_rate": 0.0001, + "loss": 218.5454, + "step": 250 + }, + { + "epoch": 0.04198542621263196, + "grad_norm": 6.36635684967041, + "learning_rate": 0.0001, + "loss": 218.5061, + "step": 260 + }, + { + "epoch": 0.04360025029773319, + "grad_norm": 3.029601812362671, + "learning_rate": 0.0001, + "loss": 216.3466, + "step": 270 + }, + { + "epoch": 0.04521507438283442, + "grad_norm": 2.6436100006103516, + "learning_rate": 0.0001, + "loss": 214.0003, + "step": 280 + }, + { + "epoch": 0.04682989846793565, + "grad_norm": 5.329655647277832, + "learning_rate": 0.0001, + "loss": 212.9169, + "step": 290 + }, + { + "epoch": 0.048444722553036876, + "grad_norm": 4.930875301361084, + "learning_rate": 0.0001, + "loss": 209.7737, + "step": 300 + }, + { + "epoch": 0.05005954663813811, + "grad_norm": 18.902585983276367, + "learning_rate": 0.0001, + "loss": 209.4267, + "step": 310 + }, + { + "epoch": 0.051674370723239334, + "grad_norm": 5.364975929260254, + "learning_rate": 0.0001, + "loss": 206.49, + "step": 320 + }, + { + "epoch": 0.053289194808340566, + "grad_norm": 3.1421146392822266, + "learning_rate": 0.0001, + "loss": 204.8132, + "step": 330 + }, + { + "epoch": 0.0549040188934418, + "grad_norm": 3.399320602416992, + "learning_rate": 0.0001, + "loss": 202.4785, + "step": 340 + }, + { + "epoch": 0.056518842978543024, + "grad_norm": 5.137377738952637, + "learning_rate": 0.0001, + "loss": 199.5143, + "step": 350 + }, + { + "epoch": 0.058133667063644257, + "grad_norm": 5.4967522621154785, + "learning_rate": 0.0001, + "loss": 196.2809, + "step": 360 + }, + { + "epoch": 0.05974849114874548, + "grad_norm": 6.435070037841797, + "learning_rate": 0.0001, + "loss": 196.8831, + "step": 370 + }, + { + "epoch": 0.061363315233846714, + "grad_norm": 5.7334370613098145, + "learning_rate": 0.0001, + "loss": 193.141, + "step": 380 + }, + { + "epoch": 0.06297813931894794, + "grad_norm": 3.6522979736328125, + "learning_rate": 0.0001, + "loss": 191.5819, + "step": 390 + }, + { + "epoch": 0.06459296340404917, + "grad_norm": 2.614051103591919, + "learning_rate": 0.0001, + "loss": 190.046, + "step": 400 + }, + { + "epoch": 0.0662077874891504, + "grad_norm": 4.845924377441406, + "learning_rate": 0.0001, + "loss": 188.7121, + "step": 410 + }, + { + "epoch": 0.06782261157425162, + "grad_norm": 4.319681644439697, + "learning_rate": 0.0001, + "loss": 185.3697, + "step": 420 + }, + { + "epoch": 0.06943743565935286, + "grad_norm": 2.7018792629241943, + "learning_rate": 0.0001, + "loss": 183.5171, + "step": 430 + }, + { + "epoch": 0.07105225974445409, + "grad_norm": 2.6407113075256348, + "learning_rate": 0.0001, + "loss": 181.245, + "step": 440 + }, + { + "epoch": 0.07266708382955532, + "grad_norm": 3.314251184463501, + "learning_rate": 0.0001, + "loss": 179.3874, + "step": 450 + }, + { + "epoch": 0.07428190791465655, + "grad_norm": 2.635136365890503, + "learning_rate": 0.0001, + "loss": 177.8257, + "step": 460 + }, + { + "epoch": 0.07589673199975777, + "grad_norm": 5.227738857269287, + "learning_rate": 0.0001, + "loss": 176.4699, + "step": 470 + }, + { + "epoch": 0.077511556084859, + "grad_norm": 4.998415470123291, + "learning_rate": 0.0001, + "loss": 175.7601, + "step": 480 + }, + { + "epoch": 0.07912638016996024, + "grad_norm": 2.946892261505127, + "learning_rate": 0.0001, + "loss": 173.936, + "step": 490 + }, + { + "epoch": 0.08074120425506147, + "grad_norm": 4.20358943939209, + "learning_rate": 0.0001, + "loss": 172.065, + "step": 500 + }, + { + "epoch": 0.08235602834016269, + "grad_norm": 3.125765562057495, + "learning_rate": 0.0001, + "loss": 171.3727, + "step": 510 + }, + { + "epoch": 0.08397085242526392, + "grad_norm": 2.4870612621307373, + "learning_rate": 0.0001, + "loss": 169.9, + "step": 520 + }, + { + "epoch": 0.08558567651036515, + "grad_norm": 3.917384147644043, + "learning_rate": 0.0001, + "loss": 168.5049, + "step": 530 + }, + { + "epoch": 0.08720050059546638, + "grad_norm": 2.302142858505249, + "learning_rate": 0.0001, + "loss": 167.2027, + "step": 540 + }, + { + "epoch": 0.08881532468056762, + "grad_norm": 4.794841289520264, + "learning_rate": 0.0001, + "loss": 166.0887, + "step": 550 + }, + { + "epoch": 0.09043014876566884, + "grad_norm": 2.200291395187378, + "learning_rate": 0.0001, + "loss": 164.5882, + "step": 560 + }, + { + "epoch": 0.09204497285077007, + "grad_norm": 1.9572436809539795, + "learning_rate": 0.0001, + "loss": 163.3007, + "step": 570 + }, + { + "epoch": 0.0936597969358713, + "grad_norm": 3.443633556365967, + "learning_rate": 0.0001, + "loss": 160.6805, + "step": 580 + }, + { + "epoch": 0.09527462102097253, + "grad_norm": 2.6079020500183105, + "learning_rate": 0.0001, + "loss": 160.3372, + "step": 590 + }, + { + "epoch": 0.09688944510607375, + "grad_norm": 2.4401192665100098, + "learning_rate": 0.0001, + "loss": 160.0956, + "step": 600 + }, + { + "epoch": 0.09850426919117498, + "grad_norm": 2.686101198196411, + "learning_rate": 0.0001, + "loss": 157.6138, + "step": 610 + }, + { + "epoch": 0.10011909327627622, + "grad_norm": 2.594151496887207, + "learning_rate": 0.0001, + "loss": 156.0726, + "step": 620 + }, + { + "epoch": 0.10173391736137745, + "grad_norm": 2.509141683578491, + "learning_rate": 0.0001, + "loss": 155.9923, + "step": 630 + }, + { + "epoch": 0.10334874144647867, + "grad_norm": 2.4077377319335938, + "learning_rate": 0.0001, + "loss": 155.214, + "step": 640 + }, + { + "epoch": 0.1049635655315799, + "grad_norm": 4.083552837371826, + "learning_rate": 0.0001, + "loss": 152.8257, + "step": 650 + }, + { + "epoch": 0.10657838961668113, + "grad_norm": 2.7188713550567627, + "learning_rate": 0.0001, + "loss": 151.875, + "step": 660 + }, + { + "epoch": 0.10819321370178236, + "grad_norm": 2.41287899017334, + "learning_rate": 0.0001, + "loss": 151.935, + "step": 670 + }, + { + "epoch": 0.1098080377868836, + "grad_norm": 3.995708465576172, + "learning_rate": 0.0001, + "loss": 151.0069, + "step": 680 + }, + { + "epoch": 0.11142286187198482, + "grad_norm": 3.217705488204956, + "learning_rate": 0.0001, + "loss": 149.5922, + "step": 690 + }, + { + "epoch": 0.11303768595708605, + "grad_norm": 3.4349842071533203, + "learning_rate": 0.0001, + "loss": 148.0044, + "step": 700 + }, + { + "epoch": 0.11465251004218728, + "grad_norm": 2.106865167617798, + "learning_rate": 0.0001, + "loss": 147.8404, + "step": 710 + }, + { + "epoch": 0.11626733412728851, + "grad_norm": 1.8729510307312012, + "learning_rate": 0.0001, + "loss": 145.1299, + "step": 720 + }, + { + "epoch": 0.11788215821238973, + "grad_norm": 4.14620304107666, + "learning_rate": 0.0001, + "loss": 144.3928, + "step": 730 + }, + { + "epoch": 0.11949698229749096, + "grad_norm": 2.202505350112915, + "learning_rate": 0.0001, + "loss": 144.8357, + "step": 740 + }, + { + "epoch": 0.1211118063825922, + "grad_norm": 2.201470375061035, + "learning_rate": 0.0001, + "loss": 143.2202, + "step": 750 + }, + { + "epoch": 0.12272663046769343, + "grad_norm": 1.5977853536605835, + "learning_rate": 0.0001, + "loss": 143.3537, + "step": 760 + }, + { + "epoch": 0.12434145455279466, + "grad_norm": 2.4403927326202393, + "learning_rate": 0.0001, + "loss": 141.2914, + "step": 770 + }, + { + "epoch": 0.12595627863789588, + "grad_norm": 1.6895846128463745, + "learning_rate": 0.0001, + "loss": 139.9639, + "step": 780 + }, + { + "epoch": 0.12757110272299713, + "grad_norm": 3.4370665550231934, + "learning_rate": 0.0001, + "loss": 140.715, + "step": 790 + }, + { + "epoch": 0.12918592680809834, + "grad_norm": 2.1449077129364014, + "learning_rate": 0.0001, + "loss": 139.2094, + "step": 800 + }, + { + "epoch": 0.13080075089319956, + "grad_norm": 2.4578912258148193, + "learning_rate": 0.0001, + "loss": 137.6724, + "step": 810 + }, + { + "epoch": 0.1324155749783008, + "grad_norm": 1.8994814157485962, + "learning_rate": 0.0001, + "loss": 136.9551, + "step": 820 + }, + { + "epoch": 0.13403039906340203, + "grad_norm": 2.6531031131744385, + "learning_rate": 0.0001, + "loss": 136.5812, + "step": 830 + }, + { + "epoch": 0.13564522314850325, + "grad_norm": 3.8680083751678467, + "learning_rate": 0.0001, + "loss": 134.3655, + "step": 840 + }, + { + "epoch": 0.1372600472336045, + "grad_norm": 1.9594208002090454, + "learning_rate": 0.0001, + "loss": 135.4958, + "step": 850 + }, + { + "epoch": 0.1388748713187057, + "grad_norm": 1.5475000143051147, + "learning_rate": 0.0001, + "loss": 135.2142, + "step": 860 + }, + { + "epoch": 0.14048969540380696, + "grad_norm": 1.6569288969039917, + "learning_rate": 0.0001, + "loss": 133.4162, + "step": 870 + }, + { + "epoch": 0.14210451948890818, + "grad_norm": 1.6724224090576172, + "learning_rate": 0.0001, + "loss": 133.7519, + "step": 880 + }, + { + "epoch": 0.1437193435740094, + "grad_norm": 1.7299500703811646, + "learning_rate": 0.0001, + "loss": 131.471, + "step": 890 + }, + { + "epoch": 0.14533416765911064, + "grad_norm": 1.9444470405578613, + "learning_rate": 0.0001, + "loss": 128.5966, + "step": 900 + }, + { + "epoch": 0.14694899174421186, + "grad_norm": 2.404723644256592, + "learning_rate": 0.0001, + "loss": 129.9824, + "step": 910 + }, + { + "epoch": 0.1485638158293131, + "grad_norm": 2.1493897438049316, + "learning_rate": 0.0001, + "loss": 128.0219, + "step": 920 + }, + { + "epoch": 0.15017863991441432, + "grad_norm": 2.8861117362976074, + "learning_rate": 0.0001, + "loss": 128.5297, + "step": 930 + }, + { + "epoch": 0.15179346399951554, + "grad_norm": 2.546415090560913, + "learning_rate": 0.0001, + "loss": 126.6817, + "step": 940 + }, + { + "epoch": 0.1534082880846168, + "grad_norm": 1.727663278579712, + "learning_rate": 0.0001, + "loss": 125.0413, + "step": 950 + }, + { + "epoch": 0.155023112169718, + "grad_norm": 3.7156920433044434, + "learning_rate": 0.0001, + "loss": 124.8085, + "step": 960 + }, + { + "epoch": 0.15663793625481925, + "grad_norm": 3.0576517581939697, + "learning_rate": 0.0001, + "loss": 125.9107, + "step": 970 + }, + { + "epoch": 0.15825276033992047, + "grad_norm": 1.9951574802398682, + "learning_rate": 0.0001, + "loss": 123.5359, + "step": 980 + }, + { + "epoch": 0.1598675844250217, + "grad_norm": 1.4620885848999023, + "learning_rate": 0.0001, + "loss": 121.9363, + "step": 990 + }, + { + "epoch": 0.16148240851012294, + "grad_norm": 1.7640169858932495, + "learning_rate": 0.0001, + "loss": 121.1764, + "step": 1000 + }, + { + "epoch": 0.16309723259522416, + "grad_norm": 2.921750545501709, + "learning_rate": 0.0001, + "loss": 119.6931, + "step": 1010 + }, + { + "epoch": 0.16471205668032537, + "grad_norm": 1.5623588562011719, + "learning_rate": 0.0001, + "loss": 119.097, + "step": 1020 + }, + { + "epoch": 0.16632688076542662, + "grad_norm": 2.2386627197265625, + "learning_rate": 0.0001, + "loss": 120.9224, + "step": 1030 + }, + { + "epoch": 0.16794170485052784, + "grad_norm": 1.3213791847229004, + "learning_rate": 0.0001, + "loss": 119.5649, + "step": 1040 + }, + { + "epoch": 0.16955652893562909, + "grad_norm": 1.5255440473556519, + "learning_rate": 0.0001, + "loss": 118.1524, + "step": 1050 + }, + { + "epoch": 0.1711713530207303, + "grad_norm": 1.6038514375686646, + "learning_rate": 0.0001, + "loss": 118.6307, + "step": 1060 + }, + { + "epoch": 0.17278617710583152, + "grad_norm": 2.356739044189453, + "learning_rate": 0.0001, + "loss": 116.8472, + "step": 1070 + }, + { + "epoch": 0.17440100119093277, + "grad_norm": 2.5326743125915527, + "learning_rate": 0.0001, + "loss": 115.6625, + "step": 1080 + }, + { + "epoch": 0.176015825276034, + "grad_norm": 1.2823396921157837, + "learning_rate": 0.0001, + "loss": 115.3312, + "step": 1090 + }, + { + "epoch": 0.17763064936113523, + "grad_norm": 1.981788158416748, + "learning_rate": 0.0001, + "loss": 116.2257, + "step": 1100 + }, + { + "epoch": 0.17924547344623645, + "grad_norm": 1.540440559387207, + "learning_rate": 0.0001, + "loss": 114.5031, + "step": 1110 + }, + { + "epoch": 0.18086029753133767, + "grad_norm": 1.7667800188064575, + "learning_rate": 0.0001, + "loss": 113.7663, + "step": 1120 + }, + { + "epoch": 0.18247512161643892, + "grad_norm": 2.2793102264404297, + "learning_rate": 0.0001, + "loss": 115.219, + "step": 1130 + }, + { + "epoch": 0.18408994570154014, + "grad_norm": 2.132046699523926, + "learning_rate": 0.0001, + "loss": 112.8482, + "step": 1140 + }, + { + "epoch": 0.18570476978664135, + "grad_norm": 1.3153988122940063, + "learning_rate": 0.0001, + "loss": 111.4546, + "step": 1150 + }, + { + "epoch": 0.1873195938717426, + "grad_norm": 1.5246292352676392, + "learning_rate": 0.0001, + "loss": 112.4348, + "step": 1160 + }, + { + "epoch": 0.18893441795684382, + "grad_norm": 1.365096092224121, + "learning_rate": 0.0001, + "loss": 109.755, + "step": 1170 + }, + { + "epoch": 0.19054924204194507, + "grad_norm": 1.2777953147888184, + "learning_rate": 0.0001, + "loss": 111.0318, + "step": 1180 + }, + { + "epoch": 0.19216406612704628, + "grad_norm": 1.8672829866409302, + "learning_rate": 0.0001, + "loss": 109.1648, + "step": 1190 + }, + { + "epoch": 0.1937788902121475, + "grad_norm": 1.709987759590149, + "learning_rate": 0.0001, + "loss": 108.1616, + "step": 1200 + }, + { + "epoch": 0.19539371429724875, + "grad_norm": 1.176277995109558, + "learning_rate": 0.0001, + "loss": 109.3591, + "step": 1210 + }, + { + "epoch": 0.19700853838234997, + "grad_norm": 1.4181180000305176, + "learning_rate": 0.0001, + "loss": 110.0216, + "step": 1220 + }, + { + "epoch": 0.19862336246745121, + "grad_norm": 1.3246116638183594, + "learning_rate": 0.0001, + "loss": 107.9769, + "step": 1230 + }, + { + "epoch": 0.20023818655255243, + "grad_norm": 1.2501498460769653, + "learning_rate": 0.0001, + "loss": 108.2601, + "step": 1240 + }, + { + "epoch": 0.20185301063765365, + "grad_norm": 1.2277404069900513, + "learning_rate": 0.0001, + "loss": 107.1258, + "step": 1250 + }, + { + "epoch": 0.2034678347227549, + "grad_norm": 1.8064894676208496, + "learning_rate": 0.0001, + "loss": 105.6278, + "step": 1260 + }, + { + "epoch": 0.20508265880785612, + "grad_norm": 1.708897352218628, + "learning_rate": 0.0001, + "loss": 106.9269, + "step": 1270 + }, + { + "epoch": 0.20669748289295733, + "grad_norm": 1.4507478475570679, + "learning_rate": 0.0001, + "loss": 106.9236, + "step": 1280 + }, + { + "epoch": 0.20831230697805858, + "grad_norm": 1.4904310703277588, + "learning_rate": 0.0001, + "loss": 106.0653, + "step": 1290 + }, + { + "epoch": 0.2099271310631598, + "grad_norm": 1.2637165784835815, + "learning_rate": 0.0001, + "loss": 104.8665, + "step": 1300 + }, + { + "epoch": 0.21154195514826105, + "grad_norm": 1.4085133075714111, + "learning_rate": 0.0001, + "loss": 104.8891, + "step": 1310 + }, + { + "epoch": 0.21315677923336226, + "grad_norm": 1.5710629224777222, + "learning_rate": 0.0001, + "loss": 104.3009, + "step": 1320 + }, + { + "epoch": 0.21477160331846348, + "grad_norm": 1.3634663820266724, + "learning_rate": 0.0001, + "loss": 104.4545, + "step": 1330 + }, + { + "epoch": 0.21638642740356473, + "grad_norm": 1.5565859079360962, + "learning_rate": 0.0001, + "loss": 103.6205, + "step": 1340 + }, + { + "epoch": 0.21800125148866595, + "grad_norm": 1.4491888284683228, + "learning_rate": 0.0001, + "loss": 104.0892, + "step": 1350 + }, + { + "epoch": 0.2196160755737672, + "grad_norm": 1.7609063386917114, + "learning_rate": 0.0001, + "loss": 104.5177, + "step": 1360 + }, + { + "epoch": 0.2212308996588684, + "grad_norm": 1.6723753213882446, + "learning_rate": 0.0001, + "loss": 104.5552, + "step": 1370 + }, + { + "epoch": 0.22284572374396963, + "grad_norm": 1.3781665563583374, + "learning_rate": 0.0001, + "loss": 102.1022, + "step": 1380 + }, + { + "epoch": 0.22446054782907088, + "grad_norm": 1.5397366285324097, + "learning_rate": 0.0001, + "loss": 102.5882, + "step": 1390 + }, + { + "epoch": 0.2260753719141721, + "grad_norm": 1.4567879438400269, + "learning_rate": 0.0001, + "loss": 101.3781, + "step": 1400 + }, + { + "epoch": 0.22769019599927334, + "grad_norm": 1.2882301807403564, + "learning_rate": 0.0001, + "loss": 100.9378, + "step": 1410 + }, + { + "epoch": 0.22930502008437456, + "grad_norm": 1.4510424137115479, + "learning_rate": 0.0001, + "loss": 101.5383, + "step": 1420 + }, + { + "epoch": 0.23091984416947578, + "grad_norm": 2.8331449031829834, + "learning_rate": 0.0001, + "loss": 100.2773, + "step": 1430 + }, + { + "epoch": 0.23253466825457703, + "grad_norm": 1.363616943359375, + "learning_rate": 0.0001, + "loss": 101.646, + "step": 1440 + }, + { + "epoch": 0.23414949233967824, + "grad_norm": 1.2184548377990723, + "learning_rate": 0.0001, + "loss": 101.1133, + "step": 1450 + }, + { + "epoch": 0.23576431642477946, + "grad_norm": 2.235656976699829, + "learning_rate": 0.0001, + "loss": 100.2044, + "step": 1460 + }, + { + "epoch": 0.2373791405098807, + "grad_norm": 1.2334418296813965, + "learning_rate": 0.0001, + "loss": 99.4031, + "step": 1470 + }, + { + "epoch": 0.23899396459498193, + "grad_norm": 1.2394684553146362, + "learning_rate": 0.0001, + "loss": 101.1247, + "step": 1480 + }, + { + "epoch": 0.24060878868008317, + "grad_norm": 1.5218186378479004, + "learning_rate": 0.0001, + "loss": 100.4413, + "step": 1490 + }, + { + "epoch": 0.2422236127651844, + "grad_norm": 1.2112364768981934, + "learning_rate": 0.0001, + "loss": 99.8722, + "step": 1500 + }, + { + "epoch": 0.2438384368502856, + "grad_norm": 1.2277610301971436, + "learning_rate": 0.0001, + "loss": 99.0276, + "step": 1510 + }, + { + "epoch": 0.24545326093538686, + "grad_norm": 1.1005053520202637, + "learning_rate": 0.0001, + "loss": 98.6764, + "step": 1520 + }, + { + "epoch": 0.24706808502048808, + "grad_norm": 1.0754456520080566, + "learning_rate": 0.0001, + "loss": 97.8968, + "step": 1530 + }, + { + "epoch": 0.24868290910558932, + "grad_norm": 1.0695950984954834, + "learning_rate": 0.0001, + "loss": 98.4032, + "step": 1540 + }, + { + "epoch": 0.25029773319069054, + "grad_norm": 1.5098627805709839, + "learning_rate": 0.0001, + "loss": 97.9391, + "step": 1550 + }, + { + "epoch": 0.25191255727579176, + "grad_norm": 1.006681203842163, + "learning_rate": 0.0001, + "loss": 96.5525, + "step": 1560 + }, + { + "epoch": 0.253527381360893, + "grad_norm": 1.3987332582473755, + "learning_rate": 0.0001, + "loss": 98.383, + "step": 1570 + }, + { + "epoch": 0.25514220544599425, + "grad_norm": 1.290887713432312, + "learning_rate": 0.0001, + "loss": 99.0901, + "step": 1580 + }, + { + "epoch": 0.25675702953109547, + "grad_norm": 0.991177499294281, + "learning_rate": 0.0001, + "loss": 97.7093, + "step": 1590 + }, + { + "epoch": 0.2583718536161967, + "grad_norm": 1.147647738456726, + "learning_rate": 0.0001, + "loss": 96.9181, + "step": 1600 + }, + { + "epoch": 0.2599866777012979, + "grad_norm": 1.1756514310836792, + "learning_rate": 0.0001, + "loss": 98.3935, + "step": 1610 + }, + { + "epoch": 0.2616015017863991, + "grad_norm": 1.1708602905273438, + "learning_rate": 0.0001, + "loss": 96.715, + "step": 1620 + }, + { + "epoch": 0.2632163258715004, + "grad_norm": 1.4577715396881104, + "learning_rate": 0.0001, + "loss": 97.2971, + "step": 1630 + }, + { + "epoch": 0.2648311499566016, + "grad_norm": 1.1172800064086914, + "learning_rate": 0.0001, + "loss": 97.6494, + "step": 1640 + }, + { + "epoch": 0.26644597404170284, + "grad_norm": 1.1491258144378662, + "learning_rate": 0.0001, + "loss": 97.5503, + "step": 1650 + }, + { + "epoch": 0.26806079812680406, + "grad_norm": 1.2188555002212524, + "learning_rate": 0.0001, + "loss": 95.3423, + "step": 1660 + }, + { + "epoch": 0.2696756222119053, + "grad_norm": 1.8595614433288574, + "learning_rate": 0.0001, + "loss": 96.0112, + "step": 1670 + }, + { + "epoch": 0.2712904462970065, + "grad_norm": 1.1887950897216797, + "learning_rate": 0.0001, + "loss": 95.4939, + "step": 1680 + }, + { + "epoch": 0.27290527038210777, + "grad_norm": 1.1777839660644531, + "learning_rate": 0.0001, + "loss": 95.9619, + "step": 1690 + }, + { + "epoch": 0.274520094467209, + "grad_norm": 1.1241233348846436, + "learning_rate": 0.0001, + "loss": 96.2834, + "step": 1700 + }, + { + "epoch": 0.2761349185523102, + "grad_norm": 1.0900121927261353, + "learning_rate": 0.0001, + "loss": 94.7518, + "step": 1710 + }, + { + "epoch": 0.2777497426374114, + "grad_norm": 1.0615205764770508, + "learning_rate": 0.0001, + "loss": 95.8032, + "step": 1720 + }, + { + "epoch": 0.27936456672251264, + "grad_norm": 1.2626168727874756, + "learning_rate": 0.0001, + "loss": 93.9454, + "step": 1730 + }, + { + "epoch": 0.2809793908076139, + "grad_norm": 0.8869941830635071, + "learning_rate": 0.0001, + "loss": 95.4085, + "step": 1740 + }, + { + "epoch": 0.28259421489271513, + "grad_norm": 1.0350773334503174, + "learning_rate": 0.0001, + "loss": 95.7954, + "step": 1750 + }, + { + "epoch": 0.28420903897781635, + "grad_norm": 1.364768147468567, + "learning_rate": 0.0001, + "loss": 93.6447, + "step": 1760 + }, + { + "epoch": 0.28582386306291757, + "grad_norm": 1.0115883350372314, + "learning_rate": 0.0001, + "loss": 93.7905, + "step": 1770 + }, + { + "epoch": 0.2874386871480188, + "grad_norm": 0.9393435120582581, + "learning_rate": 0.0001, + "loss": 94.3108, + "step": 1780 + }, + { + "epoch": 0.28905351123312006, + "grad_norm": 1.0769627094268799, + "learning_rate": 0.0001, + "loss": 92.566, + "step": 1790 + }, + { + "epoch": 0.2906683353182213, + "grad_norm": 2.526201009750366, + "learning_rate": 0.0001, + "loss": 92.8161, + "step": 1800 + }, + { + "epoch": 0.2922831594033225, + "grad_norm": 1.3066468238830566, + "learning_rate": 0.0001, + "loss": 92.0459, + "step": 1810 + }, + { + "epoch": 0.2938979834884237, + "grad_norm": 1.121713638305664, + "learning_rate": 0.0001, + "loss": 93.5032, + "step": 1820 + }, + { + "epoch": 0.29551280757352494, + "grad_norm": 1.3744316101074219, + "learning_rate": 0.0001, + "loss": 94.2623, + "step": 1830 + }, + { + "epoch": 0.2971276316586262, + "grad_norm": 1.1561540365219116, + "learning_rate": 0.0001, + "loss": 94.1611, + "step": 1840 + }, + { + "epoch": 0.29874245574372743, + "grad_norm": 1.3947464227676392, + "learning_rate": 0.0001, + "loss": 93.5962, + "step": 1850 + }, + { + "epoch": 0.30035727982882865, + "grad_norm": 1.0011566877365112, + "learning_rate": 0.0001, + "loss": 92.0199, + "step": 1860 + }, + { + "epoch": 0.30197210391392987, + "grad_norm": 0.9978747963905334, + "learning_rate": 0.0001, + "loss": 91.0945, + "step": 1870 + }, + { + "epoch": 0.3035869279990311, + "grad_norm": 0.92522132396698, + "learning_rate": 0.0001, + "loss": 93.274, + "step": 1880 + }, + { + "epoch": 0.30520175208413236, + "grad_norm": 1.0089120864868164, + "learning_rate": 0.0001, + "loss": 91.9308, + "step": 1890 + }, + { + "epoch": 0.3068165761692336, + "grad_norm": 1.0535134077072144, + "learning_rate": 0.0001, + "loss": 92.563, + "step": 1900 + }, + { + "epoch": 0.3084314002543348, + "grad_norm": 0.9782275557518005, + "learning_rate": 0.0001, + "loss": 90.3486, + "step": 1910 + }, + { + "epoch": 0.310046224339436, + "grad_norm": 1.1433162689208984, + "learning_rate": 0.0001, + "loss": 90.4977, + "step": 1920 + }, + { + "epoch": 0.31166104842453723, + "grad_norm": 0.98392653465271, + "learning_rate": 0.0001, + "loss": 92.6529, + "step": 1930 + }, + { + "epoch": 0.3132758725096385, + "grad_norm": 0.8538148403167725, + "learning_rate": 0.0001, + "loss": 90.6859, + "step": 1940 + }, + { + "epoch": 0.3148906965947397, + "grad_norm": 1.019405484199524, + "learning_rate": 0.0001, + "loss": 91.0028, + "step": 1950 + }, + { + "epoch": 0.31650552067984095, + "grad_norm": 0.9828067421913147, + "learning_rate": 0.0001, + "loss": 89.9826, + "step": 1960 + }, + { + "epoch": 0.31812034476494216, + "grad_norm": 0.9725450277328491, + "learning_rate": 0.0001, + "loss": 91.7342, + "step": 1970 + }, + { + "epoch": 0.3197351688500434, + "grad_norm": 1.0139408111572266, + "learning_rate": 0.0001, + "loss": 90.5362, + "step": 1980 + }, + { + "epoch": 0.3213499929351446, + "grad_norm": 1.2493853569030762, + "learning_rate": 0.0001, + "loss": 92.7222, + "step": 1990 + }, + { + "epoch": 0.3229648170202459, + "grad_norm": 1.0152915716171265, + "learning_rate": 0.0001, + "loss": 92.2271, + "step": 2000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.449002378087629e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6a151f546555c78ebc502dd7b703b665f0672d6f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35235df8408202e91954a585851d27187bafaf4e1d75c4e63c51fbf721136ff5 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-2000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6481887f8014318e02cf52947d9a706d2dc0586d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d242f3e01a18e88b791bd1e8a65ba5fab75637ee39df4f81279d801320fdf0e +size 1853729968 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77b2fb12a16d3f688bc0bb68328aabe3addf38a1 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9de07977b63fabea73f5942bc307eeeb1c4115522002b0a3fd7677ae90bc27b +size 1853730480 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a8b77643cdbba52e1954b5b6c8694dba899574d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aebca49afd8fb1f45f80b17165374e1ea3546247898735967c32d21614ec2bc +size 1853731312 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2558d34cc7e09d010fb41c9ce8436d72dc535f5f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eda7d12569faaf8d4e247ecc6d2fa5d2058becaa802b23c2aac7c8425cfd9fc2 +size 1853731120 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32db95253ae0b8e5693325949043fa8571f3dc16 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d39d5d2d76848b554420549ac4c7a96490ff8d906ea07ce643b6051c07115a14 +size 1853731312 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06cb5938a24a6aefe879ccd7846381cfab78c931 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04f5f2272a7f4e2f9731ac51e13fc8236cdf4bf22891f34c961cab386d382b1e +size 1853731120 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11742fa3fe9ab28e5d6b16c8a229f03992ece79d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae9c8a9f701622120aef79035d5590e80e769e5e0360b76a59b55cf37c7c1e10 +size 1853731440 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d32e60e3aaccc3e94b3b49020332a5a1445a4d4 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fc4927cdb2ade42f8eeed0e4fa37754662c8e84e43c399ad3d7f0fd4dc6acee +size 1853731056 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/mp_rank_00_model_states.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f92b5b903a6f62d5fd09f112d53d8821be398b64 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/global_step20000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eed077b61f89bdf5e8da76fcfde5fd91a13c1309ac98fd911d858d1fccac76cf +size 2471676664 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/latest b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/latest new file mode 100644 index 0000000000000000000000000000000000000000..50908603509898f37e005b455ca2e7cad40a4bb0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/latest @@ -0,0 +1 @@ +global_step20000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cfd874628aafba88ac0f1a19bc9d2ae4cfefff3e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51b6f0a49c63a173d7bf73a950a91d902caa70000a71e19a984c971efa35bf5f +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..bb8861d57e95ad5f03aa201959a028f6756e55f9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b5f50bcd069ccd3c9a48054f1c41669137846c10cdb60a5aa039a8ada768b80 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..8e6a18276635c2cb6245e93f19a8c6bf01905f50 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fce2c0b1e35519953281dbe2b21aafe9795221f8ca3af45419752ae5dd050303 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..706a09e7f99e5de76f9fe4b5028041612e52eb8e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b0731c08d841899151f7e8a20b9e19c0befa94522935a6a2abb7d9371c43a7d +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..95f20fbe4edbbd1926a0bb111f62b7c557f5f441 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:704e6e0a37cd6b395c361555a0d835423c8f9146eb7b4a1f8af0aa53ed35ec5a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..d30658593678e775332dd6a631be285b8e95fa21 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8c6a2dd9678c1e045b67eda927fd81972120c9ca67f419413e753819847653c +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..abaa4423f2bc457745e5af27caa5833af057d403 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b1020ca1b8573ee3bdde04d7f0ac57e1c3e5ece577d2ab6c16ed79a8da5236c +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..c4a2250e956779233a66f4f8adcde50db92747d9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7bf2760c666e7f7b0d3229632b612095e541e822ebd2c2b7d0e780af49cf6f1 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..49eccdb3ac5f15454c4882e93393a3bac20ba1c9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25b49e399c27b5b101a1038413547db5f678674d2abf7231691ce55b4a462786 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6ad5478feb2017608170db0b9c10db611f98e8f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f03f24b02bdd285f31b8705583130ff132519634aa9400055ed20b5470b95107 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3c7b3be2d47157ac4c636cd02dbb8962b584ede9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/trainer_state.json @@ -0,0 +1,14034 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.2294665024928846, + "eval_steps": 2000, + "global_step": 20000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0016148240851012292, + "grad_norm": 43.90101623535156, + "learning_rate": 0.0001, + "loss": 307.2084, + "step": 10 + }, + { + "epoch": 0.0032296481702024584, + "grad_norm": 19.106313705444336, + "learning_rate": 0.0001, + "loss": 320.2794, + "step": 20 + }, + { + "epoch": 0.004844472255303688, + "grad_norm": 5.771847248077393, + "learning_rate": 0.0001, + "loss": 247.4977, + "step": 30 + }, + { + "epoch": 0.006459296340404917, + "grad_norm": 2.630659580230713, + "learning_rate": 0.0001, + "loss": 229.9593, + "step": 40 + }, + { + "epoch": 0.008074120425506147, + "grad_norm": 2.487499475479126, + "learning_rate": 0.0001, + "loss": 228.65, + "step": 50 + }, + { + "epoch": 0.009688944510607376, + "grad_norm": 2.078263521194458, + "learning_rate": 0.0001, + "loss": 227.648, + "step": 60 + }, + { + "epoch": 0.011303768595708604, + "grad_norm": 3.105417490005493, + "learning_rate": 0.0001, + "loss": 227.9345, + "step": 70 + }, + { + "epoch": 0.012918592680809833, + "grad_norm": 2.2145848274230957, + "learning_rate": 0.0001, + "loss": 227.3662, + "step": 80 + }, + { + "epoch": 0.014533416765911064, + "grad_norm": 1.9644615650177002, + "learning_rate": 0.0001, + "loss": 226.9145, + "step": 90 + }, + { + "epoch": 0.016148240851012293, + "grad_norm": 1.867382287979126, + "learning_rate": 0.0001, + "loss": 227.5703, + "step": 100 + }, + { + "epoch": 0.017763064936113522, + "grad_norm": 1.4613710641860962, + "learning_rate": 0.0001, + "loss": 227.1107, + "step": 110 + }, + { + "epoch": 0.01937788902121475, + "grad_norm": 2.8953869342803955, + "learning_rate": 0.0001, + "loss": 226.8185, + "step": 120 + }, + { + "epoch": 0.02099271310631598, + "grad_norm": 1.8446482419967651, + "learning_rate": 0.0001, + "loss": 226.9624, + "step": 130 + }, + { + "epoch": 0.02260753719141721, + "grad_norm": 1.3048362731933594, + "learning_rate": 0.0001, + "loss": 226.8404, + "step": 140 + }, + { + "epoch": 0.024222361276518438, + "grad_norm": 1.7100074291229248, + "learning_rate": 0.0001, + "loss": 226.1071, + "step": 150 + }, + { + "epoch": 0.025837185361619667, + "grad_norm": 1.7875330448150635, + "learning_rate": 0.0001, + "loss": 226.0764, + "step": 160 + }, + { + "epoch": 0.0274520094467209, + "grad_norm": 1.7503769397735596, + "learning_rate": 0.0001, + "loss": 226.0772, + "step": 170 + }, + { + "epoch": 0.029066833531822128, + "grad_norm": 2.7083983421325684, + "learning_rate": 0.0001, + "loss": 224.7667, + "step": 180 + }, + { + "epoch": 0.030681657616923357, + "grad_norm": 1.8192371129989624, + "learning_rate": 0.0001, + "loss": 223.6494, + "step": 190 + }, + { + "epoch": 0.032296481702024586, + "grad_norm": 3.210358142852783, + "learning_rate": 0.0001, + "loss": 223.3545, + "step": 200 + }, + { + "epoch": 0.03391130578712581, + "grad_norm": 5.819669246673584, + "learning_rate": 0.0001, + "loss": 222.1732, + "step": 210 + }, + { + "epoch": 0.035526129872227044, + "grad_norm": 4.185793399810791, + "learning_rate": 0.0001, + "loss": 221.6866, + "step": 220 + }, + { + "epoch": 0.03714095395732828, + "grad_norm": 2.7514216899871826, + "learning_rate": 0.0001, + "loss": 221.0769, + "step": 230 + }, + { + "epoch": 0.0387557780424295, + "grad_norm": 1.5478988885879517, + "learning_rate": 0.0001, + "loss": 219.7232, + "step": 240 + }, + { + "epoch": 0.040370602127530734, + "grad_norm": 1.4958223104476929, + "learning_rate": 0.0001, + "loss": 218.5454, + "step": 250 + }, + { + "epoch": 0.04198542621263196, + "grad_norm": 6.36635684967041, + "learning_rate": 0.0001, + "loss": 218.5061, + "step": 260 + }, + { + "epoch": 0.04360025029773319, + "grad_norm": 3.029601812362671, + "learning_rate": 0.0001, + "loss": 216.3466, + "step": 270 + }, + { + "epoch": 0.04521507438283442, + "grad_norm": 2.6436100006103516, + "learning_rate": 0.0001, + "loss": 214.0003, + "step": 280 + }, + { + "epoch": 0.04682989846793565, + "grad_norm": 5.329655647277832, + "learning_rate": 0.0001, + "loss": 212.9169, + "step": 290 + }, + { + "epoch": 0.048444722553036876, + "grad_norm": 4.930875301361084, + "learning_rate": 0.0001, + "loss": 209.7737, + "step": 300 + }, + { + "epoch": 0.05005954663813811, + "grad_norm": 18.902585983276367, + "learning_rate": 0.0001, + "loss": 209.4267, + "step": 310 + }, + { + "epoch": 0.051674370723239334, + "grad_norm": 5.364975929260254, + "learning_rate": 0.0001, + "loss": 206.49, + "step": 320 + }, + { + "epoch": 0.053289194808340566, + "grad_norm": 3.1421146392822266, + "learning_rate": 0.0001, + "loss": 204.8132, + "step": 330 + }, + { + "epoch": 0.0549040188934418, + "grad_norm": 3.399320602416992, + "learning_rate": 0.0001, + "loss": 202.4785, + "step": 340 + }, + { + "epoch": 0.056518842978543024, + "grad_norm": 5.137377738952637, + "learning_rate": 0.0001, + "loss": 199.5143, + "step": 350 + }, + { + "epoch": 0.058133667063644257, + "grad_norm": 5.4967522621154785, + "learning_rate": 0.0001, + "loss": 196.2809, + "step": 360 + }, + { + "epoch": 0.05974849114874548, + "grad_norm": 6.435070037841797, + "learning_rate": 0.0001, + "loss": 196.8831, + "step": 370 + }, + { + "epoch": 0.061363315233846714, + "grad_norm": 5.7334370613098145, + "learning_rate": 0.0001, + "loss": 193.141, + "step": 380 + }, + { + "epoch": 0.06297813931894794, + "grad_norm": 3.6522979736328125, + "learning_rate": 0.0001, + "loss": 191.5819, + "step": 390 + }, + { + "epoch": 0.06459296340404917, + "grad_norm": 2.614051103591919, + "learning_rate": 0.0001, + "loss": 190.046, + "step": 400 + }, + { + "epoch": 0.0662077874891504, + "grad_norm": 4.845924377441406, + "learning_rate": 0.0001, + "loss": 188.7121, + "step": 410 + }, + { + "epoch": 0.06782261157425162, + "grad_norm": 4.319681644439697, + "learning_rate": 0.0001, + "loss": 185.3697, + "step": 420 + }, + { + "epoch": 0.06943743565935286, + "grad_norm": 2.7018792629241943, + "learning_rate": 0.0001, + "loss": 183.5171, + "step": 430 + }, + { + "epoch": 0.07105225974445409, + "grad_norm": 2.6407113075256348, + "learning_rate": 0.0001, + "loss": 181.245, + "step": 440 + }, + { + "epoch": 0.07266708382955532, + "grad_norm": 3.314251184463501, + "learning_rate": 0.0001, + "loss": 179.3874, + "step": 450 + }, + { + "epoch": 0.07428190791465655, + "grad_norm": 2.635136365890503, + "learning_rate": 0.0001, + "loss": 177.8257, + "step": 460 + }, + { + "epoch": 0.07589673199975777, + "grad_norm": 5.227738857269287, + "learning_rate": 0.0001, + "loss": 176.4699, + "step": 470 + }, + { + "epoch": 0.077511556084859, + "grad_norm": 4.998415470123291, + "learning_rate": 0.0001, + "loss": 175.7601, + "step": 480 + }, + { + "epoch": 0.07912638016996024, + "grad_norm": 2.946892261505127, + "learning_rate": 0.0001, + "loss": 173.936, + "step": 490 + }, + { + "epoch": 0.08074120425506147, + "grad_norm": 4.20358943939209, + "learning_rate": 0.0001, + "loss": 172.065, + "step": 500 + }, + { + "epoch": 0.08235602834016269, + "grad_norm": 3.125765562057495, + "learning_rate": 0.0001, + "loss": 171.3727, + "step": 510 + }, + { + "epoch": 0.08397085242526392, + "grad_norm": 2.4870612621307373, + "learning_rate": 0.0001, + "loss": 169.9, + "step": 520 + }, + { + "epoch": 0.08558567651036515, + "grad_norm": 3.917384147644043, + "learning_rate": 0.0001, + "loss": 168.5049, + "step": 530 + }, + { + "epoch": 0.08720050059546638, + "grad_norm": 2.302142858505249, + "learning_rate": 0.0001, + "loss": 167.2027, + "step": 540 + }, + { + "epoch": 0.08881532468056762, + "grad_norm": 4.794841289520264, + "learning_rate": 0.0001, + "loss": 166.0887, + "step": 550 + }, + { + "epoch": 0.09043014876566884, + "grad_norm": 2.200291395187378, + "learning_rate": 0.0001, + "loss": 164.5882, + "step": 560 + }, + { + "epoch": 0.09204497285077007, + "grad_norm": 1.9572436809539795, + "learning_rate": 0.0001, + "loss": 163.3007, + "step": 570 + }, + { + "epoch": 0.0936597969358713, + "grad_norm": 3.443633556365967, + "learning_rate": 0.0001, + "loss": 160.6805, + "step": 580 + }, + { + "epoch": 0.09527462102097253, + "grad_norm": 2.6079020500183105, + "learning_rate": 0.0001, + "loss": 160.3372, + "step": 590 + }, + { + "epoch": 0.09688944510607375, + "grad_norm": 2.4401192665100098, + "learning_rate": 0.0001, + "loss": 160.0956, + "step": 600 + }, + { + "epoch": 0.09850426919117498, + "grad_norm": 2.686101198196411, + "learning_rate": 0.0001, + "loss": 157.6138, + "step": 610 + }, + { + "epoch": 0.10011909327627622, + "grad_norm": 2.594151496887207, + "learning_rate": 0.0001, + "loss": 156.0726, + "step": 620 + }, + { + "epoch": 0.10173391736137745, + "grad_norm": 2.509141683578491, + "learning_rate": 0.0001, + "loss": 155.9923, + "step": 630 + }, + { + "epoch": 0.10334874144647867, + "grad_norm": 2.4077377319335938, + "learning_rate": 0.0001, + "loss": 155.214, + "step": 640 + }, + { + "epoch": 0.1049635655315799, + "grad_norm": 4.083552837371826, + "learning_rate": 0.0001, + "loss": 152.8257, + "step": 650 + }, + { + "epoch": 0.10657838961668113, + "grad_norm": 2.7188713550567627, + "learning_rate": 0.0001, + "loss": 151.875, + "step": 660 + }, + { + "epoch": 0.10819321370178236, + "grad_norm": 2.41287899017334, + "learning_rate": 0.0001, + "loss": 151.935, + "step": 670 + }, + { + "epoch": 0.1098080377868836, + "grad_norm": 3.995708465576172, + "learning_rate": 0.0001, + "loss": 151.0069, + "step": 680 + }, + { + "epoch": 0.11142286187198482, + "grad_norm": 3.217705488204956, + "learning_rate": 0.0001, + "loss": 149.5922, + "step": 690 + }, + { + "epoch": 0.11303768595708605, + "grad_norm": 3.4349842071533203, + "learning_rate": 0.0001, + "loss": 148.0044, + "step": 700 + }, + { + "epoch": 0.11465251004218728, + "grad_norm": 2.106865167617798, + "learning_rate": 0.0001, + "loss": 147.8404, + "step": 710 + }, + { + "epoch": 0.11626733412728851, + "grad_norm": 1.8729510307312012, + "learning_rate": 0.0001, + "loss": 145.1299, + "step": 720 + }, + { + "epoch": 0.11788215821238973, + "grad_norm": 4.14620304107666, + "learning_rate": 0.0001, + "loss": 144.3928, + "step": 730 + }, + { + "epoch": 0.11949698229749096, + "grad_norm": 2.202505350112915, + "learning_rate": 0.0001, + "loss": 144.8357, + "step": 740 + }, + { + "epoch": 0.1211118063825922, + "grad_norm": 2.201470375061035, + "learning_rate": 0.0001, + "loss": 143.2202, + "step": 750 + }, + { + "epoch": 0.12272663046769343, + "grad_norm": 1.5977853536605835, + "learning_rate": 0.0001, + "loss": 143.3537, + "step": 760 + }, + { + "epoch": 0.12434145455279466, + "grad_norm": 2.4403927326202393, + "learning_rate": 0.0001, + "loss": 141.2914, + "step": 770 + }, + { + "epoch": 0.12595627863789588, + "grad_norm": 1.6895846128463745, + "learning_rate": 0.0001, + "loss": 139.9639, + "step": 780 + }, + { + "epoch": 0.12757110272299713, + "grad_norm": 3.4370665550231934, + "learning_rate": 0.0001, + "loss": 140.715, + "step": 790 + }, + { + "epoch": 0.12918592680809834, + "grad_norm": 2.1449077129364014, + "learning_rate": 0.0001, + "loss": 139.2094, + "step": 800 + }, + { + "epoch": 0.13080075089319956, + "grad_norm": 2.4578912258148193, + "learning_rate": 0.0001, + "loss": 137.6724, + "step": 810 + }, + { + "epoch": 0.1324155749783008, + "grad_norm": 1.8994814157485962, + "learning_rate": 0.0001, + "loss": 136.9551, + "step": 820 + }, + { + "epoch": 0.13403039906340203, + "grad_norm": 2.6531031131744385, + "learning_rate": 0.0001, + "loss": 136.5812, + "step": 830 + }, + { + "epoch": 0.13564522314850325, + "grad_norm": 3.8680083751678467, + "learning_rate": 0.0001, + "loss": 134.3655, + "step": 840 + }, + { + "epoch": 0.1372600472336045, + "grad_norm": 1.9594208002090454, + "learning_rate": 0.0001, + "loss": 135.4958, + "step": 850 + }, + { + "epoch": 0.1388748713187057, + "grad_norm": 1.5475000143051147, + "learning_rate": 0.0001, + "loss": 135.2142, + "step": 860 + }, + { + "epoch": 0.14048969540380696, + "grad_norm": 1.6569288969039917, + "learning_rate": 0.0001, + "loss": 133.4162, + "step": 870 + }, + { + "epoch": 0.14210451948890818, + "grad_norm": 1.6724224090576172, + "learning_rate": 0.0001, + "loss": 133.7519, + "step": 880 + }, + { + "epoch": 0.1437193435740094, + "grad_norm": 1.7299500703811646, + "learning_rate": 0.0001, + "loss": 131.471, + "step": 890 + }, + { + "epoch": 0.14533416765911064, + "grad_norm": 1.9444470405578613, + "learning_rate": 0.0001, + "loss": 128.5966, + "step": 900 + }, + { + "epoch": 0.14694899174421186, + "grad_norm": 2.404723644256592, + "learning_rate": 0.0001, + "loss": 129.9824, + "step": 910 + }, + { + "epoch": 0.1485638158293131, + "grad_norm": 2.1493897438049316, + "learning_rate": 0.0001, + "loss": 128.0219, + "step": 920 + }, + { + "epoch": 0.15017863991441432, + "grad_norm": 2.8861117362976074, + "learning_rate": 0.0001, + "loss": 128.5297, + "step": 930 + }, + { + "epoch": 0.15179346399951554, + "grad_norm": 2.546415090560913, + "learning_rate": 0.0001, + "loss": 126.6817, + "step": 940 + }, + { + "epoch": 0.1534082880846168, + "grad_norm": 1.727663278579712, + "learning_rate": 0.0001, + "loss": 125.0413, + "step": 950 + }, + { + "epoch": 0.155023112169718, + "grad_norm": 3.7156920433044434, + "learning_rate": 0.0001, + "loss": 124.8085, + "step": 960 + }, + { + "epoch": 0.15663793625481925, + "grad_norm": 3.0576517581939697, + "learning_rate": 0.0001, + "loss": 125.9107, + "step": 970 + }, + { + "epoch": 0.15825276033992047, + "grad_norm": 1.9951574802398682, + "learning_rate": 0.0001, + "loss": 123.5359, + "step": 980 + }, + { + "epoch": 0.1598675844250217, + "grad_norm": 1.4620885848999023, + "learning_rate": 0.0001, + "loss": 121.9363, + "step": 990 + }, + { + "epoch": 0.16148240851012294, + "grad_norm": 1.7640169858932495, + "learning_rate": 0.0001, + "loss": 121.1764, + "step": 1000 + }, + { + "epoch": 0.16309723259522416, + "grad_norm": 2.921750545501709, + "learning_rate": 0.0001, + "loss": 119.6931, + "step": 1010 + }, + { + "epoch": 0.16471205668032537, + "grad_norm": 1.5623588562011719, + "learning_rate": 0.0001, + "loss": 119.097, + "step": 1020 + }, + { + "epoch": 0.16632688076542662, + "grad_norm": 2.2386627197265625, + "learning_rate": 0.0001, + "loss": 120.9224, + "step": 1030 + }, + { + "epoch": 0.16794170485052784, + "grad_norm": 1.3213791847229004, + "learning_rate": 0.0001, + "loss": 119.5649, + "step": 1040 + }, + { + "epoch": 0.16955652893562909, + "grad_norm": 1.5255440473556519, + "learning_rate": 0.0001, + "loss": 118.1524, + "step": 1050 + }, + { + "epoch": 0.1711713530207303, + "grad_norm": 1.6038514375686646, + "learning_rate": 0.0001, + "loss": 118.6307, + "step": 1060 + }, + { + "epoch": 0.17278617710583152, + "grad_norm": 2.356739044189453, + "learning_rate": 0.0001, + "loss": 116.8472, + "step": 1070 + }, + { + "epoch": 0.17440100119093277, + "grad_norm": 2.5326743125915527, + "learning_rate": 0.0001, + "loss": 115.6625, + "step": 1080 + }, + { + "epoch": 0.176015825276034, + "grad_norm": 1.2823396921157837, + "learning_rate": 0.0001, + "loss": 115.3312, + "step": 1090 + }, + { + "epoch": 0.17763064936113523, + "grad_norm": 1.981788158416748, + "learning_rate": 0.0001, + "loss": 116.2257, + "step": 1100 + }, + { + "epoch": 0.17924547344623645, + "grad_norm": 1.540440559387207, + "learning_rate": 0.0001, + "loss": 114.5031, + "step": 1110 + }, + { + "epoch": 0.18086029753133767, + "grad_norm": 1.7667800188064575, + "learning_rate": 0.0001, + "loss": 113.7663, + "step": 1120 + }, + { + "epoch": 0.18247512161643892, + "grad_norm": 2.2793102264404297, + "learning_rate": 0.0001, + "loss": 115.219, + "step": 1130 + }, + { + "epoch": 0.18408994570154014, + "grad_norm": 2.132046699523926, + "learning_rate": 0.0001, + "loss": 112.8482, + "step": 1140 + }, + { + "epoch": 0.18570476978664135, + "grad_norm": 1.3153988122940063, + "learning_rate": 0.0001, + "loss": 111.4546, + "step": 1150 + }, + { + "epoch": 0.1873195938717426, + "grad_norm": 1.5246292352676392, + "learning_rate": 0.0001, + "loss": 112.4348, + "step": 1160 + }, + { + "epoch": 0.18893441795684382, + "grad_norm": 1.365096092224121, + "learning_rate": 0.0001, + "loss": 109.755, + "step": 1170 + }, + { + "epoch": 0.19054924204194507, + "grad_norm": 1.2777953147888184, + "learning_rate": 0.0001, + "loss": 111.0318, + "step": 1180 + }, + { + "epoch": 0.19216406612704628, + "grad_norm": 1.8672829866409302, + "learning_rate": 0.0001, + "loss": 109.1648, + "step": 1190 + }, + { + "epoch": 0.1937788902121475, + "grad_norm": 1.709987759590149, + "learning_rate": 0.0001, + "loss": 108.1616, + "step": 1200 + }, + { + "epoch": 0.19539371429724875, + "grad_norm": 1.176277995109558, + "learning_rate": 0.0001, + "loss": 109.3591, + "step": 1210 + }, + { + "epoch": 0.19700853838234997, + "grad_norm": 1.4181180000305176, + "learning_rate": 0.0001, + "loss": 110.0216, + "step": 1220 + }, + { + "epoch": 0.19862336246745121, + "grad_norm": 1.3246116638183594, + "learning_rate": 0.0001, + "loss": 107.9769, + "step": 1230 + }, + { + "epoch": 0.20023818655255243, + "grad_norm": 1.2501498460769653, + "learning_rate": 0.0001, + "loss": 108.2601, + "step": 1240 + }, + { + "epoch": 0.20185301063765365, + "grad_norm": 1.2277404069900513, + "learning_rate": 0.0001, + "loss": 107.1258, + "step": 1250 + }, + { + "epoch": 0.2034678347227549, + "grad_norm": 1.8064894676208496, + "learning_rate": 0.0001, + "loss": 105.6278, + "step": 1260 + }, + { + "epoch": 0.20508265880785612, + "grad_norm": 1.708897352218628, + "learning_rate": 0.0001, + "loss": 106.9269, + "step": 1270 + }, + { + "epoch": 0.20669748289295733, + "grad_norm": 1.4507478475570679, + "learning_rate": 0.0001, + "loss": 106.9236, + "step": 1280 + }, + { + "epoch": 0.20831230697805858, + "grad_norm": 1.4904310703277588, + "learning_rate": 0.0001, + "loss": 106.0653, + "step": 1290 + }, + { + "epoch": 0.2099271310631598, + "grad_norm": 1.2637165784835815, + "learning_rate": 0.0001, + "loss": 104.8665, + "step": 1300 + }, + { + "epoch": 0.21154195514826105, + "grad_norm": 1.4085133075714111, + "learning_rate": 0.0001, + "loss": 104.8891, + "step": 1310 + }, + { + "epoch": 0.21315677923336226, + "grad_norm": 1.5710629224777222, + "learning_rate": 0.0001, + "loss": 104.3009, + "step": 1320 + }, + { + "epoch": 0.21477160331846348, + "grad_norm": 1.3634663820266724, + "learning_rate": 0.0001, + "loss": 104.4545, + "step": 1330 + }, + { + "epoch": 0.21638642740356473, + "grad_norm": 1.5565859079360962, + "learning_rate": 0.0001, + "loss": 103.6205, + "step": 1340 + }, + { + "epoch": 0.21800125148866595, + "grad_norm": 1.4491888284683228, + "learning_rate": 0.0001, + "loss": 104.0892, + "step": 1350 + }, + { + "epoch": 0.2196160755737672, + "grad_norm": 1.7609063386917114, + "learning_rate": 0.0001, + "loss": 104.5177, + "step": 1360 + }, + { + "epoch": 0.2212308996588684, + "grad_norm": 1.6723753213882446, + "learning_rate": 0.0001, + "loss": 104.5552, + "step": 1370 + }, + { + "epoch": 0.22284572374396963, + "grad_norm": 1.3781665563583374, + "learning_rate": 0.0001, + "loss": 102.1022, + "step": 1380 + }, + { + "epoch": 0.22446054782907088, + "grad_norm": 1.5397366285324097, + "learning_rate": 0.0001, + "loss": 102.5882, + "step": 1390 + }, + { + "epoch": 0.2260753719141721, + "grad_norm": 1.4567879438400269, + "learning_rate": 0.0001, + "loss": 101.3781, + "step": 1400 + }, + { + "epoch": 0.22769019599927334, + "grad_norm": 1.2882301807403564, + "learning_rate": 0.0001, + "loss": 100.9378, + "step": 1410 + }, + { + "epoch": 0.22930502008437456, + "grad_norm": 1.4510424137115479, + "learning_rate": 0.0001, + "loss": 101.5383, + "step": 1420 + }, + { + "epoch": 0.23091984416947578, + "grad_norm": 2.8331449031829834, + "learning_rate": 0.0001, + "loss": 100.2773, + "step": 1430 + }, + { + "epoch": 0.23253466825457703, + "grad_norm": 1.363616943359375, + "learning_rate": 0.0001, + "loss": 101.646, + "step": 1440 + }, + { + "epoch": 0.23414949233967824, + "grad_norm": 1.2184548377990723, + "learning_rate": 0.0001, + "loss": 101.1133, + "step": 1450 + }, + { + "epoch": 0.23576431642477946, + "grad_norm": 2.235656976699829, + "learning_rate": 0.0001, + "loss": 100.2044, + "step": 1460 + }, + { + "epoch": 0.2373791405098807, + "grad_norm": 1.2334418296813965, + "learning_rate": 0.0001, + "loss": 99.4031, + "step": 1470 + }, + { + "epoch": 0.23899396459498193, + "grad_norm": 1.2394684553146362, + "learning_rate": 0.0001, + "loss": 101.1247, + "step": 1480 + }, + { + "epoch": 0.24060878868008317, + "grad_norm": 1.5218186378479004, + "learning_rate": 0.0001, + "loss": 100.4413, + "step": 1490 + }, + { + "epoch": 0.2422236127651844, + "grad_norm": 1.2112364768981934, + "learning_rate": 0.0001, + "loss": 99.8722, + "step": 1500 + }, + { + "epoch": 0.2438384368502856, + "grad_norm": 1.2277610301971436, + "learning_rate": 0.0001, + "loss": 99.0276, + "step": 1510 + }, + { + "epoch": 0.24545326093538686, + "grad_norm": 1.1005053520202637, + "learning_rate": 0.0001, + "loss": 98.6764, + "step": 1520 + }, + { + "epoch": 0.24706808502048808, + "grad_norm": 1.0754456520080566, + "learning_rate": 0.0001, + "loss": 97.8968, + "step": 1530 + }, + { + "epoch": 0.24868290910558932, + "grad_norm": 1.0695950984954834, + "learning_rate": 0.0001, + "loss": 98.4032, + "step": 1540 + }, + { + "epoch": 0.25029773319069054, + "grad_norm": 1.5098627805709839, + "learning_rate": 0.0001, + "loss": 97.9391, + "step": 1550 + }, + { + "epoch": 0.25191255727579176, + "grad_norm": 1.006681203842163, + "learning_rate": 0.0001, + "loss": 96.5525, + "step": 1560 + }, + { + "epoch": 0.253527381360893, + "grad_norm": 1.3987332582473755, + "learning_rate": 0.0001, + "loss": 98.383, + "step": 1570 + }, + { + "epoch": 0.25514220544599425, + "grad_norm": 1.290887713432312, + "learning_rate": 0.0001, + "loss": 99.0901, + "step": 1580 + }, + { + "epoch": 0.25675702953109547, + "grad_norm": 0.991177499294281, + "learning_rate": 0.0001, + "loss": 97.7093, + "step": 1590 + }, + { + "epoch": 0.2583718536161967, + "grad_norm": 1.147647738456726, + "learning_rate": 0.0001, + "loss": 96.9181, + "step": 1600 + }, + { + "epoch": 0.2599866777012979, + "grad_norm": 1.1756514310836792, + "learning_rate": 0.0001, + "loss": 98.3935, + "step": 1610 + }, + { + "epoch": 0.2616015017863991, + "grad_norm": 1.1708602905273438, + "learning_rate": 0.0001, + "loss": 96.715, + "step": 1620 + }, + { + "epoch": 0.2632163258715004, + "grad_norm": 1.4577715396881104, + "learning_rate": 0.0001, + "loss": 97.2971, + "step": 1630 + }, + { + "epoch": 0.2648311499566016, + "grad_norm": 1.1172800064086914, + "learning_rate": 0.0001, + "loss": 97.6494, + "step": 1640 + }, + { + "epoch": 0.26644597404170284, + "grad_norm": 1.1491258144378662, + "learning_rate": 0.0001, + "loss": 97.5503, + "step": 1650 + }, + { + "epoch": 0.26806079812680406, + "grad_norm": 1.2188555002212524, + "learning_rate": 0.0001, + "loss": 95.3423, + "step": 1660 + }, + { + "epoch": 0.2696756222119053, + "grad_norm": 1.8595614433288574, + "learning_rate": 0.0001, + "loss": 96.0112, + "step": 1670 + }, + { + "epoch": 0.2712904462970065, + "grad_norm": 1.1887950897216797, + "learning_rate": 0.0001, + "loss": 95.4939, + "step": 1680 + }, + { + "epoch": 0.27290527038210777, + "grad_norm": 1.1777839660644531, + "learning_rate": 0.0001, + "loss": 95.9619, + "step": 1690 + }, + { + "epoch": 0.274520094467209, + "grad_norm": 1.1241233348846436, + "learning_rate": 0.0001, + "loss": 96.2834, + "step": 1700 + }, + { + "epoch": 0.2761349185523102, + "grad_norm": 1.0900121927261353, + "learning_rate": 0.0001, + "loss": 94.7518, + "step": 1710 + }, + { + "epoch": 0.2777497426374114, + "grad_norm": 1.0615205764770508, + "learning_rate": 0.0001, + "loss": 95.8032, + "step": 1720 + }, + { + "epoch": 0.27936456672251264, + "grad_norm": 1.2626168727874756, + "learning_rate": 0.0001, + "loss": 93.9454, + "step": 1730 + }, + { + "epoch": 0.2809793908076139, + "grad_norm": 0.8869941830635071, + "learning_rate": 0.0001, + "loss": 95.4085, + "step": 1740 + }, + { + "epoch": 0.28259421489271513, + "grad_norm": 1.0350773334503174, + "learning_rate": 0.0001, + "loss": 95.7954, + "step": 1750 + }, + { + "epoch": 0.28420903897781635, + "grad_norm": 1.364768147468567, + "learning_rate": 0.0001, + "loss": 93.6447, + "step": 1760 + }, + { + "epoch": 0.28582386306291757, + "grad_norm": 1.0115883350372314, + "learning_rate": 0.0001, + "loss": 93.7905, + "step": 1770 + }, + { + "epoch": 0.2874386871480188, + "grad_norm": 0.9393435120582581, + "learning_rate": 0.0001, + "loss": 94.3108, + "step": 1780 + }, + { + "epoch": 0.28905351123312006, + "grad_norm": 1.0769627094268799, + "learning_rate": 0.0001, + "loss": 92.566, + "step": 1790 + }, + { + "epoch": 0.2906683353182213, + "grad_norm": 2.526201009750366, + "learning_rate": 0.0001, + "loss": 92.8161, + "step": 1800 + }, + { + "epoch": 0.2922831594033225, + "grad_norm": 1.3066468238830566, + "learning_rate": 0.0001, + "loss": 92.0459, + "step": 1810 + }, + { + "epoch": 0.2938979834884237, + "grad_norm": 1.121713638305664, + "learning_rate": 0.0001, + "loss": 93.5032, + "step": 1820 + }, + { + "epoch": 0.29551280757352494, + "grad_norm": 1.3744316101074219, + "learning_rate": 0.0001, + "loss": 94.2623, + "step": 1830 + }, + { + "epoch": 0.2971276316586262, + "grad_norm": 1.1561540365219116, + "learning_rate": 0.0001, + "loss": 94.1611, + "step": 1840 + }, + { + "epoch": 0.29874245574372743, + "grad_norm": 1.3947464227676392, + "learning_rate": 0.0001, + "loss": 93.5962, + "step": 1850 + }, + { + "epoch": 0.30035727982882865, + "grad_norm": 1.0011566877365112, + "learning_rate": 0.0001, + "loss": 92.0199, + "step": 1860 + }, + { + "epoch": 0.30197210391392987, + "grad_norm": 0.9978747963905334, + "learning_rate": 0.0001, + "loss": 91.0945, + "step": 1870 + }, + { + "epoch": 0.3035869279990311, + "grad_norm": 0.92522132396698, + "learning_rate": 0.0001, + "loss": 93.274, + "step": 1880 + }, + { + "epoch": 0.30520175208413236, + "grad_norm": 1.0089120864868164, + "learning_rate": 0.0001, + "loss": 91.9308, + "step": 1890 + }, + { + "epoch": 0.3068165761692336, + "grad_norm": 1.0535134077072144, + "learning_rate": 0.0001, + "loss": 92.563, + "step": 1900 + }, + { + "epoch": 0.3084314002543348, + "grad_norm": 0.9782275557518005, + "learning_rate": 0.0001, + "loss": 90.3486, + "step": 1910 + }, + { + "epoch": 0.310046224339436, + "grad_norm": 1.1433162689208984, + "learning_rate": 0.0001, + "loss": 90.4977, + "step": 1920 + }, + { + "epoch": 0.31166104842453723, + "grad_norm": 0.98392653465271, + "learning_rate": 0.0001, + "loss": 92.6529, + "step": 1930 + }, + { + "epoch": 0.3132758725096385, + "grad_norm": 0.8538148403167725, + "learning_rate": 0.0001, + "loss": 90.6859, + "step": 1940 + }, + { + "epoch": 0.3148906965947397, + "grad_norm": 1.019405484199524, + "learning_rate": 0.0001, + "loss": 91.0028, + "step": 1950 + }, + { + "epoch": 0.31650552067984095, + "grad_norm": 0.9828067421913147, + "learning_rate": 0.0001, + "loss": 89.9826, + "step": 1960 + }, + { + "epoch": 0.31812034476494216, + "grad_norm": 0.9725450277328491, + "learning_rate": 0.0001, + "loss": 91.7342, + "step": 1970 + }, + { + "epoch": 0.3197351688500434, + "grad_norm": 1.0139408111572266, + "learning_rate": 0.0001, + "loss": 90.5362, + "step": 1980 + }, + { + "epoch": 0.3213499929351446, + "grad_norm": 1.2493853569030762, + "learning_rate": 0.0001, + "loss": 92.7222, + "step": 1990 + }, + { + "epoch": 0.3229648170202459, + "grad_norm": 1.0152915716171265, + "learning_rate": 0.0001, + "loss": 92.2271, + "step": 2000 + }, + { + "epoch": 0.3245796411053471, + "grad_norm": 0.9745819568634033, + "learning_rate": 0.0001, + "loss": 88.7034, + "step": 2010 + }, + { + "epoch": 0.3261944651904483, + "grad_norm": 1.009624719619751, + "learning_rate": 0.0001, + "loss": 90.7428, + "step": 2020 + }, + { + "epoch": 0.32780928927554953, + "grad_norm": 0.910858690738678, + "learning_rate": 0.0001, + "loss": 89.8199, + "step": 2030 + }, + { + "epoch": 0.32942411336065075, + "grad_norm": 1.1745878458023071, + "learning_rate": 0.0001, + "loss": 90.8267, + "step": 2040 + }, + { + "epoch": 0.331038937445752, + "grad_norm": 0.9409273266792297, + "learning_rate": 0.0001, + "loss": 90.2937, + "step": 2050 + }, + { + "epoch": 0.33265376153085324, + "grad_norm": 1.4152942895889282, + "learning_rate": 0.0001, + "loss": 90.3965, + "step": 2060 + }, + { + "epoch": 0.33426858561595446, + "grad_norm": 0.9529693722724915, + "learning_rate": 0.0001, + "loss": 90.4739, + "step": 2070 + }, + { + "epoch": 0.3358834097010557, + "grad_norm": 0.8531096577644348, + "learning_rate": 0.0001, + "loss": 90.9301, + "step": 2080 + }, + { + "epoch": 0.3374982337861569, + "grad_norm": 0.9314167499542236, + "learning_rate": 0.0001, + "loss": 90.4235, + "step": 2090 + }, + { + "epoch": 0.33911305787125817, + "grad_norm": 0.9253250360488892, + "learning_rate": 0.0001, + "loss": 88.1893, + "step": 2100 + }, + { + "epoch": 0.3407278819563594, + "grad_norm": 1.773053526878357, + "learning_rate": 0.0001, + "loss": 88.3417, + "step": 2110 + }, + { + "epoch": 0.3423427060414606, + "grad_norm": 0.9432951211929321, + "learning_rate": 0.0001, + "loss": 89.7141, + "step": 2120 + }, + { + "epoch": 0.34395753012656183, + "grad_norm": 3.8847527503967285, + "learning_rate": 0.0001, + "loss": 89.1009, + "step": 2130 + }, + { + "epoch": 0.34557235421166305, + "grad_norm": 1.0309821367263794, + "learning_rate": 0.0001, + "loss": 89.2965, + "step": 2140 + }, + { + "epoch": 0.3471871782967643, + "grad_norm": 0.9590495824813843, + "learning_rate": 0.0001, + "loss": 89.1734, + "step": 2150 + }, + { + "epoch": 0.34880200238186554, + "grad_norm": 0.9943560361862183, + "learning_rate": 0.0001, + "loss": 85.425, + "step": 2160 + }, + { + "epoch": 0.35041682646696676, + "grad_norm": 1.3322339057922363, + "learning_rate": 0.0001, + "loss": 89.7791, + "step": 2170 + }, + { + "epoch": 0.352031650552068, + "grad_norm": 1.0212253332138062, + "learning_rate": 0.0001, + "loss": 89.3771, + "step": 2180 + }, + { + "epoch": 0.3536464746371692, + "grad_norm": 1.0446089506149292, + "learning_rate": 0.0001, + "loss": 88.6894, + "step": 2190 + }, + { + "epoch": 0.35526129872227047, + "grad_norm": 1.05767822265625, + "learning_rate": 0.0001, + "loss": 89.5341, + "step": 2200 + }, + { + "epoch": 0.3568761228073717, + "grad_norm": 1.0373681783676147, + "learning_rate": 0.0001, + "loss": 89.7173, + "step": 2210 + }, + { + "epoch": 0.3584909468924729, + "grad_norm": 0.9098039269447327, + "learning_rate": 0.0001, + "loss": 89.2116, + "step": 2220 + }, + { + "epoch": 0.3601057709775741, + "grad_norm": 1.028130292892456, + "learning_rate": 0.0001, + "loss": 88.6627, + "step": 2230 + }, + { + "epoch": 0.36172059506267534, + "grad_norm": 1.0295171737670898, + "learning_rate": 0.0001, + "loss": 89.608, + "step": 2240 + }, + { + "epoch": 0.36333541914777656, + "grad_norm": 0.9720722436904907, + "learning_rate": 0.0001, + "loss": 88.3624, + "step": 2250 + }, + { + "epoch": 0.36495024323287784, + "grad_norm": 0.839654803276062, + "learning_rate": 0.0001, + "loss": 87.2983, + "step": 2260 + }, + { + "epoch": 0.36656506731797905, + "grad_norm": 1.1600219011306763, + "learning_rate": 0.0001, + "loss": 86.6004, + "step": 2270 + }, + { + "epoch": 0.3681798914030803, + "grad_norm": 1.0513924360275269, + "learning_rate": 0.0001, + "loss": 86.6062, + "step": 2280 + }, + { + "epoch": 0.3697947154881815, + "grad_norm": 0.9728474617004395, + "learning_rate": 0.0001, + "loss": 86.1856, + "step": 2290 + }, + { + "epoch": 0.3714095395732827, + "grad_norm": 1.0196386575698853, + "learning_rate": 0.0001, + "loss": 87.6088, + "step": 2300 + }, + { + "epoch": 0.373024363658384, + "grad_norm": 1.0439444780349731, + "learning_rate": 0.0001, + "loss": 85.2374, + "step": 2310 + }, + { + "epoch": 0.3746391877434852, + "grad_norm": 1.0020275115966797, + "learning_rate": 0.0001, + "loss": 86.0596, + "step": 2320 + }, + { + "epoch": 0.3762540118285864, + "grad_norm": 1.0107691287994385, + "learning_rate": 0.0001, + "loss": 86.9513, + "step": 2330 + }, + { + "epoch": 0.37786883591368764, + "grad_norm": 1.680018663406372, + "learning_rate": 0.0001, + "loss": 87.8885, + "step": 2340 + }, + { + "epoch": 0.37948365999878886, + "grad_norm": 0.9181709885597229, + "learning_rate": 0.0001, + "loss": 86.7662, + "step": 2350 + }, + { + "epoch": 0.38109848408389013, + "grad_norm": 0.9598278403282166, + "learning_rate": 0.0001, + "loss": 88.3775, + "step": 2360 + }, + { + "epoch": 0.38271330816899135, + "grad_norm": 1.0707905292510986, + "learning_rate": 0.0001, + "loss": 86.0997, + "step": 2370 + }, + { + "epoch": 0.38432813225409257, + "grad_norm": 1.125797152519226, + "learning_rate": 0.0001, + "loss": 85.8117, + "step": 2380 + }, + { + "epoch": 0.3859429563391938, + "grad_norm": 1.0251439809799194, + "learning_rate": 0.0001, + "loss": 86.0806, + "step": 2390 + }, + { + "epoch": 0.387557780424295, + "grad_norm": 1.0193125009536743, + "learning_rate": 0.0001, + "loss": 87.264, + "step": 2400 + }, + { + "epoch": 0.3891726045093963, + "grad_norm": 0.9309751391410828, + "learning_rate": 0.0001, + "loss": 85.9187, + "step": 2410 + }, + { + "epoch": 0.3907874285944975, + "grad_norm": 1.1025906801223755, + "learning_rate": 0.0001, + "loss": 86.4365, + "step": 2420 + }, + { + "epoch": 0.3924022526795987, + "grad_norm": 0.9880627393722534, + "learning_rate": 0.0001, + "loss": 86.2303, + "step": 2430 + }, + { + "epoch": 0.39401707676469994, + "grad_norm": 0.9520611763000488, + "learning_rate": 0.0001, + "loss": 84.9035, + "step": 2440 + }, + { + "epoch": 0.39563190084980115, + "grad_norm": 0.8641896843910217, + "learning_rate": 0.0001, + "loss": 85.4754, + "step": 2450 + }, + { + "epoch": 0.39724672493490243, + "grad_norm": 1.1162904500961304, + "learning_rate": 0.0001, + "loss": 85.6767, + "step": 2460 + }, + { + "epoch": 0.39886154902000365, + "grad_norm": 0.9073636531829834, + "learning_rate": 0.0001, + "loss": 85.4198, + "step": 2470 + }, + { + "epoch": 0.40047637310510487, + "grad_norm": 1.1135166883468628, + "learning_rate": 0.0001, + "loss": 86.0884, + "step": 2480 + }, + { + "epoch": 0.4020911971902061, + "grad_norm": 0.9273980259895325, + "learning_rate": 0.0001, + "loss": 85.5473, + "step": 2490 + }, + { + "epoch": 0.4037060212753073, + "grad_norm": 0.8653831481933594, + "learning_rate": 0.0001, + "loss": 87.279, + "step": 2500 + }, + { + "epoch": 0.4053208453604086, + "grad_norm": 0.8535159826278687, + "learning_rate": 0.0001, + "loss": 86.7072, + "step": 2510 + }, + { + "epoch": 0.4069356694455098, + "grad_norm": 0.8864684700965881, + "learning_rate": 0.0001, + "loss": 86.441, + "step": 2520 + }, + { + "epoch": 0.408550493530611, + "grad_norm": 0.8042080998420715, + "learning_rate": 0.0001, + "loss": 83.1706, + "step": 2530 + }, + { + "epoch": 0.41016531761571223, + "grad_norm": 1.0486373901367188, + "learning_rate": 0.0001, + "loss": 83.54, + "step": 2540 + }, + { + "epoch": 0.41178014170081345, + "grad_norm": 0.8087959289550781, + "learning_rate": 0.0001, + "loss": 85.7615, + "step": 2550 + }, + { + "epoch": 0.41339496578591467, + "grad_norm": 1.5032018423080444, + "learning_rate": 0.0001, + "loss": 85.7368, + "step": 2560 + }, + { + "epoch": 0.41500978987101594, + "grad_norm": 1.0786182880401611, + "learning_rate": 0.0001, + "loss": 86.3484, + "step": 2570 + }, + { + "epoch": 0.41662461395611716, + "grad_norm": 0.8744294047355652, + "learning_rate": 0.0001, + "loss": 83.5945, + "step": 2580 + }, + { + "epoch": 0.4182394380412184, + "grad_norm": 0.7840721607208252, + "learning_rate": 0.0001, + "loss": 86.3831, + "step": 2590 + }, + { + "epoch": 0.4198542621263196, + "grad_norm": 0.7977836728096008, + "learning_rate": 0.0001, + "loss": 83.6314, + "step": 2600 + }, + { + "epoch": 0.4214690862114208, + "grad_norm": 0.892484188079834, + "learning_rate": 0.0001, + "loss": 86.4223, + "step": 2610 + }, + { + "epoch": 0.4230839102965221, + "grad_norm": 0.912995457649231, + "learning_rate": 0.0001, + "loss": 83.8971, + "step": 2620 + }, + { + "epoch": 0.4246987343816233, + "grad_norm": 0.9566861987113953, + "learning_rate": 0.0001, + "loss": 85.291, + "step": 2630 + }, + { + "epoch": 0.42631355846672453, + "grad_norm": 0.8324154615402222, + "learning_rate": 0.0001, + "loss": 84.6233, + "step": 2640 + }, + { + "epoch": 0.42792838255182575, + "grad_norm": 0.9490360617637634, + "learning_rate": 0.0001, + "loss": 81.799, + "step": 2650 + }, + { + "epoch": 0.42954320663692697, + "grad_norm": 0.9744566679000854, + "learning_rate": 0.0001, + "loss": 85.4837, + "step": 2660 + }, + { + "epoch": 0.43115803072202824, + "grad_norm": 0.941614031791687, + "learning_rate": 0.0001, + "loss": 82.838, + "step": 2670 + }, + { + "epoch": 0.43277285480712946, + "grad_norm": 1.338364601135254, + "learning_rate": 0.0001, + "loss": 84.4224, + "step": 2680 + }, + { + "epoch": 0.4343876788922307, + "grad_norm": 0.8643816113471985, + "learning_rate": 0.0001, + "loss": 84.3959, + "step": 2690 + }, + { + "epoch": 0.4360025029773319, + "grad_norm": 1.0231473445892334, + "learning_rate": 0.0001, + "loss": 82.3144, + "step": 2700 + }, + { + "epoch": 0.4376173270624331, + "grad_norm": 0.9089650511741638, + "learning_rate": 0.0001, + "loss": 83.0326, + "step": 2710 + }, + { + "epoch": 0.4392321511475344, + "grad_norm": 0.893628716468811, + "learning_rate": 0.0001, + "loss": 83.0981, + "step": 2720 + }, + { + "epoch": 0.4408469752326356, + "grad_norm": 0.869076132774353, + "learning_rate": 0.0001, + "loss": 84.2571, + "step": 2730 + }, + { + "epoch": 0.4424617993177368, + "grad_norm": 0.7964235544204712, + "learning_rate": 0.0001, + "loss": 83.3329, + "step": 2740 + }, + { + "epoch": 0.44407662340283804, + "grad_norm": 3.1645138263702393, + "learning_rate": 0.0001, + "loss": 83.7232, + "step": 2750 + }, + { + "epoch": 0.44569144748793926, + "grad_norm": 0.934644877910614, + "learning_rate": 0.0001, + "loss": 85.0688, + "step": 2760 + }, + { + "epoch": 0.44730627157304054, + "grad_norm": 0.8120711445808411, + "learning_rate": 0.0001, + "loss": 82.7706, + "step": 2770 + }, + { + "epoch": 0.44892109565814176, + "grad_norm": 0.8161435127258301, + "learning_rate": 0.0001, + "loss": 84.4826, + "step": 2780 + }, + { + "epoch": 0.450535919743243, + "grad_norm": 0.8659493327140808, + "learning_rate": 0.0001, + "loss": 82.6222, + "step": 2790 + }, + { + "epoch": 0.4521507438283442, + "grad_norm": 0.9306702613830566, + "learning_rate": 0.0001, + "loss": 83.5897, + "step": 2800 + }, + { + "epoch": 0.4537655679134454, + "grad_norm": 2.2544431686401367, + "learning_rate": 0.0001, + "loss": 82.9603, + "step": 2810 + }, + { + "epoch": 0.4553803919985467, + "grad_norm": 0.8868395686149597, + "learning_rate": 0.0001, + "loss": 82.6674, + "step": 2820 + }, + { + "epoch": 0.4569952160836479, + "grad_norm": 0.9336455464363098, + "learning_rate": 0.0001, + "loss": 84.6974, + "step": 2830 + }, + { + "epoch": 0.4586100401687491, + "grad_norm": 0.8814165592193604, + "learning_rate": 0.0001, + "loss": 83.9329, + "step": 2840 + }, + { + "epoch": 0.46022486425385034, + "grad_norm": 0.8125660419464111, + "learning_rate": 0.0001, + "loss": 84.6168, + "step": 2850 + }, + { + "epoch": 0.46183968833895156, + "grad_norm": 0.9447190165519714, + "learning_rate": 0.0001, + "loss": 82.4098, + "step": 2860 + }, + { + "epoch": 0.4634545124240528, + "grad_norm": 0.8080969452857971, + "learning_rate": 0.0001, + "loss": 83.4076, + "step": 2870 + }, + { + "epoch": 0.46506933650915405, + "grad_norm": 1.0586694478988647, + "learning_rate": 0.0001, + "loss": 83.3295, + "step": 2880 + }, + { + "epoch": 0.46668416059425527, + "grad_norm": 0.8207664489746094, + "learning_rate": 0.0001, + "loss": 84.5056, + "step": 2890 + }, + { + "epoch": 0.4682989846793565, + "grad_norm": 1.0647571086883545, + "learning_rate": 0.0001, + "loss": 83.0766, + "step": 2900 + }, + { + "epoch": 0.4699138087644577, + "grad_norm": 0.9416915774345398, + "learning_rate": 0.0001, + "loss": 83.8166, + "step": 2910 + }, + { + "epoch": 0.4715286328495589, + "grad_norm": 0.7479362487792969, + "learning_rate": 0.0001, + "loss": 82.7507, + "step": 2920 + }, + { + "epoch": 0.4731434569346602, + "grad_norm": 1.0775514841079712, + "learning_rate": 0.0001, + "loss": 84.4641, + "step": 2930 + }, + { + "epoch": 0.4747582810197614, + "grad_norm": 0.8736841082572937, + "learning_rate": 0.0001, + "loss": 82.0451, + "step": 2940 + }, + { + "epoch": 0.47637310510486264, + "grad_norm": 0.9500495791435242, + "learning_rate": 0.0001, + "loss": 82.6425, + "step": 2950 + }, + { + "epoch": 0.47798792918996386, + "grad_norm": 0.8551210165023804, + "learning_rate": 0.0001, + "loss": 81.1971, + "step": 2960 + }, + { + "epoch": 0.4796027532750651, + "grad_norm": 0.8149814605712891, + "learning_rate": 0.0001, + "loss": 82.7919, + "step": 2970 + }, + { + "epoch": 0.48121757736016635, + "grad_norm": 0.9012249112129211, + "learning_rate": 0.0001, + "loss": 81.975, + "step": 2980 + }, + { + "epoch": 0.48283240144526757, + "grad_norm": 1.3784620761871338, + "learning_rate": 0.0001, + "loss": 81.541, + "step": 2990 + }, + { + "epoch": 0.4844472255303688, + "grad_norm": 0.7921801209449768, + "learning_rate": 0.0001, + "loss": 83.6215, + "step": 3000 + }, + { + "epoch": 0.48606204961547, + "grad_norm": 0.9051580429077148, + "learning_rate": 0.0001, + "loss": 82.4048, + "step": 3010 + }, + { + "epoch": 0.4876768737005712, + "grad_norm": 1.0717192888259888, + "learning_rate": 0.0001, + "loss": 82.4079, + "step": 3020 + }, + { + "epoch": 0.4892916977856725, + "grad_norm": 0.8971781730651855, + "learning_rate": 0.0001, + "loss": 83.3668, + "step": 3030 + }, + { + "epoch": 0.4909065218707737, + "grad_norm": 0.9635320901870728, + "learning_rate": 0.0001, + "loss": 82.8244, + "step": 3040 + }, + { + "epoch": 0.49252134595587493, + "grad_norm": 1.1009949445724487, + "learning_rate": 0.0001, + "loss": 82.2174, + "step": 3050 + }, + { + "epoch": 0.49413617004097615, + "grad_norm": 0.9805473685264587, + "learning_rate": 0.0001, + "loss": 83.7897, + "step": 3060 + }, + { + "epoch": 0.49575099412607737, + "grad_norm": 1.2107361555099487, + "learning_rate": 0.0001, + "loss": 82.5108, + "step": 3070 + }, + { + "epoch": 0.49736581821117865, + "grad_norm": 0.9171829223632812, + "learning_rate": 0.0001, + "loss": 80.9126, + "step": 3080 + }, + { + "epoch": 0.49898064229627986, + "grad_norm": 0.8093295097351074, + "learning_rate": 0.0001, + "loss": 83.8147, + "step": 3090 + }, + { + "epoch": 0.5005954663813811, + "grad_norm": 1.2166653871536255, + "learning_rate": 0.0001, + "loss": 82.0288, + "step": 3100 + }, + { + "epoch": 0.5022102904664824, + "grad_norm": 1.0471502542495728, + "learning_rate": 0.0001, + "loss": 82.0706, + "step": 3110 + }, + { + "epoch": 0.5038251145515835, + "grad_norm": 1.007645845413208, + "learning_rate": 0.0001, + "loss": 81.7387, + "step": 3120 + }, + { + "epoch": 0.5054399386366848, + "grad_norm": 0.7252638339996338, + "learning_rate": 0.0001, + "loss": 82.9107, + "step": 3130 + }, + { + "epoch": 0.507054762721786, + "grad_norm": 0.8565683364868164, + "learning_rate": 0.0001, + "loss": 80.2935, + "step": 3140 + }, + { + "epoch": 0.5086695868068872, + "grad_norm": 0.9905404448509216, + "learning_rate": 0.0001, + "loss": 82.5201, + "step": 3150 + }, + { + "epoch": 0.5102844108919885, + "grad_norm": 0.887428343296051, + "learning_rate": 0.0001, + "loss": 80.8528, + "step": 3160 + }, + { + "epoch": 0.5118992349770897, + "grad_norm": 1.062942624092102, + "learning_rate": 0.0001, + "loss": 80.7035, + "step": 3170 + }, + { + "epoch": 0.5135140590621909, + "grad_norm": 1.203599214553833, + "learning_rate": 0.0001, + "loss": 81.2521, + "step": 3180 + }, + { + "epoch": 0.5151288831472921, + "grad_norm": 0.8452383875846863, + "learning_rate": 0.0001, + "loss": 81.6079, + "step": 3190 + }, + { + "epoch": 0.5167437072323934, + "grad_norm": 1.2530587911605835, + "learning_rate": 0.0001, + "loss": 82.2148, + "step": 3200 + }, + { + "epoch": 0.5183585313174947, + "grad_norm": 0.9351627826690674, + "learning_rate": 0.0001, + "loss": 82.7857, + "step": 3210 + }, + { + "epoch": 0.5199733554025958, + "grad_norm": 1.0128077268600464, + "learning_rate": 0.0001, + "loss": 81.0532, + "step": 3220 + }, + { + "epoch": 0.5215881794876971, + "grad_norm": 0.9316917061805725, + "learning_rate": 0.0001, + "loss": 80.7443, + "step": 3230 + }, + { + "epoch": 0.5232030035727983, + "grad_norm": 1.502705693244934, + "learning_rate": 0.0001, + "loss": 80.0608, + "step": 3240 + }, + { + "epoch": 0.5248178276578995, + "grad_norm": 0.8166837692260742, + "learning_rate": 0.0001, + "loss": 81.2013, + "step": 3250 + }, + { + "epoch": 0.5264326517430008, + "grad_norm": 0.8210042119026184, + "learning_rate": 0.0001, + "loss": 80.2491, + "step": 3260 + }, + { + "epoch": 0.528047475828102, + "grad_norm": 0.7854771018028259, + "learning_rate": 0.0001, + "loss": 81.1017, + "step": 3270 + }, + { + "epoch": 0.5296622999132032, + "grad_norm": 0.8530565500259399, + "learning_rate": 0.0001, + "loss": 80.8123, + "step": 3280 + }, + { + "epoch": 0.5312771239983044, + "grad_norm": 0.9446498155593872, + "learning_rate": 0.0001, + "loss": 80.0029, + "step": 3290 + }, + { + "epoch": 0.5328919480834057, + "grad_norm": 0.9646537899971008, + "learning_rate": 0.0001, + "loss": 81.5413, + "step": 3300 + }, + { + "epoch": 0.5345067721685068, + "grad_norm": 0.9262019395828247, + "learning_rate": 0.0001, + "loss": 80.9399, + "step": 3310 + }, + { + "epoch": 0.5361215962536081, + "grad_norm": 0.8515486121177673, + "learning_rate": 0.0001, + "loss": 80.9421, + "step": 3320 + }, + { + "epoch": 0.5377364203387094, + "grad_norm": 1.679321050643921, + "learning_rate": 0.0001, + "loss": 82.4851, + "step": 3330 + }, + { + "epoch": 0.5393512444238105, + "grad_norm": 0.8289812207221985, + "learning_rate": 0.0001, + "loss": 80.5664, + "step": 3340 + }, + { + "epoch": 0.5409660685089118, + "grad_norm": 0.7463789582252502, + "learning_rate": 0.0001, + "loss": 81.5193, + "step": 3350 + }, + { + "epoch": 0.542580892594013, + "grad_norm": 0.8332446217536926, + "learning_rate": 0.0001, + "loss": 80.8091, + "step": 3360 + }, + { + "epoch": 0.5441957166791143, + "grad_norm": 0.7612847089767456, + "learning_rate": 0.0001, + "loss": 79.8893, + "step": 3370 + }, + { + "epoch": 0.5458105407642155, + "grad_norm": 1.0557948350906372, + "learning_rate": 0.0001, + "loss": 81.4305, + "step": 3380 + }, + { + "epoch": 0.5474253648493167, + "grad_norm": 0.7975788116455078, + "learning_rate": 0.0001, + "loss": 80.0105, + "step": 3390 + }, + { + "epoch": 0.549040188934418, + "grad_norm": 0.9049227237701416, + "learning_rate": 0.0001, + "loss": 81.2689, + "step": 3400 + }, + { + "epoch": 0.5506550130195191, + "grad_norm": 1.1195895671844482, + "learning_rate": 0.0001, + "loss": 80.7163, + "step": 3410 + }, + { + "epoch": 0.5522698371046204, + "grad_norm": 0.7506540417671204, + "learning_rate": 0.0001, + "loss": 79.0855, + "step": 3420 + }, + { + "epoch": 0.5538846611897217, + "grad_norm": 0.73989337682724, + "learning_rate": 0.0001, + "loss": 81.1865, + "step": 3430 + }, + { + "epoch": 0.5554994852748228, + "grad_norm": 0.8418401479721069, + "learning_rate": 0.0001, + "loss": 79.5714, + "step": 3440 + }, + { + "epoch": 0.5571143093599241, + "grad_norm": 0.9807226061820984, + "learning_rate": 0.0001, + "loss": 78.5699, + "step": 3450 + }, + { + "epoch": 0.5587291334450253, + "grad_norm": 1.2890545129776, + "learning_rate": 0.0001, + "loss": 81.0128, + "step": 3460 + }, + { + "epoch": 0.5603439575301266, + "grad_norm": 0.8593361973762512, + "learning_rate": 0.0001, + "loss": 80.1638, + "step": 3470 + }, + { + "epoch": 0.5619587816152278, + "grad_norm": 0.7766354084014893, + "learning_rate": 0.0001, + "loss": 81.5597, + "step": 3480 + }, + { + "epoch": 0.563573605700329, + "grad_norm": 0.7785626649856567, + "learning_rate": 0.0001, + "loss": 79.3862, + "step": 3490 + }, + { + "epoch": 0.5651884297854303, + "grad_norm": 0.8429279327392578, + "learning_rate": 0.0001, + "loss": 76.6294, + "step": 3500 + }, + { + "epoch": 0.5668032538705314, + "grad_norm": 0.8141765594482422, + "learning_rate": 0.0001, + "loss": 80.2584, + "step": 3510 + }, + { + "epoch": 0.5684180779556327, + "grad_norm": 0.7065490484237671, + "learning_rate": 0.0001, + "loss": 79.3626, + "step": 3520 + }, + { + "epoch": 0.570032902040734, + "grad_norm": 0.7002031803131104, + "learning_rate": 0.0001, + "loss": 79.283, + "step": 3530 + }, + { + "epoch": 0.5716477261258351, + "grad_norm": 0.8715317845344543, + "learning_rate": 0.0001, + "loss": 80.3145, + "step": 3540 + }, + { + "epoch": 0.5732625502109364, + "grad_norm": 0.9584882855415344, + "learning_rate": 0.0001, + "loss": 78.7443, + "step": 3550 + }, + { + "epoch": 0.5748773742960376, + "grad_norm": 0.7908946871757507, + "learning_rate": 0.0001, + "loss": 79.1906, + "step": 3560 + }, + { + "epoch": 0.5764921983811389, + "grad_norm": 0.8416746258735657, + "learning_rate": 0.0001, + "loss": 77.8172, + "step": 3570 + }, + { + "epoch": 0.5781070224662401, + "grad_norm": 0.823667049407959, + "learning_rate": 0.0001, + "loss": 81.864, + "step": 3580 + }, + { + "epoch": 0.5797218465513413, + "grad_norm": 1.4282617568969727, + "learning_rate": 0.0001, + "loss": 79.9052, + "step": 3590 + }, + { + "epoch": 0.5813366706364426, + "grad_norm": 0.8747813701629639, + "learning_rate": 0.0001, + "loss": 78.8234, + "step": 3600 + }, + { + "epoch": 0.5829514947215437, + "grad_norm": 0.8415511846542358, + "learning_rate": 0.0001, + "loss": 78.8171, + "step": 3610 + }, + { + "epoch": 0.584566318806645, + "grad_norm": 0.9014492034912109, + "learning_rate": 0.0001, + "loss": 80.5011, + "step": 3620 + }, + { + "epoch": 0.5861811428917463, + "grad_norm": 0.7535204887390137, + "learning_rate": 0.0001, + "loss": 79.4895, + "step": 3630 + }, + { + "epoch": 0.5877959669768474, + "grad_norm": 0.8196508884429932, + "learning_rate": 0.0001, + "loss": 79.7101, + "step": 3640 + }, + { + "epoch": 0.5894107910619487, + "grad_norm": 1.5975492000579834, + "learning_rate": 0.0001, + "loss": 79.0687, + "step": 3650 + }, + { + "epoch": 0.5910256151470499, + "grad_norm": 0.9251692295074463, + "learning_rate": 0.0001, + "loss": 80.2377, + "step": 3660 + }, + { + "epoch": 0.5926404392321512, + "grad_norm": 0.7143557071685791, + "learning_rate": 0.0001, + "loss": 78.0671, + "step": 3670 + }, + { + "epoch": 0.5942552633172524, + "grad_norm": 0.7876290678977966, + "learning_rate": 0.0001, + "loss": 79.1455, + "step": 3680 + }, + { + "epoch": 0.5958700874023536, + "grad_norm": 1.5361895561218262, + "learning_rate": 0.0001, + "loss": 78.1253, + "step": 3690 + }, + { + "epoch": 0.5974849114874549, + "grad_norm": 0.7637040019035339, + "learning_rate": 0.0001, + "loss": 77.4658, + "step": 3700 + }, + { + "epoch": 0.599099735572556, + "grad_norm": 0.8166250586509705, + "learning_rate": 0.0001, + "loss": 79.5806, + "step": 3710 + }, + { + "epoch": 0.6007145596576573, + "grad_norm": 1.2767380475997925, + "learning_rate": 0.0001, + "loss": 79.1894, + "step": 3720 + }, + { + "epoch": 0.6023293837427586, + "grad_norm": 1.3048640489578247, + "learning_rate": 0.0001, + "loss": 78.0593, + "step": 3730 + }, + { + "epoch": 0.6039442078278597, + "grad_norm": 0.7945445775985718, + "learning_rate": 0.0001, + "loss": 79.7623, + "step": 3740 + }, + { + "epoch": 0.605559031912961, + "grad_norm": 0.7366836667060852, + "learning_rate": 0.0001, + "loss": 77.9571, + "step": 3750 + }, + { + "epoch": 0.6071738559980622, + "grad_norm": 0.9126115441322327, + "learning_rate": 0.0001, + "loss": 79.9502, + "step": 3760 + }, + { + "epoch": 0.6087886800831634, + "grad_norm": 0.800355851650238, + "learning_rate": 0.0001, + "loss": 78.7244, + "step": 3770 + }, + { + "epoch": 0.6104035041682647, + "grad_norm": 0.8290980458259583, + "learning_rate": 0.0001, + "loss": 78.7724, + "step": 3780 + }, + { + "epoch": 0.6120183282533659, + "grad_norm": 0.8512316942214966, + "learning_rate": 0.0001, + "loss": 78.6277, + "step": 3790 + }, + { + "epoch": 0.6136331523384672, + "grad_norm": 0.8909353613853455, + "learning_rate": 0.0001, + "loss": 78.5956, + "step": 3800 + }, + { + "epoch": 0.6152479764235683, + "grad_norm": 0.8328800797462463, + "learning_rate": 0.0001, + "loss": 78.1323, + "step": 3810 + }, + { + "epoch": 0.6168628005086696, + "grad_norm": 0.7950423359870911, + "learning_rate": 0.0001, + "loss": 78.7529, + "step": 3820 + }, + { + "epoch": 0.6184776245937709, + "grad_norm": 0.9988545179367065, + "learning_rate": 0.0001, + "loss": 80.7727, + "step": 3830 + }, + { + "epoch": 0.620092448678872, + "grad_norm": 0.7243085503578186, + "learning_rate": 0.0001, + "loss": 76.7616, + "step": 3840 + }, + { + "epoch": 0.6217072727639733, + "grad_norm": 0.7674855589866638, + "learning_rate": 0.0001, + "loss": 79.1938, + "step": 3850 + }, + { + "epoch": 0.6233220968490745, + "grad_norm": 0.8979483842849731, + "learning_rate": 0.0001, + "loss": 79.1522, + "step": 3860 + }, + { + "epoch": 0.6249369209341757, + "grad_norm": 0.7759745717048645, + "learning_rate": 0.0001, + "loss": 77.7001, + "step": 3870 + }, + { + "epoch": 0.626551745019277, + "grad_norm": 0.9940371513366699, + "learning_rate": 0.0001, + "loss": 79.8712, + "step": 3880 + }, + { + "epoch": 0.6281665691043782, + "grad_norm": 0.8103354573249817, + "learning_rate": 0.0001, + "loss": 77.5229, + "step": 3890 + }, + { + "epoch": 0.6297813931894795, + "grad_norm": 0.9188411831855774, + "learning_rate": 0.0001, + "loss": 78.2933, + "step": 3900 + }, + { + "epoch": 0.6313962172745806, + "grad_norm": 0.823017954826355, + "learning_rate": 0.0001, + "loss": 77.8692, + "step": 3910 + }, + { + "epoch": 0.6330110413596819, + "grad_norm": 0.8833358287811279, + "learning_rate": 0.0001, + "loss": 78.4465, + "step": 3920 + }, + { + "epoch": 0.634625865444783, + "grad_norm": 0.6880685687065125, + "learning_rate": 0.0001, + "loss": 78.8881, + "step": 3930 + }, + { + "epoch": 0.6362406895298843, + "grad_norm": 0.778455913066864, + "learning_rate": 0.0001, + "loss": 76.0277, + "step": 3940 + }, + { + "epoch": 0.6378555136149856, + "grad_norm": 0.8132772445678711, + "learning_rate": 0.0001, + "loss": 79.1284, + "step": 3950 + }, + { + "epoch": 0.6394703377000868, + "grad_norm": 0.7703788876533508, + "learning_rate": 0.0001, + "loss": 76.9013, + "step": 3960 + }, + { + "epoch": 0.641085161785188, + "grad_norm": 0.7876151204109192, + "learning_rate": 0.0001, + "loss": 77.1354, + "step": 3970 + }, + { + "epoch": 0.6426999858702892, + "grad_norm": 0.8027064800262451, + "learning_rate": 0.0001, + "loss": 77.4936, + "step": 3980 + }, + { + "epoch": 0.6443148099553905, + "grad_norm": 0.8261818885803223, + "learning_rate": 0.0001, + "loss": 76.7634, + "step": 3990 + }, + { + "epoch": 0.6459296340404918, + "grad_norm": 0.7024852633476257, + "learning_rate": 0.0001, + "loss": 79.951, + "step": 4000 + }, + { + "epoch": 0.6475444581255929, + "grad_norm": 1.015307068824768, + "learning_rate": 0.0001, + "loss": 76.0354, + "step": 4010 + }, + { + "epoch": 0.6491592822106942, + "grad_norm": 0.715346097946167, + "learning_rate": 0.0001, + "loss": 78.9021, + "step": 4020 + }, + { + "epoch": 0.6507741062957954, + "grad_norm": 0.8994929790496826, + "learning_rate": 0.0001, + "loss": 77.0587, + "step": 4030 + }, + { + "epoch": 0.6523889303808966, + "grad_norm": 0.7683436274528503, + "learning_rate": 0.0001, + "loss": 78.6943, + "step": 4040 + }, + { + "epoch": 0.6540037544659979, + "grad_norm": 0.7273952960968018, + "learning_rate": 0.0001, + "loss": 77.8396, + "step": 4050 + }, + { + "epoch": 0.6556185785510991, + "grad_norm": 1.0676878690719604, + "learning_rate": 0.0001, + "loss": 79.3449, + "step": 4060 + }, + { + "epoch": 0.6572334026362003, + "grad_norm": 0.6916168928146362, + "learning_rate": 0.0001, + "loss": 76.5462, + "step": 4070 + }, + { + "epoch": 0.6588482267213015, + "grad_norm": 0.6466719508171082, + "learning_rate": 0.0001, + "loss": 77.9354, + "step": 4080 + }, + { + "epoch": 0.6604630508064028, + "grad_norm": 0.7406665086746216, + "learning_rate": 0.0001, + "loss": 76.0135, + "step": 4090 + }, + { + "epoch": 0.662077874891504, + "grad_norm": 0.7104353904724121, + "learning_rate": 0.0001, + "loss": 78.3727, + "step": 4100 + }, + { + "epoch": 0.6636926989766052, + "grad_norm": 0.8328452110290527, + "learning_rate": 0.0001, + "loss": 77.5354, + "step": 4110 + }, + { + "epoch": 0.6653075230617065, + "grad_norm": 0.7012260556221008, + "learning_rate": 0.0001, + "loss": 78.7406, + "step": 4120 + }, + { + "epoch": 0.6669223471468076, + "grad_norm": 1.0436632633209229, + "learning_rate": 0.0001, + "loss": 78.0063, + "step": 4130 + }, + { + "epoch": 0.6685371712319089, + "grad_norm": 0.8471246361732483, + "learning_rate": 0.0001, + "loss": 77.8871, + "step": 4140 + }, + { + "epoch": 0.6701519953170102, + "grad_norm": 0.7378994822502136, + "learning_rate": 0.0001, + "loss": 76.9109, + "step": 4150 + }, + { + "epoch": 0.6717668194021114, + "grad_norm": 0.7778745889663696, + "learning_rate": 0.0001, + "loss": 78.3374, + "step": 4160 + }, + { + "epoch": 0.6733816434872126, + "grad_norm": 0.7841216325759888, + "learning_rate": 0.0001, + "loss": 78.1657, + "step": 4170 + }, + { + "epoch": 0.6749964675723138, + "grad_norm": 1.0027995109558105, + "learning_rate": 0.0001, + "loss": 76.6232, + "step": 4180 + }, + { + "epoch": 0.6766112916574151, + "grad_norm": 1.0034840106964111, + "learning_rate": 0.0001, + "loss": 79.5785, + "step": 4190 + }, + { + "epoch": 0.6782261157425163, + "grad_norm": 0.8399350643157959, + "learning_rate": 0.0001, + "loss": 76.8101, + "step": 4200 + }, + { + "epoch": 0.6798409398276175, + "grad_norm": 0.8864424824714661, + "learning_rate": 0.0001, + "loss": 77.8671, + "step": 4210 + }, + { + "epoch": 0.6814557639127188, + "grad_norm": 0.8322188258171082, + "learning_rate": 0.0001, + "loss": 77.6597, + "step": 4220 + }, + { + "epoch": 0.68307058799782, + "grad_norm": 0.8962367177009583, + "learning_rate": 0.0001, + "loss": 77.7324, + "step": 4230 + }, + { + "epoch": 0.6846854120829212, + "grad_norm": 0.9727205038070679, + "learning_rate": 0.0001, + "loss": 77.8531, + "step": 4240 + }, + { + "epoch": 0.6863002361680225, + "grad_norm": 0.956584095954895, + "learning_rate": 0.0001, + "loss": 78.709, + "step": 4250 + }, + { + "epoch": 0.6879150602531237, + "grad_norm": 0.8531600832939148, + "learning_rate": 0.0001, + "loss": 78.7755, + "step": 4260 + }, + { + "epoch": 0.6895298843382249, + "grad_norm": 0.8059844374656677, + "learning_rate": 0.0001, + "loss": 77.1636, + "step": 4270 + }, + { + "epoch": 0.6911447084233261, + "grad_norm": 0.8373456597328186, + "learning_rate": 0.0001, + "loss": 78.6561, + "step": 4280 + }, + { + "epoch": 0.6927595325084274, + "grad_norm": 0.7819933891296387, + "learning_rate": 0.0001, + "loss": 77.6061, + "step": 4290 + }, + { + "epoch": 0.6943743565935286, + "grad_norm": 0.8829817771911621, + "learning_rate": 0.0001, + "loss": 77.3387, + "step": 4300 + }, + { + "epoch": 0.6959891806786298, + "grad_norm": 0.791772723197937, + "learning_rate": 0.0001, + "loss": 77.4368, + "step": 4310 + }, + { + "epoch": 0.6976040047637311, + "grad_norm": 0.9949231147766113, + "learning_rate": 0.0001, + "loss": 73.4359, + "step": 4320 + }, + { + "epoch": 0.6992188288488322, + "grad_norm": 0.8676188588142395, + "learning_rate": 0.0001, + "loss": 78.4836, + "step": 4330 + }, + { + "epoch": 0.7008336529339335, + "grad_norm": 1.1704007387161255, + "learning_rate": 0.0001, + "loss": 76.5091, + "step": 4340 + }, + { + "epoch": 0.7024484770190348, + "grad_norm": 0.8877903819084167, + "learning_rate": 0.0001, + "loss": 78.021, + "step": 4350 + }, + { + "epoch": 0.704063301104136, + "grad_norm": 1.381914496421814, + "learning_rate": 0.0001, + "loss": 77.8544, + "step": 4360 + }, + { + "epoch": 0.7056781251892372, + "grad_norm": 0.835023045539856, + "learning_rate": 0.0001, + "loss": 76.1552, + "step": 4370 + }, + { + "epoch": 0.7072929492743384, + "grad_norm": 0.9814149141311646, + "learning_rate": 0.0001, + "loss": 76.7646, + "step": 4380 + }, + { + "epoch": 0.7089077733594397, + "grad_norm": 0.8012816309928894, + "learning_rate": 0.0001, + "loss": 75.794, + "step": 4390 + }, + { + "epoch": 0.7105225974445409, + "grad_norm": 0.8340234160423279, + "learning_rate": 0.0001, + "loss": 77.4417, + "step": 4400 + }, + { + "epoch": 0.7121374215296421, + "grad_norm": 0.7257950305938721, + "learning_rate": 0.0001, + "loss": 77.1807, + "step": 4410 + }, + { + "epoch": 0.7137522456147434, + "grad_norm": 0.7342708706855774, + "learning_rate": 0.0001, + "loss": 75.5091, + "step": 4420 + }, + { + "epoch": 0.7153670696998445, + "grad_norm": 0.8298904895782471, + "learning_rate": 0.0001, + "loss": 76.6651, + "step": 4430 + }, + { + "epoch": 0.7169818937849458, + "grad_norm": 0.8564396500587463, + "learning_rate": 0.0001, + "loss": 76.0055, + "step": 4440 + }, + { + "epoch": 0.7185967178700471, + "grad_norm": 0.9145296812057495, + "learning_rate": 0.0001, + "loss": 76.0369, + "step": 4450 + }, + { + "epoch": 0.7202115419551482, + "grad_norm": 0.8074191212654114, + "learning_rate": 0.0001, + "loss": 77.4436, + "step": 4460 + }, + { + "epoch": 0.7218263660402495, + "grad_norm": 0.8212682008743286, + "learning_rate": 0.0001, + "loss": 76.3129, + "step": 4470 + }, + { + "epoch": 0.7234411901253507, + "grad_norm": 0.751323938369751, + "learning_rate": 0.0001, + "loss": 77.603, + "step": 4480 + }, + { + "epoch": 0.725056014210452, + "grad_norm": 0.9664170145988464, + "learning_rate": 0.0001, + "loss": 77.481, + "step": 4490 + }, + { + "epoch": 0.7266708382955531, + "grad_norm": 0.7369455695152283, + "learning_rate": 0.0001, + "loss": 76.3238, + "step": 4500 + }, + { + "epoch": 0.7282856623806544, + "grad_norm": 0.8448513746261597, + "learning_rate": 0.0001, + "loss": 76.5679, + "step": 4510 + }, + { + "epoch": 0.7299004864657557, + "grad_norm": 0.8605676293373108, + "learning_rate": 0.0001, + "loss": 75.7667, + "step": 4520 + }, + { + "epoch": 0.7315153105508568, + "grad_norm": 0.8034664392471313, + "learning_rate": 0.0001, + "loss": 76.2163, + "step": 4530 + }, + { + "epoch": 0.7331301346359581, + "grad_norm": 0.7846826910972595, + "learning_rate": 0.0001, + "loss": 73.5979, + "step": 4540 + }, + { + "epoch": 0.7347449587210593, + "grad_norm": 0.7422606348991394, + "learning_rate": 0.0001, + "loss": 76.2334, + "step": 4550 + }, + { + "epoch": 0.7363597828061605, + "grad_norm": 1.4478483200073242, + "learning_rate": 0.0001, + "loss": 77.6168, + "step": 4560 + }, + { + "epoch": 0.7379746068912618, + "grad_norm": 2.0957767963409424, + "learning_rate": 0.0001, + "loss": 76.3199, + "step": 4570 + }, + { + "epoch": 0.739589430976363, + "grad_norm": 0.6757177114486694, + "learning_rate": 0.0001, + "loss": 77.2923, + "step": 4580 + }, + { + "epoch": 0.7412042550614643, + "grad_norm": 0.9281920790672302, + "learning_rate": 0.0001, + "loss": 76.9963, + "step": 4590 + }, + { + "epoch": 0.7428190791465654, + "grad_norm": 0.8702669143676758, + "learning_rate": 0.0001, + "loss": 76.2553, + "step": 4600 + }, + { + "epoch": 0.7444339032316667, + "grad_norm": 0.7976512312889099, + "learning_rate": 0.0001, + "loss": 74.7622, + "step": 4610 + }, + { + "epoch": 0.746048727316768, + "grad_norm": 0.9116189479827881, + "learning_rate": 0.0001, + "loss": 76.3198, + "step": 4620 + }, + { + "epoch": 0.7476635514018691, + "grad_norm": 0.6239303946495056, + "learning_rate": 0.0001, + "loss": 75.7678, + "step": 4630 + }, + { + "epoch": 0.7492783754869704, + "grad_norm": 0.8380976915359497, + "learning_rate": 0.0001, + "loss": 76.8305, + "step": 4640 + }, + { + "epoch": 0.7508931995720716, + "grad_norm": 0.7394141554832458, + "learning_rate": 0.0001, + "loss": 76.5033, + "step": 4650 + }, + { + "epoch": 0.7525080236571728, + "grad_norm": 1.0694036483764648, + "learning_rate": 0.0001, + "loss": 75.0084, + "step": 4660 + }, + { + "epoch": 0.7541228477422741, + "grad_norm": 0.7530115246772766, + "learning_rate": 0.0001, + "loss": 74.4977, + "step": 4670 + }, + { + "epoch": 0.7557376718273753, + "grad_norm": 1.9561909437179565, + "learning_rate": 0.0001, + "loss": 74.7077, + "step": 4680 + }, + { + "epoch": 0.7573524959124766, + "grad_norm": 0.625341534614563, + "learning_rate": 0.0001, + "loss": 76.1238, + "step": 4690 + }, + { + "epoch": 0.7589673199975777, + "grad_norm": 0.9790500998497009, + "learning_rate": 0.0001, + "loss": 76.3997, + "step": 4700 + }, + { + "epoch": 0.760582144082679, + "grad_norm": 0.7087915539741516, + "learning_rate": 0.0001, + "loss": 75.6893, + "step": 4710 + }, + { + "epoch": 0.7621969681677803, + "grad_norm": 1.0995875597000122, + "learning_rate": 0.0001, + "loss": 74.8588, + "step": 4720 + }, + { + "epoch": 0.7638117922528814, + "grad_norm": 1.0309056043624878, + "learning_rate": 0.0001, + "loss": 73.5934, + "step": 4730 + }, + { + "epoch": 0.7654266163379827, + "grad_norm": 1.0132921934127808, + "learning_rate": 0.0001, + "loss": 73.9868, + "step": 4740 + }, + { + "epoch": 0.7670414404230839, + "grad_norm": 0.7138039469718933, + "learning_rate": 0.0001, + "loss": 76.4459, + "step": 4750 + }, + { + "epoch": 0.7686562645081851, + "grad_norm": 0.7055657505989075, + "learning_rate": 0.0001, + "loss": 75.1068, + "step": 4760 + }, + { + "epoch": 0.7702710885932864, + "grad_norm": 0.7398490905761719, + "learning_rate": 0.0001, + "loss": 75.3697, + "step": 4770 + }, + { + "epoch": 0.7718859126783876, + "grad_norm": 0.6836295127868652, + "learning_rate": 0.0001, + "loss": 77.4988, + "step": 4780 + }, + { + "epoch": 0.7735007367634888, + "grad_norm": 0.7552574276924133, + "learning_rate": 0.0001, + "loss": 75.149, + "step": 4790 + }, + { + "epoch": 0.77511556084859, + "grad_norm": 0.7930851578712463, + "learning_rate": 0.0001, + "loss": 73.8661, + "step": 4800 + }, + { + "epoch": 0.7767303849336913, + "grad_norm": 0.7549886107444763, + "learning_rate": 0.0001, + "loss": 75.042, + "step": 4810 + }, + { + "epoch": 0.7783452090187926, + "grad_norm": 0.9932214617729187, + "learning_rate": 0.0001, + "loss": 75.6619, + "step": 4820 + }, + { + "epoch": 0.7799600331038937, + "grad_norm": 0.7685471773147583, + "learning_rate": 0.0001, + "loss": 75.8404, + "step": 4830 + }, + { + "epoch": 0.781574857188995, + "grad_norm": 0.828737199306488, + "learning_rate": 0.0001, + "loss": 75.5839, + "step": 4840 + }, + { + "epoch": 0.7831896812740962, + "grad_norm": 0.8575420379638672, + "learning_rate": 0.0001, + "loss": 75.7889, + "step": 4850 + }, + { + "epoch": 0.7848045053591974, + "grad_norm": 0.7053834199905396, + "learning_rate": 0.0001, + "loss": 75.9372, + "step": 4860 + }, + { + "epoch": 0.7864193294442987, + "grad_norm": 0.7861505150794983, + "learning_rate": 0.0001, + "loss": 76.1532, + "step": 4870 + }, + { + "epoch": 0.7880341535293999, + "grad_norm": 0.8357386589050293, + "learning_rate": 0.0001, + "loss": 75.8536, + "step": 4880 + }, + { + "epoch": 0.7896489776145011, + "grad_norm": 3.448834180831909, + "learning_rate": 0.0001, + "loss": 74.1794, + "step": 4890 + }, + { + "epoch": 0.7912638016996023, + "grad_norm": 0.813522458076477, + "learning_rate": 0.0001, + "loss": 74.5202, + "step": 4900 + }, + { + "epoch": 0.7928786257847036, + "grad_norm": 0.840230405330658, + "learning_rate": 0.0001, + "loss": 74.6634, + "step": 4910 + }, + { + "epoch": 0.7944934498698049, + "grad_norm": 0.7855745553970337, + "learning_rate": 0.0001, + "loss": 73.5369, + "step": 4920 + }, + { + "epoch": 0.796108273954906, + "grad_norm": 0.7708212733268738, + "learning_rate": 0.0001, + "loss": 74.6225, + "step": 4930 + }, + { + "epoch": 0.7977230980400073, + "grad_norm": 0.7140830159187317, + "learning_rate": 0.0001, + "loss": 75.0585, + "step": 4940 + }, + { + "epoch": 0.7993379221251085, + "grad_norm": 0.7960936427116394, + "learning_rate": 0.0001, + "loss": 74.0574, + "step": 4950 + }, + { + "epoch": 0.8009527462102097, + "grad_norm": 0.7269165515899658, + "learning_rate": 0.0001, + "loss": 74.0379, + "step": 4960 + }, + { + "epoch": 0.802567570295311, + "grad_norm": 0.8505013585090637, + "learning_rate": 0.0001, + "loss": 74.7448, + "step": 4970 + }, + { + "epoch": 0.8041823943804122, + "grad_norm": 0.8935654759407043, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 4980 + }, + { + "epoch": 0.8057972184655134, + "grad_norm": 0.9447595477104187, + "learning_rate": 0.0001, + "loss": 75.9674, + "step": 4990 + }, + { + "epoch": 0.8074120425506146, + "grad_norm": 1.6739715337753296, + "learning_rate": 0.0001, + "loss": 73.9035, + "step": 5000 + }, + { + "epoch": 0.8090268666357159, + "grad_norm": 0.861902117729187, + "learning_rate": 0.0001, + "loss": 74.906, + "step": 5010 + }, + { + "epoch": 0.8106416907208172, + "grad_norm": 0.8328655362129211, + "learning_rate": 0.0001, + "loss": 74.1493, + "step": 5020 + }, + { + "epoch": 0.8122565148059183, + "grad_norm": 0.8193135261535645, + "learning_rate": 0.0001, + "loss": 74.6614, + "step": 5030 + }, + { + "epoch": 0.8138713388910196, + "grad_norm": 0.6954565048217773, + "learning_rate": 0.0001, + "loss": 73.7366, + "step": 5040 + }, + { + "epoch": 0.8154861629761208, + "grad_norm": 0.7381289005279541, + "learning_rate": 0.0001, + "loss": 73.5086, + "step": 5050 + }, + { + "epoch": 0.817100987061222, + "grad_norm": 1.090443730354309, + "learning_rate": 0.0001, + "loss": 75.0695, + "step": 5060 + }, + { + "epoch": 0.8187158111463233, + "grad_norm": 1.2544814348220825, + "learning_rate": 0.0001, + "loss": 72.89, + "step": 5070 + }, + { + "epoch": 0.8203306352314245, + "grad_norm": 0.632146418094635, + "learning_rate": 0.0001, + "loss": 74.8604, + "step": 5080 + }, + { + "epoch": 0.8219454593165257, + "grad_norm": 1.1038259267807007, + "learning_rate": 0.0001, + "loss": 74.7354, + "step": 5090 + }, + { + "epoch": 0.8235602834016269, + "grad_norm": 0.6886508464813232, + "learning_rate": 0.0001, + "loss": 75.4243, + "step": 5100 + }, + { + "epoch": 0.8251751074867282, + "grad_norm": 0.8367568850517273, + "learning_rate": 0.0001, + "loss": 75.2732, + "step": 5110 + }, + { + "epoch": 0.8267899315718293, + "grad_norm": 0.9302715063095093, + "learning_rate": 0.0001, + "loss": 75.7924, + "step": 5120 + }, + { + "epoch": 0.8284047556569306, + "grad_norm": 0.6871219873428345, + "learning_rate": 0.0001, + "loss": 74.2905, + "step": 5130 + }, + { + "epoch": 0.8300195797420319, + "grad_norm": 0.7319445013999939, + "learning_rate": 0.0001, + "loss": 73.7774, + "step": 5140 + }, + { + "epoch": 0.831634403827133, + "grad_norm": 0.7184522151947021, + "learning_rate": 0.0001, + "loss": 73.0554, + "step": 5150 + }, + { + "epoch": 0.8332492279122343, + "grad_norm": 0.8462343811988831, + "learning_rate": 0.0001, + "loss": 73.7615, + "step": 5160 + }, + { + "epoch": 0.8348640519973355, + "grad_norm": 0.9436241984367371, + "learning_rate": 0.0001, + "loss": 73.9464, + "step": 5170 + }, + { + "epoch": 0.8364788760824368, + "grad_norm": 0.9079630374908447, + "learning_rate": 0.0001, + "loss": 73.2727, + "step": 5180 + }, + { + "epoch": 0.838093700167538, + "grad_norm": 0.8280741572380066, + "learning_rate": 0.0001, + "loss": 73.9751, + "step": 5190 + }, + { + "epoch": 0.8397085242526392, + "grad_norm": 0.6475157141685486, + "learning_rate": 0.0001, + "loss": 74.7105, + "step": 5200 + }, + { + "epoch": 0.8413233483377405, + "grad_norm": 0.7090312838554382, + "learning_rate": 0.0001, + "loss": 73.6489, + "step": 5210 + }, + { + "epoch": 0.8429381724228416, + "grad_norm": 1.20967698097229, + "learning_rate": 0.0001, + "loss": 75.1335, + "step": 5220 + }, + { + "epoch": 0.8445529965079429, + "grad_norm": 2.053532600402832, + "learning_rate": 0.0001, + "loss": 75.3111, + "step": 5230 + }, + { + "epoch": 0.8461678205930442, + "grad_norm": 0.7563302516937256, + "learning_rate": 0.0001, + "loss": 72.9935, + "step": 5240 + }, + { + "epoch": 0.8477826446781453, + "grad_norm": 0.7801489233970642, + "learning_rate": 0.0001, + "loss": 74.7856, + "step": 5250 + }, + { + "epoch": 0.8493974687632466, + "grad_norm": 0.737870454788208, + "learning_rate": 0.0001, + "loss": 75.6281, + "step": 5260 + }, + { + "epoch": 0.8510122928483478, + "grad_norm": 0.704352617263794, + "learning_rate": 0.0001, + "loss": 75.4598, + "step": 5270 + }, + { + "epoch": 0.8526271169334491, + "grad_norm": 0.6452057361602783, + "learning_rate": 0.0001, + "loss": 74.0341, + "step": 5280 + }, + { + "epoch": 0.8542419410185503, + "grad_norm": 0.6863471865653992, + "learning_rate": 0.0001, + "loss": 74.3727, + "step": 5290 + }, + { + "epoch": 0.8558567651036515, + "grad_norm": 9.756170272827148, + "learning_rate": 0.0001, + "loss": 75.5231, + "step": 5300 + }, + { + "epoch": 0.8574715891887528, + "grad_norm": 0.8624544739723206, + "learning_rate": 0.0001, + "loss": 74.7821, + "step": 5310 + }, + { + "epoch": 0.8590864132738539, + "grad_norm": 0.6637155413627625, + "learning_rate": 0.0001, + "loss": 74.1153, + "step": 5320 + }, + { + "epoch": 0.8607012373589552, + "grad_norm": 1.0764445066452026, + "learning_rate": 0.0001, + "loss": 73.7208, + "step": 5330 + }, + { + "epoch": 0.8623160614440565, + "grad_norm": 0.8567569255828857, + "learning_rate": 0.0001, + "loss": 73.1399, + "step": 5340 + }, + { + "epoch": 0.8639308855291576, + "grad_norm": 0.7681576013565063, + "learning_rate": 0.0001, + "loss": 73.7577, + "step": 5350 + }, + { + "epoch": 0.8655457096142589, + "grad_norm": 0.9336384534835815, + "learning_rate": 0.0001, + "loss": 74.1022, + "step": 5360 + }, + { + "epoch": 0.8671605336993601, + "grad_norm": 0.8603724837303162, + "learning_rate": 0.0001, + "loss": 75.9579, + "step": 5370 + }, + { + "epoch": 0.8687753577844614, + "grad_norm": 0.77194744348526, + "learning_rate": 0.0001, + "loss": 72.4799, + "step": 5380 + }, + { + "epoch": 0.8703901818695626, + "grad_norm": 0.7480376958847046, + "learning_rate": 0.0001, + "loss": 76.1257, + "step": 5390 + }, + { + "epoch": 0.8720050059546638, + "grad_norm": 0.7841023206710815, + "learning_rate": 0.0001, + "loss": 73.1512, + "step": 5400 + }, + { + "epoch": 0.8736198300397651, + "grad_norm": 0.8741798996925354, + "learning_rate": 0.0001, + "loss": 74.2151, + "step": 5410 + }, + { + "epoch": 0.8752346541248662, + "grad_norm": 0.9762082099914551, + "learning_rate": 0.0001, + "loss": 76.7138, + "step": 5420 + }, + { + "epoch": 0.8768494782099675, + "grad_norm": 0.7670348286628723, + "learning_rate": 0.0001, + "loss": 72.9567, + "step": 5430 + }, + { + "epoch": 0.8784643022950688, + "grad_norm": 0.7449469566345215, + "learning_rate": 0.0001, + "loss": 74.8841, + "step": 5440 + }, + { + "epoch": 0.8800791263801699, + "grad_norm": 0.6984641551971436, + "learning_rate": 0.0001, + "loss": 73.8906, + "step": 5450 + }, + { + "epoch": 0.8816939504652712, + "grad_norm": 0.7854361534118652, + "learning_rate": 0.0001, + "loss": 74.626, + "step": 5460 + }, + { + "epoch": 0.8833087745503724, + "grad_norm": 0.9584240317344666, + "learning_rate": 0.0001, + "loss": 74.7048, + "step": 5470 + }, + { + "epoch": 0.8849235986354737, + "grad_norm": 0.6040284037590027, + "learning_rate": 0.0001, + "loss": 74.7194, + "step": 5480 + }, + { + "epoch": 0.8865384227205749, + "grad_norm": 0.7756215929985046, + "learning_rate": 0.0001, + "loss": 73.8178, + "step": 5490 + }, + { + "epoch": 0.8881532468056761, + "grad_norm": 0.9070586562156677, + "learning_rate": 0.0001, + "loss": 72.8309, + "step": 5500 + }, + { + "epoch": 0.8897680708907774, + "grad_norm": 0.8464986681938171, + "learning_rate": 0.0001, + "loss": 73.8847, + "step": 5510 + }, + { + "epoch": 0.8913828949758785, + "grad_norm": 0.7968605160713196, + "learning_rate": 0.0001, + "loss": 73.1691, + "step": 5520 + }, + { + "epoch": 0.8929977190609798, + "grad_norm": 0.9934583306312561, + "learning_rate": 0.0001, + "loss": 76.2662, + "step": 5530 + }, + { + "epoch": 0.8946125431460811, + "grad_norm": 0.7272152900695801, + "learning_rate": 0.0001, + "loss": 75.5295, + "step": 5540 + }, + { + "epoch": 0.8962273672311822, + "grad_norm": 0.857345461845398, + "learning_rate": 0.0001, + "loss": 75.3703, + "step": 5550 + }, + { + "epoch": 0.8978421913162835, + "grad_norm": 0.713918924331665, + "learning_rate": 0.0001, + "loss": 73.4563, + "step": 5560 + }, + { + "epoch": 0.8994570154013847, + "grad_norm": 0.759233832359314, + "learning_rate": 0.0001, + "loss": 73.2744, + "step": 5570 + }, + { + "epoch": 0.901071839486486, + "grad_norm": 0.7085754871368408, + "learning_rate": 0.0001, + "loss": 74.7792, + "step": 5580 + }, + { + "epoch": 0.9026866635715872, + "grad_norm": 0.7370456457138062, + "learning_rate": 0.0001, + "loss": 74.4216, + "step": 5590 + }, + { + "epoch": 0.9043014876566884, + "grad_norm": 0.779746949672699, + "learning_rate": 0.0001, + "loss": 75.9822, + "step": 5600 + }, + { + "epoch": 0.9059163117417897, + "grad_norm": 0.7915763854980469, + "learning_rate": 0.0001, + "loss": 74.3706, + "step": 5610 + }, + { + "epoch": 0.9075311358268908, + "grad_norm": 0.7886125445365906, + "learning_rate": 0.0001, + "loss": 72.9453, + "step": 5620 + }, + { + "epoch": 0.9091459599119921, + "grad_norm": 0.7804751992225647, + "learning_rate": 0.0001, + "loss": 73.514, + "step": 5630 + }, + { + "epoch": 0.9107607839970934, + "grad_norm": 0.7168806195259094, + "learning_rate": 0.0001, + "loss": 71.4159, + "step": 5640 + }, + { + "epoch": 0.9123756080821945, + "grad_norm": 0.7553626894950867, + "learning_rate": 0.0001, + "loss": 74.4353, + "step": 5650 + }, + { + "epoch": 0.9139904321672958, + "grad_norm": 0.6688916683197021, + "learning_rate": 0.0001, + "loss": 74.3786, + "step": 5660 + }, + { + "epoch": 0.915605256252397, + "grad_norm": 0.7425532341003418, + "learning_rate": 0.0001, + "loss": 73.3355, + "step": 5670 + }, + { + "epoch": 0.9172200803374982, + "grad_norm": 0.6473882794380188, + "learning_rate": 0.0001, + "loss": 74.2315, + "step": 5680 + }, + { + "epoch": 0.9188349044225994, + "grad_norm": 0.7222439050674438, + "learning_rate": 0.0001, + "loss": 74.2885, + "step": 5690 + }, + { + "epoch": 0.9204497285077007, + "grad_norm": 1.5716948509216309, + "learning_rate": 0.0001, + "loss": 74.6739, + "step": 5700 + }, + { + "epoch": 0.922064552592802, + "grad_norm": 0.8077759742736816, + "learning_rate": 0.0001, + "loss": 72.6617, + "step": 5710 + }, + { + "epoch": 0.9236793766779031, + "grad_norm": 0.6948631405830383, + "learning_rate": 0.0001, + "loss": 73.6247, + "step": 5720 + }, + { + "epoch": 0.9252942007630044, + "grad_norm": 0.8823757767677307, + "learning_rate": 0.0001, + "loss": 73.2536, + "step": 5730 + }, + { + "epoch": 0.9269090248481056, + "grad_norm": 0.7066764235496521, + "learning_rate": 0.0001, + "loss": 74.1748, + "step": 5740 + }, + { + "epoch": 0.9285238489332068, + "grad_norm": 0.6979793906211853, + "learning_rate": 0.0001, + "loss": 72.3189, + "step": 5750 + }, + { + "epoch": 0.9301386730183081, + "grad_norm": 0.6961462497711182, + "learning_rate": 0.0001, + "loss": 74.6196, + "step": 5760 + }, + { + "epoch": 0.9317534971034093, + "grad_norm": 0.7797656655311584, + "learning_rate": 0.0001, + "loss": 74.5472, + "step": 5770 + }, + { + "epoch": 0.9333683211885105, + "grad_norm": 1.0671838521957397, + "learning_rate": 0.0001, + "loss": 73.241, + "step": 5780 + }, + { + "epoch": 0.9349831452736117, + "grad_norm": 0.7566089034080505, + "learning_rate": 0.0001, + "loss": 73.6784, + "step": 5790 + }, + { + "epoch": 0.936597969358713, + "grad_norm": 0.8563273549079895, + "learning_rate": 0.0001, + "loss": 73.6871, + "step": 5800 + }, + { + "epoch": 0.9382127934438143, + "grad_norm": 0.7885987162590027, + "learning_rate": 0.0001, + "loss": 73.4447, + "step": 5810 + }, + { + "epoch": 0.9398276175289154, + "grad_norm": 0.8751112818717957, + "learning_rate": 0.0001, + "loss": 74.0531, + "step": 5820 + }, + { + "epoch": 0.9414424416140167, + "grad_norm": 0.6663336157798767, + "learning_rate": 0.0001, + "loss": 73.0557, + "step": 5830 + }, + { + "epoch": 0.9430572656991179, + "grad_norm": 0.7993099689483643, + "learning_rate": 0.0001, + "loss": 74.4467, + "step": 5840 + }, + { + "epoch": 0.9446720897842191, + "grad_norm": 0.6987188458442688, + "learning_rate": 0.0001, + "loss": 74.0327, + "step": 5850 + }, + { + "epoch": 0.9462869138693204, + "grad_norm": 0.8458583354949951, + "learning_rate": 0.0001, + "loss": 73.1757, + "step": 5860 + }, + { + "epoch": 0.9479017379544216, + "grad_norm": 0.6372436881065369, + "learning_rate": 0.0001, + "loss": 73.1067, + "step": 5870 + }, + { + "epoch": 0.9495165620395228, + "grad_norm": 0.7447753548622131, + "learning_rate": 0.0001, + "loss": 75.5461, + "step": 5880 + }, + { + "epoch": 0.951131386124624, + "grad_norm": 0.6633971929550171, + "learning_rate": 0.0001, + "loss": 72.9966, + "step": 5890 + }, + { + "epoch": 0.9527462102097253, + "grad_norm": 0.8576784729957581, + "learning_rate": 0.0001, + "loss": 71.5712, + "step": 5900 + }, + { + "epoch": 0.9543610342948265, + "grad_norm": 1.4696853160858154, + "learning_rate": 0.0001, + "loss": 74.7893, + "step": 5910 + }, + { + "epoch": 0.9559758583799277, + "grad_norm": 0.611210286617279, + "learning_rate": 0.0001, + "loss": 74.3426, + "step": 5920 + }, + { + "epoch": 0.957590682465029, + "grad_norm": 0.5947410464286804, + "learning_rate": 0.0001, + "loss": 71.2563, + "step": 5930 + }, + { + "epoch": 0.9592055065501301, + "grad_norm": 0.7169992923736572, + "learning_rate": 0.0001, + "loss": 71.6049, + "step": 5940 + }, + { + "epoch": 0.9608203306352314, + "grad_norm": 0.6773818731307983, + "learning_rate": 0.0001, + "loss": 75.0328, + "step": 5950 + }, + { + "epoch": 0.9624351547203327, + "grad_norm": 0.6714807748794556, + "learning_rate": 0.0001, + "loss": 71.5105, + "step": 5960 + }, + { + "epoch": 0.9640499788054339, + "grad_norm": 1.0064337253570557, + "learning_rate": 0.0001, + "loss": 75.8887, + "step": 5970 + }, + { + "epoch": 0.9656648028905351, + "grad_norm": 0.9802227020263672, + "learning_rate": 0.0001, + "loss": 73.5655, + "step": 5980 + }, + { + "epoch": 0.9672796269756363, + "grad_norm": 0.7097306251525879, + "learning_rate": 0.0001, + "loss": 73.298, + "step": 5990 + }, + { + "epoch": 0.9688944510607376, + "grad_norm": 0.6680361032485962, + "learning_rate": 0.0001, + "loss": 73.8873, + "step": 6000 + }, + { + "epoch": 0.9705092751458388, + "grad_norm": 1.1302506923675537, + "learning_rate": 0.0001, + "loss": 73.3328, + "step": 6010 + }, + { + "epoch": 0.97212409923094, + "grad_norm": 0.7163954377174377, + "learning_rate": 0.0001, + "loss": 70.5955, + "step": 6020 + }, + { + "epoch": 0.9737389233160413, + "grad_norm": 0.8182240128517151, + "learning_rate": 0.0001, + "loss": 72.6397, + "step": 6030 + }, + { + "epoch": 0.9753537474011424, + "grad_norm": 0.6752052903175354, + "learning_rate": 0.0001, + "loss": 73.1188, + "step": 6040 + }, + { + "epoch": 0.9769685714862437, + "grad_norm": 0.8829680681228638, + "learning_rate": 0.0001, + "loss": 75.7346, + "step": 6050 + }, + { + "epoch": 0.978583395571345, + "grad_norm": 0.9254767894744873, + "learning_rate": 0.0001, + "loss": 73.6221, + "step": 6060 + }, + { + "epoch": 0.9801982196564462, + "grad_norm": 0.7527495622634888, + "learning_rate": 0.0001, + "loss": 73.948, + "step": 6070 + }, + { + "epoch": 0.9818130437415474, + "grad_norm": 1.0135066509246826, + "learning_rate": 0.0001, + "loss": 72.8929, + "step": 6080 + }, + { + "epoch": 0.9834278678266486, + "grad_norm": 0.7093394994735718, + "learning_rate": 0.0001, + "loss": 73.5826, + "step": 6090 + }, + { + "epoch": 0.9850426919117499, + "grad_norm": 0.6651231646537781, + "learning_rate": 0.0001, + "loss": 73.167, + "step": 6100 + }, + { + "epoch": 0.9866575159968511, + "grad_norm": 0.720552921295166, + "learning_rate": 0.0001, + "loss": 74.8985, + "step": 6110 + }, + { + "epoch": 0.9882723400819523, + "grad_norm": 1.7821632623672485, + "learning_rate": 0.0001, + "loss": 74.1414, + "step": 6120 + }, + { + "epoch": 0.9898871641670536, + "grad_norm": 0.6405150890350342, + "learning_rate": 0.0001, + "loss": 73.5989, + "step": 6130 + }, + { + "epoch": 0.9915019882521547, + "grad_norm": 0.8207482695579529, + "learning_rate": 0.0001, + "loss": 73.0977, + "step": 6140 + }, + { + "epoch": 0.993116812337256, + "grad_norm": 0.7320757508277893, + "learning_rate": 0.0001, + "loss": 72.0241, + "step": 6150 + }, + { + "epoch": 0.9947316364223573, + "grad_norm": 0.9522950649261475, + "learning_rate": 0.0001, + "loss": 73.9831, + "step": 6160 + }, + { + "epoch": 0.9963464605074585, + "grad_norm": 0.7437159419059753, + "learning_rate": 0.0001, + "loss": 72.1054, + "step": 6170 + }, + { + "epoch": 0.9979612845925597, + "grad_norm": 0.7035623788833618, + "learning_rate": 0.0001, + "loss": 73.959, + "step": 6180 + }, + { + "epoch": 0.9995761086776609, + "grad_norm": 0.7559277415275574, + "learning_rate": 0.0001, + "loss": 71.4651, + "step": 6190 + }, + { + "epoch": 1.001130376859571, + "grad_norm": 0.852342426776886, + "learning_rate": 0.0001, + "loss": 71.8844, + "step": 6200 + }, + { + "epoch": 1.002745200944672, + "grad_norm": 0.8018911480903625, + "learning_rate": 0.0001, + "loss": 74.5093, + "step": 6210 + }, + { + "epoch": 1.0043600250297733, + "grad_norm": 0.6613730192184448, + "learning_rate": 0.0001, + "loss": 71.9953, + "step": 6220 + }, + { + "epoch": 1.0059748491148746, + "grad_norm": 0.9815977215766907, + "learning_rate": 0.0001, + "loss": 70.8632, + "step": 6230 + }, + { + "epoch": 1.0075896731999758, + "grad_norm": 0.781581699848175, + "learning_rate": 0.0001, + "loss": 72.3603, + "step": 6240 + }, + { + "epoch": 1.009204497285077, + "grad_norm": 0.8079054355621338, + "learning_rate": 0.0001, + "loss": 73.6153, + "step": 6250 + }, + { + "epoch": 1.0108193213701782, + "grad_norm": 0.8356053233146667, + "learning_rate": 0.0001, + "loss": 73.1562, + "step": 6260 + }, + { + "epoch": 1.0124341454552794, + "grad_norm": 0.7107778787612915, + "learning_rate": 0.0001, + "loss": 74.5276, + "step": 6270 + }, + { + "epoch": 1.0140489695403807, + "grad_norm": 0.8601720929145813, + "learning_rate": 0.0001, + "loss": 70.103, + "step": 6280 + }, + { + "epoch": 1.015663793625482, + "grad_norm": 0.675837516784668, + "learning_rate": 0.0001, + "loss": 72.6102, + "step": 6290 + }, + { + "epoch": 1.0172786177105833, + "grad_norm": 0.7160016298294067, + "learning_rate": 0.0001, + "loss": 72.2404, + "step": 6300 + }, + { + "epoch": 1.0188934417956843, + "grad_norm": 0.7286407351493835, + "learning_rate": 0.0001, + "loss": 72.7149, + "step": 6310 + }, + { + "epoch": 1.0205082658807856, + "grad_norm": 0.8183980584144592, + "learning_rate": 0.0001, + "loss": 73.5827, + "step": 6320 + }, + { + "epoch": 1.0221230899658869, + "grad_norm": 0.7300626039505005, + "learning_rate": 0.0001, + "loss": 72.3459, + "step": 6330 + }, + { + "epoch": 1.0237379140509881, + "grad_norm": 0.7222613096237183, + "learning_rate": 0.0001, + "loss": 71.881, + "step": 6340 + }, + { + "epoch": 1.0253527381360894, + "grad_norm": 0.7324674725532532, + "learning_rate": 0.0001, + "loss": 74.4781, + "step": 6350 + }, + { + "epoch": 1.0269675622211905, + "grad_norm": 0.9173555374145508, + "learning_rate": 0.0001, + "loss": 74.3222, + "step": 6360 + }, + { + "epoch": 1.0285823863062917, + "grad_norm": 0.8404964208602905, + "learning_rate": 0.0001, + "loss": 71.8959, + "step": 6370 + }, + { + "epoch": 1.030197210391393, + "grad_norm": 0.8735223412513733, + "learning_rate": 0.0001, + "loss": 72.7251, + "step": 6380 + }, + { + "epoch": 1.0318120344764943, + "grad_norm": 0.8467233777046204, + "learning_rate": 0.0001, + "loss": 72.5218, + "step": 6390 + }, + { + "epoch": 1.0334268585615956, + "grad_norm": 0.673565149307251, + "learning_rate": 0.0001, + "loss": 70.6392, + "step": 6400 + }, + { + "epoch": 1.0350416826466966, + "grad_norm": 0.6558571457862854, + "learning_rate": 0.0001, + "loss": 73.5305, + "step": 6410 + }, + { + "epoch": 1.0366565067317979, + "grad_norm": 0.7251600623130798, + "learning_rate": 0.0001, + "loss": 72.76, + "step": 6420 + }, + { + "epoch": 1.0382713308168992, + "grad_norm": 0.7458305358886719, + "learning_rate": 0.0001, + "loss": 73.6812, + "step": 6430 + }, + { + "epoch": 1.0398861549020004, + "grad_norm": 1.493802547454834, + "learning_rate": 0.0001, + "loss": 74.2455, + "step": 6440 + }, + { + "epoch": 1.0415009789871017, + "grad_norm": 0.894632875919342, + "learning_rate": 0.0001, + "loss": 74.2072, + "step": 6450 + }, + { + "epoch": 1.0431158030722028, + "grad_norm": 0.7142218947410583, + "learning_rate": 0.0001, + "loss": 71.4293, + "step": 6460 + }, + { + "epoch": 1.044730627157304, + "grad_norm": 0.702639639377594, + "learning_rate": 0.0001, + "loss": 71.5383, + "step": 6470 + }, + { + "epoch": 1.0463454512424053, + "grad_norm": 0.9378212094306946, + "learning_rate": 0.0001, + "loss": 72.8916, + "step": 6480 + }, + { + "epoch": 1.0479602753275066, + "grad_norm": 0.8298363089561462, + "learning_rate": 0.0001, + "loss": 72.3344, + "step": 6490 + }, + { + "epoch": 1.0495750994126078, + "grad_norm": 0.6617016792297363, + "learning_rate": 0.0001, + "loss": 71.0487, + "step": 6500 + }, + { + "epoch": 1.051189923497709, + "grad_norm": 0.638520359992981, + "learning_rate": 0.0001, + "loss": 72.2583, + "step": 6510 + }, + { + "epoch": 1.0528047475828102, + "grad_norm": 0.7101534008979797, + "learning_rate": 0.0001, + "loss": 70.8697, + "step": 6520 + }, + { + "epoch": 1.0544195716679114, + "grad_norm": 0.9425339698791504, + "learning_rate": 0.0001, + "loss": 71.0641, + "step": 6530 + }, + { + "epoch": 1.0560343957530127, + "grad_norm": 0.6572297811508179, + "learning_rate": 0.0001, + "loss": 72.6265, + "step": 6540 + }, + { + "epoch": 1.057649219838114, + "grad_norm": 0.745622456073761, + "learning_rate": 0.0001, + "loss": 73.0587, + "step": 6550 + }, + { + "epoch": 1.059264043923215, + "grad_norm": 0.7667096257209778, + "learning_rate": 0.0001, + "loss": 72.1779, + "step": 6560 + }, + { + "epoch": 1.0608788680083163, + "grad_norm": 0.7322410941123962, + "learning_rate": 0.0001, + "loss": 71.5503, + "step": 6570 + }, + { + "epoch": 1.0624936920934176, + "grad_norm": 0.5739616751670837, + "learning_rate": 0.0001, + "loss": 72.3846, + "step": 6580 + }, + { + "epoch": 1.0641085161785189, + "grad_norm": 0.7149553894996643, + "learning_rate": 0.0001, + "loss": 71.5459, + "step": 6590 + }, + { + "epoch": 1.0657233402636201, + "grad_norm": 0.6634597778320312, + "learning_rate": 0.0001, + "loss": 72.2528, + "step": 6600 + }, + { + "epoch": 1.0673381643487212, + "grad_norm": 0.7208803296089172, + "learning_rate": 0.0001, + "loss": 71.1115, + "step": 6610 + }, + { + "epoch": 1.0689529884338225, + "grad_norm": 0.9845868945121765, + "learning_rate": 0.0001, + "loss": 72.6418, + "step": 6620 + }, + { + "epoch": 1.0705678125189237, + "grad_norm": 0.9322167634963989, + "learning_rate": 0.0001, + "loss": 71.937, + "step": 6630 + }, + { + "epoch": 1.072182636604025, + "grad_norm": 0.6323093771934509, + "learning_rate": 0.0001, + "loss": 71.1537, + "step": 6640 + }, + { + "epoch": 1.0737974606891263, + "grad_norm": 0.6476802229881287, + "learning_rate": 0.0001, + "loss": 73.7811, + "step": 6650 + }, + { + "epoch": 1.0754122847742273, + "grad_norm": 0.7882822751998901, + "learning_rate": 0.0001, + "loss": 71.9548, + "step": 6660 + }, + { + "epoch": 1.0770271088593286, + "grad_norm": 0.7651168704032898, + "learning_rate": 0.0001, + "loss": 73.4066, + "step": 6670 + }, + { + "epoch": 1.07864193294443, + "grad_norm": 0.7536904215812683, + "learning_rate": 0.0001, + "loss": 72.1063, + "step": 6680 + }, + { + "epoch": 1.0802567570295312, + "grad_norm": 0.7908427119255066, + "learning_rate": 0.0001, + "loss": 72.6374, + "step": 6690 + }, + { + "epoch": 1.0818715811146324, + "grad_norm": 0.7796691060066223, + "learning_rate": 0.0001, + "loss": 72.2938, + "step": 6700 + }, + { + "epoch": 1.0834864051997335, + "grad_norm": 0.8512274622917175, + "learning_rate": 0.0001, + "loss": 71.556, + "step": 6710 + }, + { + "epoch": 1.0851012292848348, + "grad_norm": 0.7285851836204529, + "learning_rate": 0.0001, + "loss": 72.406, + "step": 6720 + }, + { + "epoch": 1.086716053369936, + "grad_norm": 0.6613008975982666, + "learning_rate": 0.0001, + "loss": 70.6509, + "step": 6730 + }, + { + "epoch": 1.0883308774550373, + "grad_norm": 0.7158271074295044, + "learning_rate": 0.0001, + "loss": 71.8434, + "step": 6740 + }, + { + "epoch": 1.0899457015401384, + "grad_norm": 0.8635967373847961, + "learning_rate": 0.0001, + "loss": 71.8279, + "step": 6750 + }, + { + "epoch": 1.0915605256252396, + "grad_norm": 0.7020159959793091, + "learning_rate": 0.0001, + "loss": 72.2417, + "step": 6760 + }, + { + "epoch": 1.093175349710341, + "grad_norm": 0.6389853954315186, + "learning_rate": 0.0001, + "loss": 72.1879, + "step": 6770 + }, + { + "epoch": 1.0947901737954422, + "grad_norm": 0.6460697650909424, + "learning_rate": 0.0001, + "loss": 72.3356, + "step": 6780 + }, + { + "epoch": 1.0964049978805435, + "grad_norm": 0.7152006030082703, + "learning_rate": 0.0001, + "loss": 71.3725, + "step": 6790 + }, + { + "epoch": 1.0980198219656445, + "grad_norm": 1.0050997734069824, + "learning_rate": 0.0001, + "loss": 72.5452, + "step": 6800 + }, + { + "epoch": 1.0996346460507458, + "grad_norm": 0.6967180371284485, + "learning_rate": 0.0001, + "loss": 72.0832, + "step": 6810 + }, + { + "epoch": 1.101249470135847, + "grad_norm": 0.7472837567329407, + "learning_rate": 0.0001, + "loss": 72.8122, + "step": 6820 + }, + { + "epoch": 1.1028642942209483, + "grad_norm": 0.8198382258415222, + "learning_rate": 0.0001, + "loss": 72.5116, + "step": 6830 + }, + { + "epoch": 1.1044791183060496, + "grad_norm": 0.8112207651138306, + "learning_rate": 0.0001, + "loss": 72.4218, + "step": 6840 + }, + { + "epoch": 1.1060939423911507, + "grad_norm": 0.8412114977836609, + "learning_rate": 0.0001, + "loss": 71.0198, + "step": 6850 + }, + { + "epoch": 1.107708766476252, + "grad_norm": 0.6730719804763794, + "learning_rate": 0.0001, + "loss": 71.2045, + "step": 6860 + }, + { + "epoch": 1.1093235905613532, + "grad_norm": 0.6895949244499207, + "learning_rate": 0.0001, + "loss": 72.3056, + "step": 6870 + }, + { + "epoch": 1.1109384146464545, + "grad_norm": 2.476609945297241, + "learning_rate": 0.0001, + "loss": 70.3865, + "step": 6880 + }, + { + "epoch": 1.1125532387315558, + "grad_norm": 0.7285001873970032, + "learning_rate": 0.0001, + "loss": 71.5761, + "step": 6890 + }, + { + "epoch": 1.1141680628166568, + "grad_norm": 0.7546679973602295, + "learning_rate": 0.0001, + "loss": 72.5979, + "step": 6900 + }, + { + "epoch": 1.115782886901758, + "grad_norm": 0.8974841237068176, + "learning_rate": 0.0001, + "loss": 74.5888, + "step": 6910 + }, + { + "epoch": 1.1173977109868594, + "grad_norm": 0.8785758018493652, + "learning_rate": 0.0001, + "loss": 71.662, + "step": 6920 + }, + { + "epoch": 1.1190125350719606, + "grad_norm": 0.6805328130722046, + "learning_rate": 0.0001, + "loss": 72.201, + "step": 6930 + }, + { + "epoch": 1.120627359157062, + "grad_norm": 0.8132315278053284, + "learning_rate": 0.0001, + "loss": 72.7473, + "step": 6940 + }, + { + "epoch": 1.122242183242163, + "grad_norm": 1.1625338792800903, + "learning_rate": 0.0001, + "loss": 70.5987, + "step": 6950 + }, + { + "epoch": 1.1238570073272642, + "grad_norm": 1.0984737873077393, + "learning_rate": 0.0001, + "loss": 72.7335, + "step": 6960 + }, + { + "epoch": 1.1254718314123655, + "grad_norm": 0.6798779368400574, + "learning_rate": 0.0001, + "loss": 72.1942, + "step": 6970 + }, + { + "epoch": 1.1270866554974668, + "grad_norm": 0.7546010613441467, + "learning_rate": 0.0001, + "loss": 70.8014, + "step": 6980 + }, + { + "epoch": 1.128701479582568, + "grad_norm": 1.0238053798675537, + "learning_rate": 0.0001, + "loss": 72.5617, + "step": 6990 + }, + { + "epoch": 1.130316303667669, + "grad_norm": 0.6535419821739197, + "learning_rate": 0.0001, + "loss": 70.2193, + "step": 7000 + }, + { + "epoch": 1.1319311277527704, + "grad_norm": 0.7354230284690857, + "learning_rate": 0.0001, + "loss": 72.0685, + "step": 7010 + }, + { + "epoch": 1.1335459518378717, + "grad_norm": 0.7848799824714661, + "learning_rate": 0.0001, + "loss": 72.2869, + "step": 7020 + }, + { + "epoch": 1.135160775922973, + "grad_norm": 0.6519262790679932, + "learning_rate": 0.0001, + "loss": 72.2023, + "step": 7030 + }, + { + "epoch": 1.1367756000080742, + "grad_norm": 0.8428511023521423, + "learning_rate": 0.0001, + "loss": 71.3396, + "step": 7040 + }, + { + "epoch": 1.1383904240931753, + "grad_norm": 0.8873165249824524, + "learning_rate": 0.0001, + "loss": 69.8284, + "step": 7050 + }, + { + "epoch": 1.1400052481782765, + "grad_norm": 0.9141374230384827, + "learning_rate": 0.0001, + "loss": 70.3666, + "step": 7060 + }, + { + "epoch": 1.1416200722633778, + "grad_norm": 0.6556552648544312, + "learning_rate": 0.0001, + "loss": 71.1316, + "step": 7070 + }, + { + "epoch": 1.143234896348479, + "grad_norm": 0.6736201047897339, + "learning_rate": 0.0001, + "loss": 70.2601, + "step": 7080 + }, + { + "epoch": 1.1448497204335804, + "grad_norm": 0.6965599656105042, + "learning_rate": 0.0001, + "loss": 71.9409, + "step": 7090 + }, + { + "epoch": 1.1464645445186814, + "grad_norm": 1.0600581169128418, + "learning_rate": 0.0001, + "loss": 72.7344, + "step": 7100 + }, + { + "epoch": 1.1480793686037827, + "grad_norm": 0.7501764297485352, + "learning_rate": 0.0001, + "loss": 72.3129, + "step": 7110 + }, + { + "epoch": 1.149694192688884, + "grad_norm": 0.7105675935745239, + "learning_rate": 0.0001, + "loss": 71.1356, + "step": 7120 + }, + { + "epoch": 1.1513090167739852, + "grad_norm": 0.7446026802062988, + "learning_rate": 0.0001, + "loss": 69.8208, + "step": 7130 + }, + { + "epoch": 1.1529238408590865, + "grad_norm": 0.6664865016937256, + "learning_rate": 0.0001, + "loss": 72.2466, + "step": 7140 + }, + { + "epoch": 1.1545386649441876, + "grad_norm": 1.7590374946594238, + "learning_rate": 0.0001, + "loss": 71.4987, + "step": 7150 + }, + { + "epoch": 1.1561534890292888, + "grad_norm": 0.7941271066665649, + "learning_rate": 0.0001, + "loss": 70.787, + "step": 7160 + }, + { + "epoch": 1.15776831311439, + "grad_norm": 0.7473573088645935, + "learning_rate": 0.0001, + "loss": 71.2516, + "step": 7170 + }, + { + "epoch": 1.1593831371994914, + "grad_norm": 0.7589637041091919, + "learning_rate": 0.0001, + "loss": 71.4781, + "step": 7180 + }, + { + "epoch": 1.1609979612845926, + "grad_norm": 0.8148510456085205, + "learning_rate": 0.0001, + "loss": 71.1491, + "step": 7190 + }, + { + "epoch": 1.1626127853696937, + "grad_norm": 0.6803365349769592, + "learning_rate": 0.0001, + "loss": 71.7937, + "step": 7200 + }, + { + "epoch": 1.164227609454795, + "grad_norm": 0.7314755320549011, + "learning_rate": 0.0001, + "loss": 70.6566, + "step": 7210 + }, + { + "epoch": 1.1658424335398963, + "grad_norm": 0.6494631171226501, + "learning_rate": 0.0001, + "loss": 70.5057, + "step": 7220 + }, + { + "epoch": 1.1674572576249975, + "grad_norm": 0.6076239943504333, + "learning_rate": 0.0001, + "loss": 72.2985, + "step": 7230 + }, + { + "epoch": 1.1690720817100988, + "grad_norm": 0.6650206446647644, + "learning_rate": 0.0001, + "loss": 71.0719, + "step": 7240 + }, + { + "epoch": 1.1706869057951999, + "grad_norm": 0.6926266551017761, + "learning_rate": 0.0001, + "loss": 70.3714, + "step": 7250 + }, + { + "epoch": 1.1723017298803011, + "grad_norm": 0.6534716486930847, + "learning_rate": 0.0001, + "loss": 71.3194, + "step": 7260 + }, + { + "epoch": 1.1739165539654024, + "grad_norm": 0.7194528579711914, + "learning_rate": 0.0001, + "loss": 72.3277, + "step": 7270 + }, + { + "epoch": 1.1755313780505037, + "grad_norm": 0.7496437430381775, + "learning_rate": 0.0001, + "loss": 71.1136, + "step": 7280 + }, + { + "epoch": 1.177146202135605, + "grad_norm": 0.690522313117981, + "learning_rate": 0.0001, + "loss": 70.065, + "step": 7290 + }, + { + "epoch": 1.178761026220706, + "grad_norm": 0.6966914534568787, + "learning_rate": 0.0001, + "loss": 73.1137, + "step": 7300 + }, + { + "epoch": 1.1803758503058073, + "grad_norm": 0.7304641604423523, + "learning_rate": 0.0001, + "loss": 71.2466, + "step": 7310 + }, + { + "epoch": 1.1819906743909085, + "grad_norm": 0.7564008235931396, + "learning_rate": 0.0001, + "loss": 71.4147, + "step": 7320 + }, + { + "epoch": 1.1836054984760098, + "grad_norm": 0.61886066198349, + "learning_rate": 0.0001, + "loss": 70.3297, + "step": 7330 + }, + { + "epoch": 1.185220322561111, + "grad_norm": 0.6281574964523315, + "learning_rate": 0.0001, + "loss": 70.9607, + "step": 7340 + }, + { + "epoch": 1.1868351466462121, + "grad_norm": 0.6583316326141357, + "learning_rate": 0.0001, + "loss": 70.3443, + "step": 7350 + }, + { + "epoch": 1.1884499707313134, + "grad_norm": 1.8159942626953125, + "learning_rate": 0.0001, + "loss": 71.4792, + "step": 7360 + }, + { + "epoch": 1.1900647948164147, + "grad_norm": 0.7253140807151794, + "learning_rate": 0.0001, + "loss": 72.2182, + "step": 7370 + }, + { + "epoch": 1.191679618901516, + "grad_norm": 0.9141016602516174, + "learning_rate": 0.0001, + "loss": 68.7443, + "step": 7380 + }, + { + "epoch": 1.193294442986617, + "grad_norm": 0.7392430901527405, + "learning_rate": 0.0001, + "loss": 69.1888, + "step": 7390 + }, + { + "epoch": 1.1949092670717183, + "grad_norm": 0.7713924646377563, + "learning_rate": 0.0001, + "loss": 69.1301, + "step": 7400 + }, + { + "epoch": 1.1965240911568196, + "grad_norm": 0.6000798344612122, + "learning_rate": 0.0001, + "loss": 71.2576, + "step": 7410 + }, + { + "epoch": 1.1981389152419208, + "grad_norm": 0.687111496925354, + "learning_rate": 0.0001, + "loss": 70.2919, + "step": 7420 + }, + { + "epoch": 1.1997537393270221, + "grad_norm": 1.1718226671218872, + "learning_rate": 0.0001, + "loss": 71.9498, + "step": 7430 + }, + { + "epoch": 1.2013685634121232, + "grad_norm": 0.7176038026809692, + "learning_rate": 0.0001, + "loss": 71.91, + "step": 7440 + }, + { + "epoch": 1.2029833874972244, + "grad_norm": 0.7705256938934326, + "learning_rate": 0.0001, + "loss": 70.1241, + "step": 7450 + }, + { + "epoch": 1.2045982115823257, + "grad_norm": 0.8004609942436218, + "learning_rate": 0.0001, + "loss": 69.5121, + "step": 7460 + }, + { + "epoch": 1.206213035667427, + "grad_norm": 0.6708329319953918, + "learning_rate": 0.0001, + "loss": 73.0173, + "step": 7470 + }, + { + "epoch": 1.2078278597525283, + "grad_norm": 0.8777983784675598, + "learning_rate": 0.0001, + "loss": 71.6843, + "step": 7480 + }, + { + "epoch": 1.2094426838376293, + "grad_norm": 0.7683864831924438, + "learning_rate": 0.0001, + "loss": 71.6914, + "step": 7490 + }, + { + "epoch": 1.2110575079227306, + "grad_norm": 0.8022210597991943, + "learning_rate": 0.0001, + "loss": 69.1724, + "step": 7500 + }, + { + "epoch": 1.2126723320078319, + "grad_norm": 0.7250640392303467, + "learning_rate": 0.0001, + "loss": 70.6568, + "step": 7510 + }, + { + "epoch": 1.2142871560929331, + "grad_norm": 0.5625214576721191, + "learning_rate": 0.0001, + "loss": 69.4849, + "step": 7520 + }, + { + "epoch": 1.2159019801780344, + "grad_norm": 0.7980468273162842, + "learning_rate": 0.0001, + "loss": 70.8164, + "step": 7530 + }, + { + "epoch": 1.2175168042631355, + "grad_norm": 0.6900951862335205, + "learning_rate": 0.0001, + "loss": 71.1719, + "step": 7540 + }, + { + "epoch": 1.2191316283482367, + "grad_norm": 0.6284971833229065, + "learning_rate": 0.0001, + "loss": 70.1751, + "step": 7550 + }, + { + "epoch": 1.220746452433338, + "grad_norm": 0.6641798615455627, + "learning_rate": 0.0001, + "loss": 71.9043, + "step": 7560 + }, + { + "epoch": 1.2223612765184393, + "grad_norm": 0.7439492344856262, + "learning_rate": 0.0001, + "loss": 70.7286, + "step": 7570 + }, + { + "epoch": 1.2239761006035406, + "grad_norm": 0.8018355369567871, + "learning_rate": 0.0001, + "loss": 71.638, + "step": 7580 + }, + { + "epoch": 1.2255909246886416, + "grad_norm": 0.7318522930145264, + "learning_rate": 0.0001, + "loss": 70.0867, + "step": 7590 + }, + { + "epoch": 1.2272057487737429, + "grad_norm": 0.6799285411834717, + "learning_rate": 0.0001, + "loss": 71.3427, + "step": 7600 + }, + { + "epoch": 1.2288205728588442, + "grad_norm": 0.6796864867210388, + "learning_rate": 0.0001, + "loss": 71.1698, + "step": 7610 + }, + { + "epoch": 1.2304353969439454, + "grad_norm": 0.7256141304969788, + "learning_rate": 0.0001, + "loss": 71.1051, + "step": 7620 + }, + { + "epoch": 1.2320502210290467, + "grad_norm": 0.8318199515342712, + "learning_rate": 0.0001, + "loss": 71.3179, + "step": 7630 + }, + { + "epoch": 1.2336650451141478, + "grad_norm": 0.6594699025154114, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 7640 + }, + { + "epoch": 1.235279869199249, + "grad_norm": 0.7900195717811584, + "learning_rate": 0.0001, + "loss": 72.0139, + "step": 7650 + }, + { + "epoch": 1.2368946932843503, + "grad_norm": 0.7845672965049744, + "learning_rate": 0.0001, + "loss": 70.808, + "step": 7660 + }, + { + "epoch": 1.2385095173694516, + "grad_norm": 0.6598045229911804, + "learning_rate": 0.0001, + "loss": 69.8953, + "step": 7670 + }, + { + "epoch": 1.2401243414545529, + "grad_norm": 1.3222978115081787, + "learning_rate": 0.0001, + "loss": 68.787, + "step": 7680 + }, + { + "epoch": 1.241739165539654, + "grad_norm": 0.6660508513450623, + "learning_rate": 0.0001, + "loss": 71.5831, + "step": 7690 + }, + { + "epoch": 1.2433539896247552, + "grad_norm": 0.6305698156356812, + "learning_rate": 0.0001, + "loss": 70.3232, + "step": 7700 + }, + { + "epoch": 1.2449688137098565, + "grad_norm": 1.2648323774337769, + "learning_rate": 0.0001, + "loss": 67.8901, + "step": 7710 + }, + { + "epoch": 1.2465836377949577, + "grad_norm": 0.7177619934082031, + "learning_rate": 0.0001, + "loss": 69.6582, + "step": 7720 + }, + { + "epoch": 1.248198461880059, + "grad_norm": 0.6716091632843018, + "learning_rate": 0.0001, + "loss": 70.2034, + "step": 7730 + }, + { + "epoch": 1.24981328596516, + "grad_norm": 0.6050158143043518, + "learning_rate": 0.0001, + "loss": 69.3942, + "step": 7740 + }, + { + "epoch": 1.2514281100502613, + "grad_norm": 0.7001014947891235, + "learning_rate": 0.0001, + "loss": 69.5971, + "step": 7750 + }, + { + "epoch": 1.2530429341353626, + "grad_norm": 0.7221280932426453, + "learning_rate": 0.0001, + "loss": 71.3651, + "step": 7760 + }, + { + "epoch": 1.2546577582204639, + "grad_norm": 0.9130340218544006, + "learning_rate": 0.0001, + "loss": 68.3038, + "step": 7770 + }, + { + "epoch": 1.2562725823055652, + "grad_norm": 0.7703199982643127, + "learning_rate": 0.0001, + "loss": 71.6127, + "step": 7780 + }, + { + "epoch": 1.2578874063906662, + "grad_norm": 0.6334695816040039, + "learning_rate": 0.0001, + "loss": 68.9074, + "step": 7790 + }, + { + "epoch": 1.2595022304757675, + "grad_norm": 0.7074242234230042, + "learning_rate": 0.0001, + "loss": 70.0569, + "step": 7800 + }, + { + "epoch": 1.2611170545608688, + "grad_norm": 0.6956025958061218, + "learning_rate": 0.0001, + "loss": 69.892, + "step": 7810 + }, + { + "epoch": 1.26273187864597, + "grad_norm": 0.7235708236694336, + "learning_rate": 0.0001, + "loss": 71.1777, + "step": 7820 + }, + { + "epoch": 1.2643467027310713, + "grad_norm": 0.6157007813453674, + "learning_rate": 0.0001, + "loss": 71.5091, + "step": 7830 + }, + { + "epoch": 1.2659615268161724, + "grad_norm": 0.5352533459663391, + "learning_rate": 0.0001, + "loss": 70.083, + "step": 7840 + }, + { + "epoch": 1.2675763509012736, + "grad_norm": 0.7171182632446289, + "learning_rate": 0.0001, + "loss": 68.4585, + "step": 7850 + }, + { + "epoch": 1.269191174986375, + "grad_norm": 0.7503113746643066, + "learning_rate": 0.0001, + "loss": 70.4206, + "step": 7860 + }, + { + "epoch": 1.2708059990714762, + "grad_norm": 0.6552693843841553, + "learning_rate": 0.0001, + "loss": 70.8455, + "step": 7870 + }, + { + "epoch": 1.2724208231565775, + "grad_norm": 0.7173179984092712, + "learning_rate": 0.0001, + "loss": 68.82, + "step": 7880 + }, + { + "epoch": 1.2740356472416785, + "grad_norm": 0.7352942824363708, + "learning_rate": 0.0001, + "loss": 70.4974, + "step": 7890 + }, + { + "epoch": 1.2756504713267798, + "grad_norm": 0.6120197176933289, + "learning_rate": 0.0001, + "loss": 72.1638, + "step": 7900 + }, + { + "epoch": 1.277265295411881, + "grad_norm": 0.6271925568580627, + "learning_rate": 0.0001, + "loss": 69.3267, + "step": 7910 + }, + { + "epoch": 1.2788801194969823, + "grad_norm": 0.752636194229126, + "learning_rate": 0.0001, + "loss": 70.6761, + "step": 7920 + }, + { + "epoch": 1.2804949435820836, + "grad_norm": 0.7307374477386475, + "learning_rate": 0.0001, + "loss": 70.2909, + "step": 7930 + }, + { + "epoch": 1.2821097676671847, + "grad_norm": 0.5977495908737183, + "learning_rate": 0.0001, + "loss": 72.2969, + "step": 7940 + }, + { + "epoch": 1.283724591752286, + "grad_norm": 0.7927457094192505, + "learning_rate": 0.0001, + "loss": 70.3784, + "step": 7950 + }, + { + "epoch": 1.2853394158373872, + "grad_norm": 0.7520087957382202, + "learning_rate": 0.0001, + "loss": 70.7122, + "step": 7960 + }, + { + "epoch": 1.2869542399224885, + "grad_norm": 0.6036237478256226, + "learning_rate": 0.0001, + "loss": 69.8718, + "step": 7970 + }, + { + "epoch": 1.2885690640075897, + "grad_norm": 0.6805875897407532, + "learning_rate": 0.0001, + "loss": 70.2391, + "step": 7980 + }, + { + "epoch": 1.2901838880926908, + "grad_norm": 0.774559497833252, + "learning_rate": 0.0001, + "loss": 71.1605, + "step": 7990 + }, + { + "epoch": 1.291798712177792, + "grad_norm": 0.6991063356399536, + "learning_rate": 0.0001, + "loss": 69.9779, + "step": 8000 + }, + { + "epoch": 1.2934135362628933, + "grad_norm": 0.5877949595451355, + "learning_rate": 0.0001, + "loss": 71.16, + "step": 8010 + }, + { + "epoch": 1.2950283603479946, + "grad_norm": 0.8840436339378357, + "learning_rate": 0.0001, + "loss": 70.9454, + "step": 8020 + }, + { + "epoch": 1.296643184433096, + "grad_norm": 0.7532126307487488, + "learning_rate": 0.0001, + "loss": 70.141, + "step": 8030 + }, + { + "epoch": 1.298258008518197, + "grad_norm": 0.6880044341087341, + "learning_rate": 0.0001, + "loss": 70.0746, + "step": 8040 + }, + { + "epoch": 1.2998728326032982, + "grad_norm": 0.6319366097450256, + "learning_rate": 0.0001, + "loss": 69.9396, + "step": 8050 + }, + { + "epoch": 1.3014876566883995, + "grad_norm": 0.6329445838928223, + "learning_rate": 0.0001, + "loss": 69.9043, + "step": 8060 + }, + { + "epoch": 1.3031024807735008, + "grad_norm": 0.8892332315444946, + "learning_rate": 0.0001, + "loss": 69.4187, + "step": 8070 + }, + { + "epoch": 1.304717304858602, + "grad_norm": 0.696500301361084, + "learning_rate": 0.0001, + "loss": 68.1885, + "step": 8080 + }, + { + "epoch": 1.306332128943703, + "grad_norm": 0.5933213233947754, + "learning_rate": 0.0001, + "loss": 68.8017, + "step": 8090 + }, + { + "epoch": 1.3079469530288044, + "grad_norm": 0.9274749159812927, + "learning_rate": 0.0001, + "loss": 71.5875, + "step": 8100 + }, + { + "epoch": 1.3095617771139056, + "grad_norm": 0.7540446519851685, + "learning_rate": 0.0001, + "loss": 71.1233, + "step": 8110 + }, + { + "epoch": 1.311176601199007, + "grad_norm": 0.6701658964157104, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8120 + }, + { + "epoch": 1.3127914252841082, + "grad_norm": 0.6096424460411072, + "learning_rate": 0.0001, + "loss": 69.9583, + "step": 8130 + }, + { + "epoch": 1.3144062493692092, + "grad_norm": 0.809524416923523, + "learning_rate": 0.0001, + "loss": 71.8149, + "step": 8140 + }, + { + "epoch": 1.3160210734543105, + "grad_norm": 0.7928242683410645, + "learning_rate": 0.0001, + "loss": 70.2084, + "step": 8150 + }, + { + "epoch": 1.3176358975394118, + "grad_norm": 0.7462850213050842, + "learning_rate": 0.0001, + "loss": 71.4789, + "step": 8160 + }, + { + "epoch": 1.319250721624513, + "grad_norm": 0.8166030049324036, + "learning_rate": 0.0001, + "loss": 69.3154, + "step": 8170 + }, + { + "epoch": 1.3208655457096143, + "grad_norm": 0.7002414464950562, + "learning_rate": 0.0001, + "loss": 70.8048, + "step": 8180 + }, + { + "epoch": 1.3224803697947154, + "grad_norm": 0.7653809189796448, + "learning_rate": 0.0001, + "loss": 70.0221, + "step": 8190 + }, + { + "epoch": 1.3240951938798167, + "grad_norm": 0.6777089238166809, + "learning_rate": 0.0001, + "loss": 70.614, + "step": 8200 + }, + { + "epoch": 1.325710017964918, + "grad_norm": 0.692739725112915, + "learning_rate": 0.0001, + "loss": 71.2311, + "step": 8210 + }, + { + "epoch": 1.3273248420500192, + "grad_norm": 0.7985519766807556, + "learning_rate": 0.0001, + "loss": 68.1982, + "step": 8220 + }, + { + "epoch": 1.3289396661351205, + "grad_norm": 0.7002689242362976, + "learning_rate": 0.0001, + "loss": 70.9181, + "step": 8230 + }, + { + "epoch": 1.3305544902202215, + "grad_norm": 0.7029191255569458, + "learning_rate": 0.0001, + "loss": 70.8659, + "step": 8240 + }, + { + "epoch": 1.3321693143053228, + "grad_norm": 0.7292912602424622, + "learning_rate": 0.0001, + "loss": 71.5386, + "step": 8250 + }, + { + "epoch": 1.333784138390424, + "grad_norm": 0.6141605973243713, + "learning_rate": 0.0001, + "loss": 72.3396, + "step": 8260 + }, + { + "epoch": 1.3353989624755254, + "grad_norm": 0.6486957669258118, + "learning_rate": 0.0001, + "loss": 69.7607, + "step": 8270 + }, + { + "epoch": 1.3370137865606266, + "grad_norm": 0.7813800573348999, + "learning_rate": 0.0001, + "loss": 68.8979, + "step": 8280 + }, + { + "epoch": 1.3386286106457277, + "grad_norm": 0.6805126070976257, + "learning_rate": 0.0001, + "loss": 70.2179, + "step": 8290 + }, + { + "epoch": 1.340243434730829, + "grad_norm": 0.7202772498130798, + "learning_rate": 0.0001, + "loss": 72.5008, + "step": 8300 + }, + { + "epoch": 1.3418582588159302, + "grad_norm": 0.6881810426712036, + "learning_rate": 0.0001, + "loss": 70.1987, + "step": 8310 + }, + { + "epoch": 1.3434730829010315, + "grad_norm": 0.6337536573410034, + "learning_rate": 0.0001, + "loss": 70.4073, + "step": 8320 + }, + { + "epoch": 1.3450879069861328, + "grad_norm": 0.7313241958618164, + "learning_rate": 0.0001, + "loss": 69.3598, + "step": 8330 + }, + { + "epoch": 1.3467027310712338, + "grad_norm": 0.635947585105896, + "learning_rate": 0.0001, + "loss": 69.5143, + "step": 8340 + }, + { + "epoch": 1.3483175551563351, + "grad_norm": 0.6219074726104736, + "learning_rate": 0.0001, + "loss": 69.9903, + "step": 8350 + }, + { + "epoch": 1.3499323792414364, + "grad_norm": 0.5928048491477966, + "learning_rate": 0.0001, + "loss": 70.4222, + "step": 8360 + }, + { + "epoch": 1.3515472033265377, + "grad_norm": 0.5927143096923828, + "learning_rate": 0.0001, + "loss": 69.8642, + "step": 8370 + }, + { + "epoch": 1.353162027411639, + "grad_norm": 0.7614993453025818, + "learning_rate": 0.0001, + "loss": 69.5787, + "step": 8380 + }, + { + "epoch": 1.35477685149674, + "grad_norm": 0.5488380193710327, + "learning_rate": 0.0001, + "loss": 69.0028, + "step": 8390 + }, + { + "epoch": 1.3563916755818413, + "grad_norm": 0.6219164133071899, + "learning_rate": 0.0001, + "loss": 69.4591, + "step": 8400 + }, + { + "epoch": 1.3580064996669425, + "grad_norm": 0.6940358877182007, + "learning_rate": 0.0001, + "loss": 71.3795, + "step": 8410 + }, + { + "epoch": 1.3596213237520438, + "grad_norm": 0.5919192433357239, + "learning_rate": 0.0001, + "loss": 71.2761, + "step": 8420 + }, + { + "epoch": 1.361236147837145, + "grad_norm": 1.0803378820419312, + "learning_rate": 0.0001, + "loss": 69.0046, + "step": 8430 + }, + { + "epoch": 1.3628509719222461, + "grad_norm": 0.7028629183769226, + "learning_rate": 0.0001, + "loss": 71.5996, + "step": 8440 + }, + { + "epoch": 1.3644657960073474, + "grad_norm": 0.6829854249954224, + "learning_rate": 0.0001, + "loss": 67.8831, + "step": 8450 + }, + { + "epoch": 1.3660806200924487, + "grad_norm": 0.6937337517738342, + "learning_rate": 0.0001, + "loss": 69.8321, + "step": 8460 + }, + { + "epoch": 1.36769544417755, + "grad_norm": 0.7725892066955566, + "learning_rate": 0.0001, + "loss": 71.0784, + "step": 8470 + }, + { + "epoch": 1.3693102682626512, + "grad_norm": 0.7171871066093445, + "learning_rate": 0.0001, + "loss": 69.0828, + "step": 8480 + }, + { + "epoch": 1.3709250923477523, + "grad_norm": 0.6605113744735718, + "learning_rate": 0.0001, + "loss": 70.9461, + "step": 8490 + }, + { + "epoch": 1.3725399164328536, + "grad_norm": 0.9005604982376099, + "learning_rate": 0.0001, + "loss": 71.474, + "step": 8500 + }, + { + "epoch": 1.3741547405179548, + "grad_norm": 0.7454419732093811, + "learning_rate": 0.0001, + "loss": 70.3113, + "step": 8510 + }, + { + "epoch": 1.375769564603056, + "grad_norm": 0.7574180960655212, + "learning_rate": 0.0001, + "loss": 70.9015, + "step": 8520 + }, + { + "epoch": 1.3773843886881574, + "grad_norm": 0.6525025367736816, + "learning_rate": 0.0001, + "loss": 72.3132, + "step": 8530 + }, + { + "epoch": 1.3789992127732584, + "grad_norm": 0.6548861861228943, + "learning_rate": 0.0001, + "loss": 68.6495, + "step": 8540 + }, + { + "epoch": 1.3806140368583597, + "grad_norm": 0.7881495952606201, + "learning_rate": 0.0001, + "loss": 70.1335, + "step": 8550 + }, + { + "epoch": 1.382228860943461, + "grad_norm": 0.619380533695221, + "learning_rate": 0.0001, + "loss": 71.7469, + "step": 8560 + }, + { + "epoch": 1.3838436850285623, + "grad_norm": 0.7637182474136353, + "learning_rate": 0.0001, + "loss": 69.4691, + "step": 8570 + }, + { + "epoch": 1.3854585091136635, + "grad_norm": 0.8711991906166077, + "learning_rate": 0.0001, + "loss": 70.8131, + "step": 8580 + }, + { + "epoch": 1.3870733331987646, + "grad_norm": 0.6550050973892212, + "learning_rate": 0.0001, + "loss": 70.0096, + "step": 8590 + }, + { + "epoch": 1.3886881572838659, + "grad_norm": 0.6310496926307678, + "learning_rate": 0.0001, + "loss": 70.4305, + "step": 8600 + }, + { + "epoch": 1.3903029813689671, + "grad_norm": 0.5780142545700073, + "learning_rate": 0.0001, + "loss": 68.9639, + "step": 8610 + }, + { + "epoch": 1.3919178054540684, + "grad_norm": 0.6526662111282349, + "learning_rate": 0.0001, + "loss": 69.1345, + "step": 8620 + }, + { + "epoch": 1.3935326295391697, + "grad_norm": 1.2579848766326904, + "learning_rate": 0.0001, + "loss": 71.6908, + "step": 8630 + }, + { + "epoch": 1.3951474536242707, + "grad_norm": 0.7702625393867493, + "learning_rate": 0.0001, + "loss": 69.1064, + "step": 8640 + }, + { + "epoch": 1.396762277709372, + "grad_norm": 0.7414085865020752, + "learning_rate": 0.0001, + "loss": 69.5106, + "step": 8650 + }, + { + "epoch": 1.3983771017944733, + "grad_norm": 0.666235089302063, + "learning_rate": 0.0001, + "loss": 67.938, + "step": 8660 + }, + { + "epoch": 1.3999919258795746, + "grad_norm": 0.7879519462585449, + "learning_rate": 0.0001, + "loss": 69.5422, + "step": 8670 + }, + { + "epoch": 1.4016067499646758, + "grad_norm": 0.6552250981330872, + "learning_rate": 0.0001, + "loss": 69.4161, + "step": 8680 + }, + { + "epoch": 1.4032215740497769, + "grad_norm": 0.7558363676071167, + "learning_rate": 0.0001, + "loss": 71.3052, + "step": 8690 + }, + { + "epoch": 1.4048363981348782, + "grad_norm": 0.677007794380188, + "learning_rate": 0.0001, + "loss": 70.0824, + "step": 8700 + }, + { + "epoch": 1.4064512222199794, + "grad_norm": 0.6416738629341125, + "learning_rate": 0.0001, + "loss": 68.3357, + "step": 8710 + }, + { + "epoch": 1.4080660463050807, + "grad_norm": 0.8067912459373474, + "learning_rate": 0.0001, + "loss": 69.8551, + "step": 8720 + }, + { + "epoch": 1.409680870390182, + "grad_norm": 0.6017255783081055, + "learning_rate": 0.0001, + "loss": 70.3936, + "step": 8730 + }, + { + "epoch": 1.411295694475283, + "grad_norm": 0.5473611354827881, + "learning_rate": 0.0001, + "loss": 69.1556, + "step": 8740 + }, + { + "epoch": 1.4129105185603843, + "grad_norm": 0.641984760761261, + "learning_rate": 0.0001, + "loss": 71.5302, + "step": 8750 + }, + { + "epoch": 1.4145253426454856, + "grad_norm": 0.7509932518005371, + "learning_rate": 0.0001, + "loss": 68.793, + "step": 8760 + }, + { + "epoch": 1.4161401667305868, + "grad_norm": 0.615459144115448, + "learning_rate": 0.0001, + "loss": 69.0787, + "step": 8770 + }, + { + "epoch": 1.4177549908156881, + "grad_norm": 0.625276505947113, + "learning_rate": 0.0001, + "loss": 68.5643, + "step": 8780 + }, + { + "epoch": 1.4193698149007892, + "grad_norm": 0.7294536828994751, + "learning_rate": 0.0001, + "loss": 71.1114, + "step": 8790 + }, + { + "epoch": 1.4209846389858904, + "grad_norm": 0.8188890814781189, + "learning_rate": 0.0001, + "loss": 70.7486, + "step": 8800 + }, + { + "epoch": 1.4225994630709917, + "grad_norm": 0.8662137389183044, + "learning_rate": 0.0001, + "loss": 69.0625, + "step": 8810 + }, + { + "epoch": 1.424214287156093, + "grad_norm": 0.7548071146011353, + "learning_rate": 0.0001, + "loss": 68.9828, + "step": 8820 + }, + { + "epoch": 1.4258291112411943, + "grad_norm": 0.7295185327529907, + "learning_rate": 0.0001, + "loss": 69.0606, + "step": 8830 + }, + { + "epoch": 1.4274439353262953, + "grad_norm": 0.7937467098236084, + "learning_rate": 0.0001, + "loss": 69.0135, + "step": 8840 + }, + { + "epoch": 1.4290587594113966, + "grad_norm": 0.5957842469215393, + "learning_rate": 0.0001, + "loss": 68.3992, + "step": 8850 + }, + { + "epoch": 1.4306735834964979, + "grad_norm": 0.786580502986908, + "learning_rate": 0.0001, + "loss": 69.4485, + "step": 8860 + }, + { + "epoch": 1.4322884075815991, + "grad_norm": 0.8539725542068481, + "learning_rate": 0.0001, + "loss": 67.1037, + "step": 8870 + }, + { + "epoch": 1.4339032316667004, + "grad_norm": 0.684476375579834, + "learning_rate": 0.0001, + "loss": 68.9814, + "step": 8880 + }, + { + "epoch": 1.4355180557518015, + "grad_norm": 0.6335768103599548, + "learning_rate": 0.0001, + "loss": 69.9716, + "step": 8890 + }, + { + "epoch": 1.4371328798369027, + "grad_norm": 0.9881808757781982, + "learning_rate": 0.0001, + "loss": 68.0338, + "step": 8900 + }, + { + "epoch": 1.438747703922004, + "grad_norm": 0.783983588218689, + "learning_rate": 0.0001, + "loss": 70.124, + "step": 8910 + }, + { + "epoch": 1.4403625280071053, + "grad_norm": 0.817820131778717, + "learning_rate": 0.0001, + "loss": 67.933, + "step": 8920 + }, + { + "epoch": 1.4419773520922066, + "grad_norm": 0.6900199055671692, + "learning_rate": 0.0001, + "loss": 69.8871, + "step": 8930 + }, + { + "epoch": 1.4435921761773076, + "grad_norm": 0.6702716946601868, + "learning_rate": 0.0001, + "loss": 71.921, + "step": 8940 + }, + { + "epoch": 1.445207000262409, + "grad_norm": 0.6699396967887878, + "learning_rate": 0.0001, + "loss": 67.8654, + "step": 8950 + }, + { + "epoch": 1.4468218243475102, + "grad_norm": 0.6388489603996277, + "learning_rate": 0.0001, + "loss": 69.5605, + "step": 8960 + }, + { + "epoch": 1.4484366484326114, + "grad_norm": 0.735910952091217, + "learning_rate": 0.0001, + "loss": 66.7865, + "step": 8970 + }, + { + "epoch": 1.4500514725177127, + "grad_norm": 0.6826426386833191, + "learning_rate": 0.0001, + "loss": 68.2896, + "step": 8980 + }, + { + "epoch": 1.4516662966028138, + "grad_norm": 0.8190213441848755, + "learning_rate": 0.0001, + "loss": 70.3524, + "step": 8990 + }, + { + "epoch": 1.453281120687915, + "grad_norm": 0.6774603724479675, + "learning_rate": 0.0001, + "loss": 68.9084, + "step": 9000 + }, + { + "epoch": 1.4548959447730163, + "grad_norm": 0.8953077793121338, + "learning_rate": 0.0001, + "loss": 69.228, + "step": 9010 + }, + { + "epoch": 1.4565107688581176, + "grad_norm": 1.8509501218795776, + "learning_rate": 0.0001, + "loss": 68.0847, + "step": 9020 + }, + { + "epoch": 1.4581255929432189, + "grad_norm": 0.671280026435852, + "learning_rate": 0.0001, + "loss": 70.0691, + "step": 9030 + }, + { + "epoch": 1.45974041702832, + "grad_norm": 0.7242400646209717, + "learning_rate": 0.0001, + "loss": 68.8001, + "step": 9040 + }, + { + "epoch": 1.4613552411134212, + "grad_norm": 0.6609033942222595, + "learning_rate": 0.0001, + "loss": 69.0398, + "step": 9050 + }, + { + "epoch": 1.4629700651985225, + "grad_norm": 0.6850711703300476, + "learning_rate": 0.0001, + "loss": 70.8849, + "step": 9060 + }, + { + "epoch": 1.4645848892836237, + "grad_norm": 0.7409449219703674, + "learning_rate": 0.0001, + "loss": 68.0411, + "step": 9070 + }, + { + "epoch": 1.466199713368725, + "grad_norm": 0.6832754611968994, + "learning_rate": 0.0001, + "loss": 69.1219, + "step": 9080 + }, + { + "epoch": 1.467814537453826, + "grad_norm": 0.6671826243400574, + "learning_rate": 0.0001, + "loss": 68.1225, + "step": 9090 + }, + { + "epoch": 1.4694293615389273, + "grad_norm": 0.6758706569671631, + "learning_rate": 0.0001, + "loss": 68.6981, + "step": 9100 + }, + { + "epoch": 1.4710441856240286, + "grad_norm": 0.7885150909423828, + "learning_rate": 0.0001, + "loss": 69.29, + "step": 9110 + }, + { + "epoch": 1.4726590097091299, + "grad_norm": 0.7636108994483948, + "learning_rate": 0.0001, + "loss": 67.3151, + "step": 9120 + }, + { + "epoch": 1.4742738337942312, + "grad_norm": 0.6518180966377258, + "learning_rate": 0.0001, + "loss": 68.6208, + "step": 9130 + }, + { + "epoch": 1.4758886578793322, + "grad_norm": 0.6715161204338074, + "learning_rate": 0.0001, + "loss": 70.5106, + "step": 9140 + }, + { + "epoch": 1.4775034819644335, + "grad_norm": 0.6342995762825012, + "learning_rate": 0.0001, + "loss": 70.3067, + "step": 9150 + }, + { + "epoch": 1.4791183060495348, + "grad_norm": 0.6773675084114075, + "learning_rate": 0.0001, + "loss": 69.2888, + "step": 9160 + }, + { + "epoch": 1.480733130134636, + "grad_norm": 0.6887578368186951, + "learning_rate": 0.0001, + "loss": 69.2788, + "step": 9170 + }, + { + "epoch": 1.4823479542197373, + "grad_norm": 0.6793885231018066, + "learning_rate": 0.0001, + "loss": 68.3523, + "step": 9180 + }, + { + "epoch": 1.4839627783048384, + "grad_norm": 0.7483563423156738, + "learning_rate": 0.0001, + "loss": 67.9991, + "step": 9190 + }, + { + "epoch": 1.4855776023899396, + "grad_norm": 0.7007381916046143, + "learning_rate": 0.0001, + "loss": 67.5335, + "step": 9200 + }, + { + "epoch": 1.487192426475041, + "grad_norm": 0.7565139532089233, + "learning_rate": 0.0001, + "loss": 69.1093, + "step": 9210 + }, + { + "epoch": 1.4888072505601422, + "grad_norm": 1.0212360620498657, + "learning_rate": 0.0001, + "loss": 68.5061, + "step": 9220 + }, + { + "epoch": 1.4904220746452435, + "grad_norm": 0.9198092818260193, + "learning_rate": 0.0001, + "loss": 67.9735, + "step": 9230 + }, + { + "epoch": 1.4920368987303445, + "grad_norm": 0.6228001713752747, + "learning_rate": 0.0001, + "loss": 69.2231, + "step": 9240 + }, + { + "epoch": 1.4936517228154458, + "grad_norm": 0.6618618965148926, + "learning_rate": 0.0001, + "loss": 68.0434, + "step": 9250 + }, + { + "epoch": 1.495266546900547, + "grad_norm": 0.6241064071655273, + "learning_rate": 0.0001, + "loss": 68.4004, + "step": 9260 + }, + { + "epoch": 1.4968813709856483, + "grad_norm": 0.7336916327476501, + "learning_rate": 0.0001, + "loss": 69.0389, + "step": 9270 + }, + { + "epoch": 1.4984961950707496, + "grad_norm": 0.718125581741333, + "learning_rate": 0.0001, + "loss": 68.4923, + "step": 9280 + }, + { + "epoch": 1.5001110191558507, + "grad_norm": 0.6694948673248291, + "learning_rate": 0.0001, + "loss": 68.1006, + "step": 9290 + }, + { + "epoch": 1.501725843240952, + "grad_norm": 0.5815121531486511, + "learning_rate": 0.0001, + "loss": 68.5943, + "step": 9300 + }, + { + "epoch": 1.5033406673260532, + "grad_norm": 1.4076049327850342, + "learning_rate": 0.0001, + "loss": 68.1251, + "step": 9310 + }, + { + "epoch": 1.5049554914111543, + "grad_norm": 0.7294307947158813, + "learning_rate": 0.0001, + "loss": 68.0539, + "step": 9320 + }, + { + "epoch": 1.5065703154962558, + "grad_norm": 0.627821683883667, + "learning_rate": 0.0001, + "loss": 68.6859, + "step": 9330 + }, + { + "epoch": 1.5081851395813568, + "grad_norm": 0.6683267951011658, + "learning_rate": 0.0001, + "loss": 68.2727, + "step": 9340 + }, + { + "epoch": 1.509799963666458, + "grad_norm": 0.7070158123970032, + "learning_rate": 0.0001, + "loss": 69.1171, + "step": 9350 + }, + { + "epoch": 1.5114147877515594, + "grad_norm": 0.6324070692062378, + "learning_rate": 0.0001, + "loss": 69.7498, + "step": 9360 + }, + { + "epoch": 1.5130296118366604, + "grad_norm": 0.9814477562904358, + "learning_rate": 0.0001, + "loss": 69.1426, + "step": 9370 + }, + { + "epoch": 1.514644435921762, + "grad_norm": 0.5883511304855347, + "learning_rate": 0.0001, + "loss": 68.5363, + "step": 9380 + }, + { + "epoch": 1.516259260006863, + "grad_norm": 0.8325281739234924, + "learning_rate": 0.0001, + "loss": 67.5951, + "step": 9390 + }, + { + "epoch": 1.5178740840919642, + "grad_norm": 0.7456872463226318, + "learning_rate": 0.0001, + "loss": 68.1876, + "step": 9400 + }, + { + "epoch": 1.5194889081770655, + "grad_norm": 0.7692755460739136, + "learning_rate": 0.0001, + "loss": 69.1677, + "step": 9410 + }, + { + "epoch": 1.5211037322621666, + "grad_norm": 0.7346193194389343, + "learning_rate": 0.0001, + "loss": 69.8608, + "step": 9420 + }, + { + "epoch": 1.522718556347268, + "grad_norm": 0.7658166289329529, + "learning_rate": 0.0001, + "loss": 68.8584, + "step": 9430 + }, + { + "epoch": 1.524333380432369, + "grad_norm": 0.7892371416091919, + "learning_rate": 0.0001, + "loss": 68.3882, + "step": 9440 + }, + { + "epoch": 1.5259482045174704, + "grad_norm": 0.5943968296051025, + "learning_rate": 0.0001, + "loss": 68.7935, + "step": 9450 + }, + { + "epoch": 1.5275630286025716, + "grad_norm": 0.6611922383308411, + "learning_rate": 0.0001, + "loss": 69.0865, + "step": 9460 + }, + { + "epoch": 1.5291778526876727, + "grad_norm": 0.6697778105735779, + "learning_rate": 0.0001, + "loss": 68.6131, + "step": 9470 + }, + { + "epoch": 1.5307926767727742, + "grad_norm": 0.862293541431427, + "learning_rate": 0.0001, + "loss": 71.0167, + "step": 9480 + }, + { + "epoch": 1.5324075008578752, + "grad_norm": 0.7098677754402161, + "learning_rate": 0.0001, + "loss": 67.7216, + "step": 9490 + }, + { + "epoch": 1.5340223249429765, + "grad_norm": 0.6578338742256165, + "learning_rate": 0.0001, + "loss": 69.872, + "step": 9500 + }, + { + "epoch": 1.5356371490280778, + "grad_norm": 0.7272375822067261, + "learning_rate": 0.0001, + "loss": 70.0971, + "step": 9510 + }, + { + "epoch": 1.5372519731131788, + "grad_norm": 0.6497064828872681, + "learning_rate": 0.0001, + "loss": 70.5921, + "step": 9520 + }, + { + "epoch": 1.5388667971982803, + "grad_norm": 0.6691288352012634, + "learning_rate": 0.0001, + "loss": 68.5213, + "step": 9530 + }, + { + "epoch": 1.5404816212833814, + "grad_norm": 0.9180259108543396, + "learning_rate": 0.0001, + "loss": 67.0699, + "step": 9540 + }, + { + "epoch": 1.5420964453684827, + "grad_norm": 0.7062082886695862, + "learning_rate": 0.0001, + "loss": 69.3965, + "step": 9550 + }, + { + "epoch": 1.543711269453584, + "grad_norm": 0.711613655090332, + "learning_rate": 0.0001, + "loss": 67.8276, + "step": 9560 + }, + { + "epoch": 1.545326093538685, + "grad_norm": 0.670032799243927, + "learning_rate": 0.0001, + "loss": 68.1687, + "step": 9570 + }, + { + "epoch": 1.5469409176237865, + "grad_norm": 0.7531955242156982, + "learning_rate": 0.0001, + "loss": 69.2144, + "step": 9580 + }, + { + "epoch": 1.5485557417088875, + "grad_norm": 0.6759967803955078, + "learning_rate": 0.0001, + "loss": 69.0756, + "step": 9590 + }, + { + "epoch": 1.5501705657939888, + "grad_norm": 0.6134868860244751, + "learning_rate": 0.0001, + "loss": 67.4348, + "step": 9600 + }, + { + "epoch": 1.55178538987909, + "grad_norm": 0.638785719871521, + "learning_rate": 0.0001, + "loss": 68.4635, + "step": 9610 + }, + { + "epoch": 1.5534002139641911, + "grad_norm": 0.7901169657707214, + "learning_rate": 0.0001, + "loss": 68.1079, + "step": 9620 + }, + { + "epoch": 1.5550150380492926, + "grad_norm": 0.6970444917678833, + "learning_rate": 0.0001, + "loss": 69.3883, + "step": 9630 + }, + { + "epoch": 1.5566298621343937, + "grad_norm": 0.6954355239868164, + "learning_rate": 0.0001, + "loss": 69.3841, + "step": 9640 + }, + { + "epoch": 1.558244686219495, + "grad_norm": 0.7097641825675964, + "learning_rate": 0.0001, + "loss": 68.3417, + "step": 9650 + }, + { + "epoch": 1.5598595103045962, + "grad_norm": 0.7082354426383972, + "learning_rate": 0.0001, + "loss": 69.9314, + "step": 9660 + }, + { + "epoch": 1.5614743343896973, + "grad_norm": 0.6570103764533997, + "learning_rate": 0.0001, + "loss": 69.7219, + "step": 9670 + }, + { + "epoch": 1.5630891584747988, + "grad_norm": 0.5705522894859314, + "learning_rate": 0.0001, + "loss": 67.5177, + "step": 9680 + }, + { + "epoch": 1.5647039825598998, + "grad_norm": 0.6992695331573486, + "learning_rate": 0.0001, + "loss": 67.6538, + "step": 9690 + }, + { + "epoch": 1.5663188066450011, + "grad_norm": 0.7523599863052368, + "learning_rate": 0.0001, + "loss": 66.869, + "step": 9700 + }, + { + "epoch": 1.5679336307301024, + "grad_norm": 0.6267555356025696, + "learning_rate": 0.0001, + "loss": 66.6805, + "step": 9710 + }, + { + "epoch": 1.5695484548152034, + "grad_norm": 0.6138116717338562, + "learning_rate": 0.0001, + "loss": 71.3217, + "step": 9720 + }, + { + "epoch": 1.571163278900305, + "grad_norm": 0.7408527731895447, + "learning_rate": 0.0001, + "loss": 69.2759, + "step": 9730 + }, + { + "epoch": 1.572778102985406, + "grad_norm": 0.813761830329895, + "learning_rate": 0.0001, + "loss": 70.4959, + "step": 9740 + }, + { + "epoch": 1.5743929270705073, + "grad_norm": 1.0363599061965942, + "learning_rate": 0.0001, + "loss": 68.1996, + "step": 9750 + }, + { + "epoch": 1.5760077511556085, + "grad_norm": 0.7773903012275696, + "learning_rate": 0.0001, + "loss": 67.9354, + "step": 9760 + }, + { + "epoch": 1.5776225752407096, + "grad_norm": 0.6465738415718079, + "learning_rate": 0.0001, + "loss": 69.5247, + "step": 9770 + }, + { + "epoch": 1.579237399325811, + "grad_norm": 0.8433385491371155, + "learning_rate": 0.0001, + "loss": 66.9448, + "step": 9780 + }, + { + "epoch": 1.5808522234109121, + "grad_norm": 0.8651545643806458, + "learning_rate": 0.0001, + "loss": 68.6994, + "step": 9790 + }, + { + "epoch": 1.5824670474960134, + "grad_norm": 0.6940448880195618, + "learning_rate": 0.0001, + "loss": 67.622, + "step": 9800 + }, + { + "epoch": 1.5840818715811147, + "grad_norm": 0.622756838798523, + "learning_rate": 0.0001, + "loss": 69.0994, + "step": 9810 + }, + { + "epoch": 1.5856966956662157, + "grad_norm": 0.6119604110717773, + "learning_rate": 0.0001, + "loss": 67.7682, + "step": 9820 + }, + { + "epoch": 1.5873115197513172, + "grad_norm": 0.5729681849479675, + "learning_rate": 0.0001, + "loss": 67.3113, + "step": 9830 + }, + { + "epoch": 1.5889263438364183, + "grad_norm": 0.7143844366073608, + "learning_rate": 0.0001, + "loss": 68.8128, + "step": 9840 + }, + { + "epoch": 1.5905411679215196, + "grad_norm": 0.9814873337745667, + "learning_rate": 0.0001, + "loss": 69.3099, + "step": 9850 + }, + { + "epoch": 1.5921559920066208, + "grad_norm": 0.8044945001602173, + "learning_rate": 0.0001, + "loss": 69.3939, + "step": 9860 + }, + { + "epoch": 1.5937708160917219, + "grad_norm": 0.8494839668273926, + "learning_rate": 0.0001, + "loss": 69.3889, + "step": 9870 + }, + { + "epoch": 1.5953856401768234, + "grad_norm": 0.6468870639801025, + "learning_rate": 0.0001, + "loss": 67.2047, + "step": 9880 + }, + { + "epoch": 1.5970004642619244, + "grad_norm": 0.6270474791526794, + "learning_rate": 0.0001, + "loss": 69.6813, + "step": 9890 + }, + { + "epoch": 1.5986152883470257, + "grad_norm": 0.7102645039558411, + "learning_rate": 0.0001, + "loss": 69.1546, + "step": 9900 + }, + { + "epoch": 1.600230112432127, + "grad_norm": 0.5929123759269714, + "learning_rate": 0.0001, + "loss": 67.2187, + "step": 9910 + }, + { + "epoch": 1.601844936517228, + "grad_norm": 0.6466200947761536, + "learning_rate": 0.0001, + "loss": 68.6876, + "step": 9920 + }, + { + "epoch": 1.6034597606023295, + "grad_norm": 0.6441522836685181, + "learning_rate": 0.0001, + "loss": 68.7955, + "step": 9930 + }, + { + "epoch": 1.6050745846874306, + "grad_norm": 0.7264062166213989, + "learning_rate": 0.0001, + "loss": 67.8876, + "step": 9940 + }, + { + "epoch": 1.6066894087725319, + "grad_norm": 0.6973199248313904, + "learning_rate": 0.0001, + "loss": 69.0904, + "step": 9950 + }, + { + "epoch": 1.6083042328576331, + "grad_norm": 0.582053005695343, + "learning_rate": 0.0001, + "loss": 69.3876, + "step": 9960 + }, + { + "epoch": 1.6099190569427342, + "grad_norm": 0.6431038975715637, + "learning_rate": 0.0001, + "loss": 67.4232, + "step": 9970 + }, + { + "epoch": 1.6115338810278357, + "grad_norm": 0.6313247084617615, + "learning_rate": 0.0001, + "loss": 68.3759, + "step": 9980 + }, + { + "epoch": 1.6131487051129367, + "grad_norm": 1.2003700733184814, + "learning_rate": 0.0001, + "loss": 68.3413, + "step": 9990 + }, + { + "epoch": 1.614763529198038, + "grad_norm": 1.0262068510055542, + "learning_rate": 0.0001, + "loss": 66.7308, + "step": 10000 + }, + { + "epoch": 1.6163783532831393, + "grad_norm": 0.5975973010063171, + "learning_rate": 0.0001, + "loss": 68.6893, + "step": 10010 + }, + { + "epoch": 1.6179931773682403, + "grad_norm": 0.6045264601707458, + "learning_rate": 0.0001, + "loss": 69.5241, + "step": 10020 + }, + { + "epoch": 1.6196080014533418, + "grad_norm": 0.654019832611084, + "learning_rate": 0.0001, + "loss": 66.1208, + "step": 10030 + }, + { + "epoch": 1.6212228255384429, + "grad_norm": 0.533981204032898, + "learning_rate": 0.0001, + "loss": 68.2129, + "step": 10040 + }, + { + "epoch": 1.6228376496235442, + "grad_norm": 1.184734582901001, + "learning_rate": 0.0001, + "loss": 70.0547, + "step": 10050 + }, + { + "epoch": 1.6244524737086454, + "grad_norm": 0.5836975574493408, + "learning_rate": 0.0001, + "loss": 68.1098, + "step": 10060 + }, + { + "epoch": 1.6260672977937465, + "grad_norm": 0.5856032967567444, + "learning_rate": 0.0001, + "loss": 67.483, + "step": 10070 + }, + { + "epoch": 1.627682121878848, + "grad_norm": 0.6735382080078125, + "learning_rate": 0.0001, + "loss": 68.8628, + "step": 10080 + }, + { + "epoch": 1.629296945963949, + "grad_norm": 0.5753204226493835, + "learning_rate": 0.0001, + "loss": 66.2777, + "step": 10090 + }, + { + "epoch": 1.6309117700490503, + "grad_norm": 0.8327603340148926, + "learning_rate": 0.0001, + "loss": 67.9309, + "step": 10100 + }, + { + "epoch": 1.6325265941341516, + "grad_norm": 0.6873740553855896, + "learning_rate": 0.0001, + "loss": 68.7627, + "step": 10110 + }, + { + "epoch": 1.6341414182192526, + "grad_norm": 0.7218474745750427, + "learning_rate": 0.0001, + "loss": 69.3637, + "step": 10120 + }, + { + "epoch": 1.6357562423043541, + "grad_norm": 0.8393228650093079, + "learning_rate": 0.0001, + "loss": 66.7686, + "step": 10130 + }, + { + "epoch": 1.6373710663894552, + "grad_norm": 0.6433869004249573, + "learning_rate": 0.0001, + "loss": 68.0896, + "step": 10140 + }, + { + "epoch": 1.6389858904745565, + "grad_norm": 0.7419982552528381, + "learning_rate": 0.0001, + "loss": 67.9707, + "step": 10150 + }, + { + "epoch": 1.6406007145596577, + "grad_norm": 0.5884562730789185, + "learning_rate": 0.0001, + "loss": 68.0287, + "step": 10160 + }, + { + "epoch": 1.6422155386447588, + "grad_norm": 0.729557454586029, + "learning_rate": 0.0001, + "loss": 69.942, + "step": 10170 + }, + { + "epoch": 1.6438303627298603, + "grad_norm": 0.6026669144630432, + "learning_rate": 0.0001, + "loss": 66.8192, + "step": 10180 + }, + { + "epoch": 1.6454451868149613, + "grad_norm": 0.6921732425689697, + "learning_rate": 0.0001, + "loss": 66.7087, + "step": 10190 + }, + { + "epoch": 1.6470600109000626, + "grad_norm": 0.755172848701477, + "learning_rate": 0.0001, + "loss": 70.799, + "step": 10200 + }, + { + "epoch": 1.6486748349851639, + "grad_norm": 0.7318310737609863, + "learning_rate": 0.0001, + "loss": 67.6957, + "step": 10210 + }, + { + "epoch": 1.650289659070265, + "grad_norm": 0.6915041208267212, + "learning_rate": 0.0001, + "loss": 69.3388, + "step": 10220 + }, + { + "epoch": 1.6519044831553664, + "grad_norm": 0.7173444628715515, + "learning_rate": 0.0001, + "loss": 68.3612, + "step": 10230 + }, + { + "epoch": 1.6535193072404675, + "grad_norm": 0.7485487461090088, + "learning_rate": 0.0001, + "loss": 68.5366, + "step": 10240 + }, + { + "epoch": 1.6551341313255687, + "grad_norm": 0.5838190913200378, + "learning_rate": 0.0001, + "loss": 68.3841, + "step": 10250 + }, + { + "epoch": 1.65674895541067, + "grad_norm": 0.5908819437026978, + "learning_rate": 0.0001, + "loss": 67.8741, + "step": 10260 + }, + { + "epoch": 1.658363779495771, + "grad_norm": 0.7475612163543701, + "learning_rate": 0.0001, + "loss": 69.0636, + "step": 10270 + }, + { + "epoch": 1.6599786035808726, + "grad_norm": 0.5451382994651794, + "learning_rate": 0.0001, + "loss": 68.6695, + "step": 10280 + }, + { + "epoch": 1.6615934276659736, + "grad_norm": 0.6277157068252563, + "learning_rate": 0.0001, + "loss": 67.5432, + "step": 10290 + }, + { + "epoch": 1.663208251751075, + "grad_norm": 0.8144198060035706, + "learning_rate": 0.0001, + "loss": 66.7892, + "step": 10300 + }, + { + "epoch": 1.6648230758361762, + "grad_norm": 0.7400529980659485, + "learning_rate": 0.0001, + "loss": 66.2653, + "step": 10310 + }, + { + "epoch": 1.6664378999212772, + "grad_norm": 0.74524986743927, + "learning_rate": 0.0001, + "loss": 67.8252, + "step": 10320 + }, + { + "epoch": 1.6680527240063787, + "grad_norm": 0.5554162263870239, + "learning_rate": 0.0001, + "loss": 69.7704, + "step": 10330 + }, + { + "epoch": 1.6696675480914798, + "grad_norm": 0.8194338083267212, + "learning_rate": 0.0001, + "loss": 69.4668, + "step": 10340 + }, + { + "epoch": 1.671282372176581, + "grad_norm": 0.7947783470153809, + "learning_rate": 0.0001, + "loss": 64.9276, + "step": 10350 + }, + { + "epoch": 1.6728971962616823, + "grad_norm": 0.6145230531692505, + "learning_rate": 0.0001, + "loss": 68.1815, + "step": 10360 + }, + { + "epoch": 1.6745120203467834, + "grad_norm": 0.6293516755104065, + "learning_rate": 0.0001, + "loss": 67.8994, + "step": 10370 + }, + { + "epoch": 1.6761268444318849, + "grad_norm": 0.6160871386528015, + "learning_rate": 0.0001, + "loss": 66.7463, + "step": 10380 + }, + { + "epoch": 1.677741668516986, + "grad_norm": 0.6925204992294312, + "learning_rate": 0.0001, + "loss": 68.5261, + "step": 10390 + }, + { + "epoch": 1.6793564926020872, + "grad_norm": 0.6892788410186768, + "learning_rate": 0.0001, + "loss": 66.8928, + "step": 10400 + }, + { + "epoch": 1.6809713166871885, + "grad_norm": 0.9066945910453796, + "learning_rate": 0.0001, + "loss": 66.2556, + "step": 10410 + }, + { + "epoch": 1.6825861407722895, + "grad_norm": 0.5857477784156799, + "learning_rate": 0.0001, + "loss": 67.1866, + "step": 10420 + }, + { + "epoch": 1.684200964857391, + "grad_norm": 0.7989841103553772, + "learning_rate": 0.0001, + "loss": 67.7762, + "step": 10430 + }, + { + "epoch": 1.685815788942492, + "grad_norm": 0.559428870677948, + "learning_rate": 0.0001, + "loss": 67.5628, + "step": 10440 + }, + { + "epoch": 1.6874306130275933, + "grad_norm": 0.6718077659606934, + "learning_rate": 0.0001, + "loss": 69.2877, + "step": 10450 + }, + { + "epoch": 1.6890454371126946, + "grad_norm": 0.6952136754989624, + "learning_rate": 0.0001, + "loss": 67.2734, + "step": 10460 + }, + { + "epoch": 1.6906602611977957, + "grad_norm": 0.6079897880554199, + "learning_rate": 0.0001, + "loss": 67.4677, + "step": 10470 + }, + { + "epoch": 1.692275085282897, + "grad_norm": 0.6139684915542603, + "learning_rate": 0.0001, + "loss": 69.2464, + "step": 10480 + }, + { + "epoch": 1.6938899093679982, + "grad_norm": 0.7009351253509521, + "learning_rate": 0.0001, + "loss": 67.5424, + "step": 10490 + }, + { + "epoch": 1.6955047334530995, + "grad_norm": 0.8567383885383606, + "learning_rate": 0.0001, + "loss": 65.6573, + "step": 10500 + }, + { + "epoch": 1.6971195575382008, + "grad_norm": 0.6903256177902222, + "learning_rate": 0.0001, + "loss": 69.2218, + "step": 10510 + }, + { + "epoch": 1.6987343816233018, + "grad_norm": 0.5958806872367859, + "learning_rate": 0.0001, + "loss": 67.3718, + "step": 10520 + }, + { + "epoch": 1.700349205708403, + "grad_norm": 0.5983722805976868, + "learning_rate": 0.0001, + "loss": 68.7944, + "step": 10530 + }, + { + "epoch": 1.7019640297935044, + "grad_norm": 0.8119397759437561, + "learning_rate": 0.0001, + "loss": 66.7936, + "step": 10540 + }, + { + "epoch": 1.7035788538786056, + "grad_norm": 0.6383484601974487, + "learning_rate": 0.0001, + "loss": 67.8636, + "step": 10550 + }, + { + "epoch": 1.705193677963707, + "grad_norm": 0.6574084162712097, + "learning_rate": 0.0001, + "loss": 67.26, + "step": 10560 + }, + { + "epoch": 1.706808502048808, + "grad_norm": 0.6716108918190002, + "learning_rate": 0.0001, + "loss": 66.485, + "step": 10570 + }, + { + "epoch": 1.7084233261339092, + "grad_norm": 0.6998292803764343, + "learning_rate": 0.0001, + "loss": 66.4201, + "step": 10580 + }, + { + "epoch": 1.7100381502190105, + "grad_norm": 0.7300788760185242, + "learning_rate": 0.0001, + "loss": 67.1446, + "step": 10590 + }, + { + "epoch": 1.7116529743041118, + "grad_norm": 0.7066853642463684, + "learning_rate": 0.0001, + "loss": 70.8954, + "step": 10600 + }, + { + "epoch": 1.713267798389213, + "grad_norm": 0.6933701038360596, + "learning_rate": 0.0001, + "loss": 67.8473, + "step": 10610 + }, + { + "epoch": 1.7148826224743141, + "grad_norm": 0.6885779500007629, + "learning_rate": 0.0001, + "loss": 65.3276, + "step": 10620 + }, + { + "epoch": 1.7164974465594154, + "grad_norm": 0.6542202830314636, + "learning_rate": 0.0001, + "loss": 69.2596, + "step": 10630 + }, + { + "epoch": 1.7181122706445167, + "grad_norm": 0.5719691514968872, + "learning_rate": 0.0001, + "loss": 69.1907, + "step": 10640 + }, + { + "epoch": 1.719727094729618, + "grad_norm": 0.6596891283988953, + "learning_rate": 0.0001, + "loss": 69.0778, + "step": 10650 + }, + { + "epoch": 1.7213419188147192, + "grad_norm": 0.6860799193382263, + "learning_rate": 0.0001, + "loss": 68.1416, + "step": 10660 + }, + { + "epoch": 1.7229567428998203, + "grad_norm": 0.7912564277648926, + "learning_rate": 0.0001, + "loss": 68.2091, + "step": 10670 + }, + { + "epoch": 1.7245715669849215, + "grad_norm": 0.6673903465270996, + "learning_rate": 0.0001, + "loss": 68.24, + "step": 10680 + }, + { + "epoch": 1.7261863910700228, + "grad_norm": 0.6243769526481628, + "learning_rate": 0.0001, + "loss": 68.6396, + "step": 10690 + }, + { + "epoch": 1.727801215155124, + "grad_norm": 0.6622587442398071, + "learning_rate": 0.0001, + "loss": 67.9406, + "step": 10700 + }, + { + "epoch": 1.7294160392402254, + "grad_norm": 0.6808138489723206, + "learning_rate": 0.0001, + "loss": 69.1403, + "step": 10710 + }, + { + "epoch": 1.7310308633253264, + "grad_norm": 0.8498415350914001, + "learning_rate": 0.0001, + "loss": 70.1438, + "step": 10720 + }, + { + "epoch": 1.7326456874104277, + "grad_norm": 1.0091742277145386, + "learning_rate": 0.0001, + "loss": 67.0196, + "step": 10730 + }, + { + "epoch": 1.734260511495529, + "grad_norm": 0.6997767686843872, + "learning_rate": 0.0001, + "loss": 67.0307, + "step": 10740 + }, + { + "epoch": 1.7358753355806302, + "grad_norm": 0.6618793606758118, + "learning_rate": 0.0001, + "loss": 67.906, + "step": 10750 + }, + { + "epoch": 1.7374901596657315, + "grad_norm": 0.7082014679908752, + "learning_rate": 0.0001, + "loss": 68.0665, + "step": 10760 + }, + { + "epoch": 1.7391049837508326, + "grad_norm": 0.8494391441345215, + "learning_rate": 0.0001, + "loss": 68.4202, + "step": 10770 + }, + { + "epoch": 1.7407198078359338, + "grad_norm": 0.6247998476028442, + "learning_rate": 0.0001, + "loss": 66.4399, + "step": 10780 + }, + { + "epoch": 1.742334631921035, + "grad_norm": 0.8239936828613281, + "learning_rate": 0.0001, + "loss": 68.4374, + "step": 10790 + }, + { + "epoch": 1.7439494560061364, + "grad_norm": 0.6734293103218079, + "learning_rate": 0.0001, + "loss": 66.3411, + "step": 10800 + }, + { + "epoch": 1.7455642800912377, + "grad_norm": 0.7066450119018555, + "learning_rate": 0.0001, + "loss": 67.7102, + "step": 10810 + }, + { + "epoch": 1.7471791041763387, + "grad_norm": 0.7440647482872009, + "learning_rate": 0.0001, + "loss": 67.7221, + "step": 10820 + }, + { + "epoch": 1.74879392826144, + "grad_norm": 0.7771329283714294, + "learning_rate": 0.0001, + "loss": 67.5907, + "step": 10830 + }, + { + "epoch": 1.7504087523465413, + "grad_norm": 1.2484230995178223, + "learning_rate": 0.0001, + "loss": 66.3242, + "step": 10840 + }, + { + "epoch": 1.7520235764316425, + "grad_norm": 0.6946772933006287, + "learning_rate": 0.0001, + "loss": 67.8741, + "step": 10850 + }, + { + "epoch": 1.7536384005167438, + "grad_norm": 0.7676035761833191, + "learning_rate": 0.0001, + "loss": 67.3776, + "step": 10860 + }, + { + "epoch": 1.7552532246018449, + "grad_norm": 0.5534589290618896, + "learning_rate": 0.0001, + "loss": 67.6656, + "step": 10870 + }, + { + "epoch": 1.7568680486869461, + "grad_norm": 1.2745810747146606, + "learning_rate": 0.0001, + "loss": 68.5959, + "step": 10880 + }, + { + "epoch": 1.7584828727720474, + "grad_norm": 0.6241582036018372, + "learning_rate": 0.0001, + "loss": 67.7142, + "step": 10890 + }, + { + "epoch": 1.7600976968571487, + "grad_norm": 0.9308313131332397, + "learning_rate": 0.0001, + "loss": 67.0322, + "step": 10900 + }, + { + "epoch": 1.76171252094225, + "grad_norm": 0.5639570951461792, + "learning_rate": 0.0001, + "loss": 67.928, + "step": 10910 + }, + { + "epoch": 1.763327345027351, + "grad_norm": 0.6692782044410706, + "learning_rate": 0.0001, + "loss": 67.625, + "step": 10920 + }, + { + "epoch": 1.7649421691124523, + "grad_norm": 0.7322304248809814, + "learning_rate": 0.0001, + "loss": 66.2372, + "step": 10930 + }, + { + "epoch": 1.7665569931975535, + "grad_norm": 0.6275238394737244, + "learning_rate": 0.0001, + "loss": 67.7628, + "step": 10940 + }, + { + "epoch": 1.7681718172826548, + "grad_norm": 0.6832380890846252, + "learning_rate": 0.0001, + "loss": 67.6587, + "step": 10950 + }, + { + "epoch": 1.769786641367756, + "grad_norm": 0.6934792399406433, + "learning_rate": 0.0001, + "loss": 67.1411, + "step": 10960 + }, + { + "epoch": 1.7714014654528571, + "grad_norm": 0.7031943202018738, + "learning_rate": 0.0001, + "loss": 69.2734, + "step": 10970 + }, + { + "epoch": 1.7730162895379584, + "grad_norm": 0.91269451379776, + "learning_rate": 0.0001, + "loss": 68.1353, + "step": 10980 + }, + { + "epoch": 1.7746311136230597, + "grad_norm": 0.8927266001701355, + "learning_rate": 0.0001, + "loss": 68.003, + "step": 10990 + }, + { + "epoch": 1.776245937708161, + "grad_norm": 0.9115116596221924, + "learning_rate": 0.0001, + "loss": 69.1506, + "step": 11000 + }, + { + "epoch": 1.7778607617932622, + "grad_norm": 0.6055391430854797, + "learning_rate": 0.0001, + "loss": 68.9785, + "step": 11010 + }, + { + "epoch": 1.7794755858783633, + "grad_norm": 0.872809112071991, + "learning_rate": 0.0001, + "loss": 66.373, + "step": 11020 + }, + { + "epoch": 1.7810904099634646, + "grad_norm": 0.5838716626167297, + "learning_rate": 0.0001, + "loss": 69.507, + "step": 11030 + }, + { + "epoch": 1.7827052340485658, + "grad_norm": 0.6108883619308472, + "learning_rate": 0.0001, + "loss": 67.3743, + "step": 11040 + }, + { + "epoch": 1.784320058133667, + "grad_norm": 0.5457114577293396, + "learning_rate": 0.0001, + "loss": 66.1342, + "step": 11050 + }, + { + "epoch": 1.7859348822187684, + "grad_norm": 0.7696211338043213, + "learning_rate": 0.0001, + "loss": 67.9779, + "step": 11060 + }, + { + "epoch": 1.7875497063038694, + "grad_norm": 0.5795412659645081, + "learning_rate": 0.0001, + "loss": 67.1763, + "step": 11070 + }, + { + "epoch": 1.7891645303889707, + "grad_norm": 0.7401878833770752, + "learning_rate": 0.0001, + "loss": 67.4972, + "step": 11080 + }, + { + "epoch": 1.790779354474072, + "grad_norm": 0.666051983833313, + "learning_rate": 0.0001, + "loss": 68.7621, + "step": 11090 + }, + { + "epoch": 1.792394178559173, + "grad_norm": 0.9650185108184814, + "learning_rate": 0.0001, + "loss": 66.7721, + "step": 11100 + }, + { + "epoch": 1.7940090026442745, + "grad_norm": 0.6026908159255981, + "learning_rate": 0.0001, + "loss": 66.2993, + "step": 11110 + }, + { + "epoch": 1.7956238267293756, + "grad_norm": 0.6560576558113098, + "learning_rate": 0.0001, + "loss": 67.8543, + "step": 11120 + }, + { + "epoch": 1.7972386508144769, + "grad_norm": 0.9523794054985046, + "learning_rate": 0.0001, + "loss": 66.7881, + "step": 11130 + }, + { + "epoch": 1.7988534748995781, + "grad_norm": 0.7241806983947754, + "learning_rate": 0.0001, + "loss": 68.1071, + "step": 11140 + }, + { + "epoch": 1.8004682989846792, + "grad_norm": 0.639866292476654, + "learning_rate": 0.0001, + "loss": 68.992, + "step": 11150 + }, + { + "epoch": 1.8020831230697807, + "grad_norm": 0.6050712466239929, + "learning_rate": 0.0001, + "loss": 67.7993, + "step": 11160 + }, + { + "epoch": 1.8036979471548817, + "grad_norm": 0.5777723789215088, + "learning_rate": 0.0001, + "loss": 66.9515, + "step": 11170 + }, + { + "epoch": 1.805312771239983, + "grad_norm": 0.6153426766395569, + "learning_rate": 0.0001, + "loss": 68.3413, + "step": 11180 + }, + { + "epoch": 1.8069275953250843, + "grad_norm": 0.6118552088737488, + "learning_rate": 0.0001, + "loss": 67.7853, + "step": 11190 + }, + { + "epoch": 1.8085424194101853, + "grad_norm": 0.6520341038703918, + "learning_rate": 0.0001, + "loss": 67.3511, + "step": 11200 + }, + { + "epoch": 1.8101572434952868, + "grad_norm": 0.6189568638801575, + "learning_rate": 0.0001, + "loss": 69.28, + "step": 11210 + }, + { + "epoch": 1.811772067580388, + "grad_norm": 0.602411150932312, + "learning_rate": 0.0001, + "loss": 67.5133, + "step": 11220 + }, + { + "epoch": 1.8133868916654892, + "grad_norm": 0.7359414100646973, + "learning_rate": 0.0001, + "loss": 68.631, + "step": 11230 + }, + { + "epoch": 1.8150017157505904, + "grad_norm": 0.6216995120048523, + "learning_rate": 0.0001, + "loss": 68.3588, + "step": 11240 + }, + { + "epoch": 1.8166165398356915, + "grad_norm": 0.7683486938476562, + "learning_rate": 0.0001, + "loss": 67.1434, + "step": 11250 + }, + { + "epoch": 1.818231363920793, + "grad_norm": 0.5565283894538879, + "learning_rate": 0.0001, + "loss": 66.9995, + "step": 11260 + }, + { + "epoch": 1.819846188005894, + "grad_norm": 0.5623223781585693, + "learning_rate": 0.0001, + "loss": 68.1316, + "step": 11270 + }, + { + "epoch": 1.8214610120909953, + "grad_norm": 0.532821774482727, + "learning_rate": 0.0001, + "loss": 69.1903, + "step": 11280 + }, + { + "epoch": 1.8230758361760966, + "grad_norm": 0.6297196745872498, + "learning_rate": 0.0001, + "loss": 68.4965, + "step": 11290 + }, + { + "epoch": 1.8246906602611976, + "grad_norm": 0.839862048625946, + "learning_rate": 0.0001, + "loss": 67.1782, + "step": 11300 + }, + { + "epoch": 1.8263054843462991, + "grad_norm": 0.7021651864051819, + "learning_rate": 0.0001, + "loss": 67.4355, + "step": 11310 + }, + { + "epoch": 1.8279203084314002, + "grad_norm": 0.676034152507782, + "learning_rate": 0.0001, + "loss": 68.3721, + "step": 11320 + }, + { + "epoch": 1.8295351325165015, + "grad_norm": 0.6621078252792358, + "learning_rate": 0.0001, + "loss": 67.3255, + "step": 11330 + }, + { + "epoch": 1.8311499566016027, + "grad_norm": 0.5902897119522095, + "learning_rate": 0.0001, + "loss": 68.0641, + "step": 11340 + }, + { + "epoch": 1.8327647806867038, + "grad_norm": 0.6334371566772461, + "learning_rate": 0.0001, + "loss": 68.2682, + "step": 11350 + }, + { + "epoch": 1.8343796047718053, + "grad_norm": 0.6753348708152771, + "learning_rate": 0.0001, + "loss": 66.6749, + "step": 11360 + }, + { + "epoch": 1.8359944288569063, + "grad_norm": 0.7221750617027283, + "learning_rate": 0.0001, + "loss": 68.4142, + "step": 11370 + }, + { + "epoch": 1.8376092529420076, + "grad_norm": 0.7476344108581543, + "learning_rate": 0.0001, + "loss": 67.1416, + "step": 11380 + }, + { + "epoch": 1.8392240770271089, + "grad_norm": 0.5927479863166809, + "learning_rate": 0.0001, + "loss": 67.6368, + "step": 11390 + }, + { + "epoch": 1.84083890111221, + "grad_norm": 0.6753717660903931, + "learning_rate": 0.0001, + "loss": 67.1026, + "step": 11400 + }, + { + "epoch": 1.8424537251973114, + "grad_norm": 0.6500268578529358, + "learning_rate": 0.0001, + "loss": 68.7537, + "step": 11410 + }, + { + "epoch": 1.8440685492824125, + "grad_norm": 0.5930709838867188, + "learning_rate": 0.0001, + "loss": 65.7429, + "step": 11420 + }, + { + "epoch": 1.8456833733675138, + "grad_norm": 0.6467777490615845, + "learning_rate": 0.0001, + "loss": 68.1333, + "step": 11430 + }, + { + "epoch": 1.847298197452615, + "grad_norm": 0.6935669183731079, + "learning_rate": 0.0001, + "loss": 66.7728, + "step": 11440 + }, + { + "epoch": 1.848913021537716, + "grad_norm": 0.5838116407394409, + "learning_rate": 0.0001, + "loss": 66.6005, + "step": 11450 + }, + { + "epoch": 1.8505278456228176, + "grad_norm": 0.8621205687522888, + "learning_rate": 0.0001, + "loss": 66.0859, + "step": 11460 + }, + { + "epoch": 1.8521426697079186, + "grad_norm": 0.7119213342666626, + "learning_rate": 0.0001, + "loss": 66.6198, + "step": 11470 + }, + { + "epoch": 1.85375749379302, + "grad_norm": 0.6861911416053772, + "learning_rate": 0.0001, + "loss": 67.2275, + "step": 11480 + }, + { + "epoch": 1.8553723178781212, + "grad_norm": 0.5653937458992004, + "learning_rate": 0.0001, + "loss": 67.827, + "step": 11490 + }, + { + "epoch": 1.8569871419632222, + "grad_norm": 0.6458482146263123, + "learning_rate": 0.0001, + "loss": 66.3906, + "step": 11500 + }, + { + "epoch": 1.8586019660483237, + "grad_norm": 0.6701959371566772, + "learning_rate": 0.0001, + "loss": 67.88, + "step": 11510 + }, + { + "epoch": 1.8602167901334248, + "grad_norm": 0.6598711609840393, + "learning_rate": 0.0001, + "loss": 66.8373, + "step": 11520 + }, + { + "epoch": 1.861831614218526, + "grad_norm": 0.6095236539840698, + "learning_rate": 0.0001, + "loss": 67.6478, + "step": 11530 + }, + { + "epoch": 1.8634464383036273, + "grad_norm": 0.8192022442817688, + "learning_rate": 0.0001, + "loss": 67.7343, + "step": 11540 + }, + { + "epoch": 1.8650612623887284, + "grad_norm": 0.5788753628730774, + "learning_rate": 0.0001, + "loss": 66.7909, + "step": 11550 + }, + { + "epoch": 1.8666760864738299, + "grad_norm": 0.5383989214897156, + "learning_rate": 0.0001, + "loss": 68.5349, + "step": 11560 + }, + { + "epoch": 1.868290910558931, + "grad_norm": 0.6224480271339417, + "learning_rate": 0.0001, + "loss": 68.1938, + "step": 11570 + }, + { + "epoch": 1.8699057346440322, + "grad_norm": 0.5852788686752319, + "learning_rate": 0.0001, + "loss": 66.4678, + "step": 11580 + }, + { + "epoch": 1.8715205587291335, + "grad_norm": 0.730378270149231, + "learning_rate": 0.0001, + "loss": 67.57, + "step": 11590 + }, + { + "epoch": 1.8731353828142345, + "grad_norm": 0.6197090744972229, + "learning_rate": 0.0001, + "loss": 65.7801, + "step": 11600 + }, + { + "epoch": 1.874750206899336, + "grad_norm": 0.518974781036377, + "learning_rate": 0.0001, + "loss": 68.9308, + "step": 11610 + }, + { + "epoch": 1.876365030984437, + "grad_norm": 0.6601742506027222, + "learning_rate": 0.0001, + "loss": 66.821, + "step": 11620 + }, + { + "epoch": 1.8779798550695384, + "grad_norm": 0.7409974932670593, + "learning_rate": 0.0001, + "loss": 66.6883, + "step": 11630 + }, + { + "epoch": 1.8795946791546396, + "grad_norm": 0.6372189521789551, + "learning_rate": 0.0001, + "loss": 67.2109, + "step": 11640 + }, + { + "epoch": 1.8812095032397407, + "grad_norm": 0.6107574701309204, + "learning_rate": 0.0001, + "loss": 67.029, + "step": 11650 + }, + { + "epoch": 1.8828243273248422, + "grad_norm": 0.6543708443641663, + "learning_rate": 0.0001, + "loss": 64.2133, + "step": 11660 + }, + { + "epoch": 1.8844391514099432, + "grad_norm": 0.5161535739898682, + "learning_rate": 0.0001, + "loss": 66.7091, + "step": 11670 + }, + { + "epoch": 1.8860539754950445, + "grad_norm": 0.6089748740196228, + "learning_rate": 0.0001, + "loss": 69.2073, + "step": 11680 + }, + { + "epoch": 1.8876687995801458, + "grad_norm": 0.6930866837501526, + "learning_rate": 0.0001, + "loss": 68.7227, + "step": 11690 + }, + { + "epoch": 1.8892836236652468, + "grad_norm": 0.6563196182250977, + "learning_rate": 0.0001, + "loss": 68.27, + "step": 11700 + }, + { + "epoch": 1.8908984477503483, + "grad_norm": 0.6173767447471619, + "learning_rate": 0.0001, + "loss": 68.5535, + "step": 11710 + }, + { + "epoch": 1.8925132718354494, + "grad_norm": 0.5498561263084412, + "learning_rate": 0.0001, + "loss": 67.7681, + "step": 11720 + }, + { + "epoch": 1.8941280959205506, + "grad_norm": 0.584891676902771, + "learning_rate": 0.0001, + "loss": 68.1709, + "step": 11730 + }, + { + "epoch": 1.895742920005652, + "grad_norm": 0.7491183280944824, + "learning_rate": 0.0001, + "loss": 65.7435, + "step": 11740 + }, + { + "epoch": 1.897357744090753, + "grad_norm": 0.6519315242767334, + "learning_rate": 0.0001, + "loss": 67.5436, + "step": 11750 + }, + { + "epoch": 1.8989725681758545, + "grad_norm": 0.5740950107574463, + "learning_rate": 0.0001, + "loss": 66.3092, + "step": 11760 + }, + { + "epoch": 1.9005873922609555, + "grad_norm": 0.7978914380073547, + "learning_rate": 0.0001, + "loss": 66.8598, + "step": 11770 + }, + { + "epoch": 1.9022022163460568, + "grad_norm": 0.5536732077598572, + "learning_rate": 0.0001, + "loss": 68.1609, + "step": 11780 + }, + { + "epoch": 1.903817040431158, + "grad_norm": 0.6358754634857178, + "learning_rate": 0.0001, + "loss": 68.2948, + "step": 11790 + }, + { + "epoch": 1.9054318645162591, + "grad_norm": 0.8400535583496094, + "learning_rate": 0.0001, + "loss": 67.7722, + "step": 11800 + }, + { + "epoch": 1.9070466886013606, + "grad_norm": 0.8256928324699402, + "learning_rate": 0.0001, + "loss": 65.8444, + "step": 11810 + }, + { + "epoch": 1.9086615126864617, + "grad_norm": 0.9160122871398926, + "learning_rate": 0.0001, + "loss": 67.4363, + "step": 11820 + }, + { + "epoch": 1.910276336771563, + "grad_norm": 0.6507587432861328, + "learning_rate": 0.0001, + "loss": 65.9515, + "step": 11830 + }, + { + "epoch": 1.9118911608566642, + "grad_norm": 0.5940394401550293, + "learning_rate": 0.0001, + "loss": 65.8083, + "step": 11840 + }, + { + "epoch": 1.9135059849417653, + "grad_norm": 0.7403995990753174, + "learning_rate": 0.0001, + "loss": 66.7136, + "step": 11850 + }, + { + "epoch": 1.9151208090268668, + "grad_norm": 0.6039106845855713, + "learning_rate": 0.0001, + "loss": 67.6913, + "step": 11860 + }, + { + "epoch": 1.9167356331119678, + "grad_norm": 0.9714781641960144, + "learning_rate": 0.0001, + "loss": 67.9658, + "step": 11870 + }, + { + "epoch": 1.918350457197069, + "grad_norm": 0.6880549192428589, + "learning_rate": 0.0001, + "loss": 68.4463, + "step": 11880 + }, + { + "epoch": 1.9199652812821704, + "grad_norm": 0.5718733668327332, + "learning_rate": 0.0001, + "loss": 66.9625, + "step": 11890 + }, + { + "epoch": 1.9215801053672714, + "grad_norm": 0.6208164691925049, + "learning_rate": 0.0001, + "loss": 65.9462, + "step": 11900 + }, + { + "epoch": 1.923194929452373, + "grad_norm": 0.5957098007202148, + "learning_rate": 0.0001, + "loss": 66.9398, + "step": 11910 + }, + { + "epoch": 1.924809753537474, + "grad_norm": 0.6295427680015564, + "learning_rate": 0.0001, + "loss": 66.2246, + "step": 11920 + }, + { + "epoch": 1.9264245776225752, + "grad_norm": 0.8903796672821045, + "learning_rate": 0.0001, + "loss": 66.2999, + "step": 11930 + }, + { + "epoch": 1.9280394017076765, + "grad_norm": 0.6410874128341675, + "learning_rate": 0.0001, + "loss": 67.6857, + "step": 11940 + }, + { + "epoch": 1.9296542257927776, + "grad_norm": 0.5800307393074036, + "learning_rate": 0.0001, + "loss": 66.0943, + "step": 11950 + }, + { + "epoch": 1.931269049877879, + "grad_norm": 0.6834051609039307, + "learning_rate": 0.0001, + "loss": 67.6365, + "step": 11960 + }, + { + "epoch": 1.9328838739629801, + "grad_norm": 0.6112056970596313, + "learning_rate": 0.0001, + "loss": 66.8131, + "step": 11970 + }, + { + "epoch": 1.9344986980480814, + "grad_norm": 0.6657601594924927, + "learning_rate": 0.0001, + "loss": 65.7584, + "step": 11980 + }, + { + "epoch": 1.9361135221331827, + "grad_norm": 0.7570573091506958, + "learning_rate": 0.0001, + "loss": 68.6683, + "step": 11990 + }, + { + "epoch": 1.9377283462182837, + "grad_norm": 0.6231463551521301, + "learning_rate": 0.0001, + "loss": 67.71, + "step": 12000 + }, + { + "epoch": 1.9393431703033852, + "grad_norm": 0.5517798662185669, + "learning_rate": 0.0001, + "loss": 65.5666, + "step": 12010 + }, + { + "epoch": 1.9409579943884863, + "grad_norm": 0.6614659428596497, + "learning_rate": 0.0001, + "loss": 65.4867, + "step": 12020 + }, + { + "epoch": 1.9425728184735875, + "grad_norm": 1.027906060218811, + "learning_rate": 0.0001, + "loss": 65.2863, + "step": 12030 + }, + { + "epoch": 1.9441876425586888, + "grad_norm": 0.5714905261993408, + "learning_rate": 0.0001, + "loss": 67.52, + "step": 12040 + }, + { + "epoch": 1.9458024666437899, + "grad_norm": 0.6580681204795837, + "learning_rate": 0.0001, + "loss": 68.3794, + "step": 12050 + }, + { + "epoch": 1.9474172907288914, + "grad_norm": 0.5728104710578918, + "learning_rate": 0.0001, + "loss": 65.6871, + "step": 12060 + }, + { + "epoch": 1.9490321148139924, + "grad_norm": 0.6828856468200684, + "learning_rate": 0.0001, + "loss": 66.6735, + "step": 12070 + }, + { + "epoch": 1.9506469388990937, + "grad_norm": 0.7448258996009827, + "learning_rate": 0.0001, + "loss": 67.4387, + "step": 12080 + }, + { + "epoch": 1.952261762984195, + "grad_norm": 0.5058867931365967, + "learning_rate": 0.0001, + "loss": 67.3999, + "step": 12090 + }, + { + "epoch": 1.953876587069296, + "grad_norm": 0.5884380340576172, + "learning_rate": 0.0001, + "loss": 66.6189, + "step": 12100 + }, + { + "epoch": 1.9554914111543975, + "grad_norm": 0.7010582685470581, + "learning_rate": 0.0001, + "loss": 66.4048, + "step": 12110 + }, + { + "epoch": 1.9571062352394986, + "grad_norm": 0.556446373462677, + "learning_rate": 0.0001, + "loss": 67.7398, + "step": 12120 + }, + { + "epoch": 1.9587210593245998, + "grad_norm": 0.7856518626213074, + "learning_rate": 0.0001, + "loss": 64.4189, + "step": 12130 + }, + { + "epoch": 1.960335883409701, + "grad_norm": 0.573811948299408, + "learning_rate": 0.0001, + "loss": 66.081, + "step": 12140 + }, + { + "epoch": 1.9619507074948022, + "grad_norm": 0.6113094687461853, + "learning_rate": 0.0001, + "loss": 65.6262, + "step": 12150 + }, + { + "epoch": 1.9635655315799037, + "grad_norm": 0.7158921360969543, + "learning_rate": 0.0001, + "loss": 66.646, + "step": 12160 + }, + { + "epoch": 1.9651803556650047, + "grad_norm": 0.6789208054542542, + "learning_rate": 0.0001, + "loss": 66.9576, + "step": 12170 + }, + { + "epoch": 1.966795179750106, + "grad_norm": 0.6550324559211731, + "learning_rate": 0.0001, + "loss": 67.2544, + "step": 12180 + }, + { + "epoch": 1.9684100038352073, + "grad_norm": 0.6580352187156677, + "learning_rate": 0.0001, + "loss": 67.6755, + "step": 12190 + }, + { + "epoch": 1.9700248279203083, + "grad_norm": 0.7927120327949524, + "learning_rate": 0.0001, + "loss": 66.9819, + "step": 12200 + }, + { + "epoch": 1.9716396520054098, + "grad_norm": 0.5637071132659912, + "learning_rate": 0.0001, + "loss": 67.9308, + "step": 12210 + }, + { + "epoch": 1.9732544760905109, + "grad_norm": 0.5903081297874451, + "learning_rate": 0.0001, + "loss": 67.0279, + "step": 12220 + }, + { + "epoch": 1.9748693001756121, + "grad_norm": 0.7173757553100586, + "learning_rate": 0.0001, + "loss": 66.1328, + "step": 12230 + }, + { + "epoch": 1.9764841242607134, + "grad_norm": 0.7677658796310425, + "learning_rate": 0.0001, + "loss": 66.8729, + "step": 12240 + }, + { + "epoch": 1.9780989483458145, + "grad_norm": 0.5733856558799744, + "learning_rate": 0.0001, + "loss": 65.7204, + "step": 12250 + }, + { + "epoch": 1.979713772430916, + "grad_norm": 0.575779378414154, + "learning_rate": 0.0001, + "loss": 65.082, + "step": 12260 + }, + { + "epoch": 1.981328596516017, + "grad_norm": 0.7983589768409729, + "learning_rate": 0.0001, + "loss": 66.7954, + "step": 12270 + }, + { + "epoch": 1.9829434206011183, + "grad_norm": 0.6657546758651733, + "learning_rate": 0.0001, + "loss": 66.1977, + "step": 12280 + }, + { + "epoch": 1.9845582446862196, + "grad_norm": 0.723057746887207, + "learning_rate": 0.0001, + "loss": 65.2823, + "step": 12290 + }, + { + "epoch": 1.9861730687713206, + "grad_norm": 0.7241561412811279, + "learning_rate": 0.0001, + "loss": 66.5179, + "step": 12300 + }, + { + "epoch": 1.987787892856422, + "grad_norm": 0.7907382249832153, + "learning_rate": 0.0001, + "loss": 67.1835, + "step": 12310 + }, + { + "epoch": 1.9894027169415232, + "grad_norm": 1.0993635654449463, + "learning_rate": 0.0001, + "loss": 66.2844, + "step": 12320 + }, + { + "epoch": 1.9910175410266244, + "grad_norm": 0.6327719688415527, + "learning_rate": 0.0001, + "loss": 64.3111, + "step": 12330 + }, + { + "epoch": 1.9926323651117257, + "grad_norm": 0.5955413579940796, + "learning_rate": 0.0001, + "loss": 67.3991, + "step": 12340 + }, + { + "epoch": 1.9942471891968268, + "grad_norm": 0.7130835056304932, + "learning_rate": 0.0001, + "loss": 67.2216, + "step": 12350 + }, + { + "epoch": 1.9958620132819282, + "grad_norm": 0.5579924583435059, + "learning_rate": 0.0001, + "loss": 65.8894, + "step": 12360 + }, + { + "epoch": 1.9974768373670293, + "grad_norm": 0.7395238876342773, + "learning_rate": 0.0001, + "loss": 66.5182, + "step": 12370 + }, + { + "epoch": 1.9990916614521306, + "grad_norm": 0.6070617437362671, + "learning_rate": 0.0001, + "loss": 67.0429, + "step": 12380 + }, + { + "epoch": 2.0006459296340404, + "grad_norm": 0.6658774018287659, + "learning_rate": 0.0001, + "loss": 65.1403, + "step": 12390 + }, + { + "epoch": 2.002260753719142, + "grad_norm": 0.7703005075454712, + "learning_rate": 0.0001, + "loss": 66.0792, + "step": 12400 + }, + { + "epoch": 2.003875577804243, + "grad_norm": 0.6887365579605103, + "learning_rate": 0.0001, + "loss": 67.5743, + "step": 12410 + }, + { + "epoch": 2.005490401889344, + "grad_norm": 0.6181573271751404, + "learning_rate": 0.0001, + "loss": 65.7725, + "step": 12420 + }, + { + "epoch": 2.0071052259744455, + "grad_norm": 0.6926079988479614, + "learning_rate": 0.0001, + "loss": 66.3818, + "step": 12430 + }, + { + "epoch": 2.0087200500595466, + "grad_norm": 0.9466767311096191, + "learning_rate": 0.0001, + "loss": 66.8674, + "step": 12440 + }, + { + "epoch": 2.010334874144648, + "grad_norm": 0.693327784538269, + "learning_rate": 0.0001, + "loss": 67.1573, + "step": 12450 + }, + { + "epoch": 2.011949698229749, + "grad_norm": 0.6354495286941528, + "learning_rate": 0.0001, + "loss": 67.6646, + "step": 12460 + }, + { + "epoch": 2.01356452231485, + "grad_norm": 0.6671343445777893, + "learning_rate": 0.0001, + "loss": 66.9442, + "step": 12470 + }, + { + "epoch": 2.0151793463999517, + "grad_norm": 0.5987095236778259, + "learning_rate": 0.0001, + "loss": 68.265, + "step": 12480 + }, + { + "epoch": 2.0167941704850527, + "grad_norm": 0.8267726302146912, + "learning_rate": 0.0001, + "loss": 67.7709, + "step": 12490 + }, + { + "epoch": 2.018408994570154, + "grad_norm": 0.5237421989440918, + "learning_rate": 0.0001, + "loss": 64.8426, + "step": 12500 + }, + { + "epoch": 2.0200238186552553, + "grad_norm": 0.5602227449417114, + "learning_rate": 0.0001, + "loss": 65.4287, + "step": 12510 + }, + { + "epoch": 2.0216386427403563, + "grad_norm": 0.6129289269447327, + "learning_rate": 0.0001, + "loss": 65.9583, + "step": 12520 + }, + { + "epoch": 2.023253466825458, + "grad_norm": 0.6047223210334778, + "learning_rate": 0.0001, + "loss": 66.9257, + "step": 12530 + }, + { + "epoch": 2.024868290910559, + "grad_norm": 0.6117433905601501, + "learning_rate": 0.0001, + "loss": 63.0344, + "step": 12540 + }, + { + "epoch": 2.0264831149956604, + "grad_norm": 0.6688846945762634, + "learning_rate": 0.0001, + "loss": 66.2295, + "step": 12550 + }, + { + "epoch": 2.0280979390807614, + "grad_norm": 0.779574453830719, + "learning_rate": 0.0001, + "loss": 67.066, + "step": 12560 + }, + { + "epoch": 2.0297127631658625, + "grad_norm": 0.624839186668396, + "learning_rate": 0.0001, + "loss": 66.7905, + "step": 12570 + }, + { + "epoch": 2.031327587250964, + "grad_norm": 0.5970683693885803, + "learning_rate": 0.0001, + "loss": 67.6997, + "step": 12580 + }, + { + "epoch": 2.032942411336065, + "grad_norm": 0.6005929708480835, + "learning_rate": 0.0001, + "loss": 66.717, + "step": 12590 + }, + { + "epoch": 2.0345572354211665, + "grad_norm": 0.6358321905136108, + "learning_rate": 0.0001, + "loss": 67.8494, + "step": 12600 + }, + { + "epoch": 2.0361720595062676, + "grad_norm": 0.8642273545265198, + "learning_rate": 0.0001, + "loss": 67.1791, + "step": 12610 + }, + { + "epoch": 2.0377868835913686, + "grad_norm": 0.587282121181488, + "learning_rate": 0.0001, + "loss": 66.8169, + "step": 12620 + }, + { + "epoch": 2.03940170767647, + "grad_norm": 0.5977926850318909, + "learning_rate": 0.0001, + "loss": 66.8393, + "step": 12630 + }, + { + "epoch": 2.041016531761571, + "grad_norm": 0.5654817223548889, + "learning_rate": 0.0001, + "loss": 68.8243, + "step": 12640 + }, + { + "epoch": 2.0426313558466727, + "grad_norm": 0.5778902173042297, + "learning_rate": 0.0001, + "loss": 66.8893, + "step": 12650 + }, + { + "epoch": 2.0442461799317737, + "grad_norm": 0.6533665657043457, + "learning_rate": 0.0001, + "loss": 66.6174, + "step": 12660 + }, + { + "epoch": 2.0458610040168748, + "grad_norm": 0.5945654511451721, + "learning_rate": 0.0001, + "loss": 67.8193, + "step": 12670 + }, + { + "epoch": 2.0474758281019763, + "grad_norm": 0.5898523926734924, + "learning_rate": 0.0001, + "loss": 65.6194, + "step": 12680 + }, + { + "epoch": 2.0490906521870773, + "grad_norm": 0.5497958660125732, + "learning_rate": 0.0001, + "loss": 66.4571, + "step": 12690 + }, + { + "epoch": 2.050705476272179, + "grad_norm": 0.6195752024650574, + "learning_rate": 0.0001, + "loss": 66.0555, + "step": 12700 + }, + { + "epoch": 2.05232030035728, + "grad_norm": 0.6400334239006042, + "learning_rate": 0.0001, + "loss": 66.7153, + "step": 12710 + }, + { + "epoch": 2.053935124442381, + "grad_norm": 0.7953216433525085, + "learning_rate": 0.0001, + "loss": 66.5822, + "step": 12720 + }, + { + "epoch": 2.0555499485274824, + "grad_norm": 0.8052420616149902, + "learning_rate": 0.0001, + "loss": 66.4395, + "step": 12730 + }, + { + "epoch": 2.0571647726125835, + "grad_norm": 0.567957878112793, + "learning_rate": 0.0001, + "loss": 65.7762, + "step": 12740 + }, + { + "epoch": 2.058779596697685, + "grad_norm": 0.6517317891120911, + "learning_rate": 0.0001, + "loss": 66.7526, + "step": 12750 + }, + { + "epoch": 2.060394420782786, + "grad_norm": 0.6032442450523376, + "learning_rate": 0.0001, + "loss": 64.2725, + "step": 12760 + }, + { + "epoch": 2.062009244867887, + "grad_norm": 0.6162487268447876, + "learning_rate": 0.0001, + "loss": 65.4797, + "step": 12770 + }, + { + "epoch": 2.0636240689529886, + "grad_norm": 0.7512190937995911, + "learning_rate": 0.0001, + "loss": 65.9602, + "step": 12780 + }, + { + "epoch": 2.0652388930380896, + "grad_norm": 0.7601141929626465, + "learning_rate": 0.0001, + "loss": 66.0806, + "step": 12790 + }, + { + "epoch": 2.066853717123191, + "grad_norm": 0.5737994909286499, + "learning_rate": 0.0001, + "loss": 66.8124, + "step": 12800 + }, + { + "epoch": 2.068468541208292, + "grad_norm": 0.6629387736320496, + "learning_rate": 0.0001, + "loss": 65.8944, + "step": 12810 + }, + { + "epoch": 2.070083365293393, + "grad_norm": 0.6374203562736511, + "learning_rate": 0.0001, + "loss": 67.1589, + "step": 12820 + }, + { + "epoch": 2.0716981893784947, + "grad_norm": 0.6732942461967468, + "learning_rate": 0.0001, + "loss": 67.1901, + "step": 12830 + }, + { + "epoch": 2.0733130134635958, + "grad_norm": 0.6219559907913208, + "learning_rate": 0.0001, + "loss": 66.613, + "step": 12840 + }, + { + "epoch": 2.0749278375486973, + "grad_norm": 0.591238796710968, + "learning_rate": 0.0001, + "loss": 67.6111, + "step": 12850 + }, + { + "epoch": 2.0765426616337983, + "grad_norm": 0.6472163200378418, + "learning_rate": 0.0001, + "loss": 66.305, + "step": 12860 + }, + { + "epoch": 2.0781574857188994, + "grad_norm": 0.6349777579307556, + "learning_rate": 0.0001, + "loss": 65.2865, + "step": 12870 + }, + { + "epoch": 2.079772309804001, + "grad_norm": 0.9257296323776245, + "learning_rate": 0.0001, + "loss": 67.6326, + "step": 12880 + }, + { + "epoch": 2.081387133889102, + "grad_norm": 0.5744728446006775, + "learning_rate": 0.0001, + "loss": 65.7023, + "step": 12890 + }, + { + "epoch": 2.0830019579742034, + "grad_norm": 0.6725465059280396, + "learning_rate": 0.0001, + "loss": 65.1233, + "step": 12900 + }, + { + "epoch": 2.0846167820593045, + "grad_norm": 0.6141199469566345, + "learning_rate": 0.0001, + "loss": 65.5593, + "step": 12910 + }, + { + "epoch": 2.0862316061444055, + "grad_norm": 0.6364370584487915, + "learning_rate": 0.0001, + "loss": 65.3899, + "step": 12920 + }, + { + "epoch": 2.087846430229507, + "grad_norm": 0.5948928594589233, + "learning_rate": 0.0001, + "loss": 63.5976, + "step": 12930 + }, + { + "epoch": 2.089461254314608, + "grad_norm": 0.630434513092041, + "learning_rate": 0.0001, + "loss": 66.764, + "step": 12940 + }, + { + "epoch": 2.091076078399709, + "grad_norm": 0.6178410649299622, + "learning_rate": 0.0001, + "loss": 66.3948, + "step": 12950 + }, + { + "epoch": 2.0926909024848106, + "grad_norm": 0.7695903778076172, + "learning_rate": 0.0001, + "loss": 68.2218, + "step": 12960 + }, + { + "epoch": 2.0943057265699117, + "grad_norm": 0.6712848544120789, + "learning_rate": 0.0001, + "loss": 66.8801, + "step": 12970 + }, + { + "epoch": 2.095920550655013, + "grad_norm": 0.7133443355560303, + "learning_rate": 0.0001, + "loss": 65.343, + "step": 12980 + }, + { + "epoch": 2.097535374740114, + "grad_norm": 0.6836514472961426, + "learning_rate": 0.0001, + "loss": 64.4993, + "step": 12990 + }, + { + "epoch": 2.0991501988252157, + "grad_norm": 0.6734977960586548, + "learning_rate": 0.0001, + "loss": 67.6078, + "step": 13000 + }, + { + "epoch": 2.1007650229103167, + "grad_norm": 0.590630054473877, + "learning_rate": 0.0001, + "loss": 65.4567, + "step": 13010 + }, + { + "epoch": 2.102379846995418, + "grad_norm": 0.7174671292304993, + "learning_rate": 0.0001, + "loss": 66.2743, + "step": 13020 + }, + { + "epoch": 2.1039946710805193, + "grad_norm": 0.5266520977020264, + "learning_rate": 0.0001, + "loss": 65.0063, + "step": 13030 + }, + { + "epoch": 2.1056094951656203, + "grad_norm": 0.550865113735199, + "learning_rate": 0.0001, + "loss": 67.0641, + "step": 13040 + }, + { + "epoch": 2.1072243192507214, + "grad_norm": 0.7976025938987732, + "learning_rate": 0.0001, + "loss": 67.0135, + "step": 13050 + }, + { + "epoch": 2.108839143335823, + "grad_norm": 0.7592310905456543, + "learning_rate": 0.0001, + "loss": 65.9875, + "step": 13060 + }, + { + "epoch": 2.110453967420924, + "grad_norm": 0.7613953948020935, + "learning_rate": 0.0001, + "loss": 66.0958, + "step": 13070 + }, + { + "epoch": 2.1120687915060254, + "grad_norm": 0.6951309442520142, + "learning_rate": 0.0001, + "loss": 65.949, + "step": 13080 + }, + { + "epoch": 2.1136836155911265, + "grad_norm": 0.6439896821975708, + "learning_rate": 0.0001, + "loss": 66.4056, + "step": 13090 + }, + { + "epoch": 2.115298439676228, + "grad_norm": 0.6922518610954285, + "learning_rate": 0.0001, + "loss": 67.537, + "step": 13100 + }, + { + "epoch": 2.116913263761329, + "grad_norm": 0.5330061316490173, + "learning_rate": 0.0001, + "loss": 67.0438, + "step": 13110 + }, + { + "epoch": 2.11852808784643, + "grad_norm": 0.6507022380828857, + "learning_rate": 0.0001, + "loss": 66.9908, + "step": 13120 + }, + { + "epoch": 2.1201429119315316, + "grad_norm": 0.994196891784668, + "learning_rate": 0.0001, + "loss": 63.9818, + "step": 13130 + }, + { + "epoch": 2.1217577360166326, + "grad_norm": 0.7431663870811462, + "learning_rate": 0.0001, + "loss": 65.8259, + "step": 13140 + }, + { + "epoch": 2.1233725601017337, + "grad_norm": 0.5852991938591003, + "learning_rate": 0.0001, + "loss": 65.918, + "step": 13150 + }, + { + "epoch": 2.124987384186835, + "grad_norm": 0.551194429397583, + "learning_rate": 0.0001, + "loss": 64.9907, + "step": 13160 + }, + { + "epoch": 2.1266022082719362, + "grad_norm": 0.6882205605506897, + "learning_rate": 0.0001, + "loss": 66.7266, + "step": 13170 + }, + { + "epoch": 2.1282170323570377, + "grad_norm": 0.8906047940254211, + "learning_rate": 0.0001, + "loss": 65.3441, + "step": 13180 + }, + { + "epoch": 2.129831856442139, + "grad_norm": 0.5340781211853027, + "learning_rate": 0.0001, + "loss": 65.6772, + "step": 13190 + }, + { + "epoch": 2.1314466805272403, + "grad_norm": 0.7657933831214905, + "learning_rate": 0.0001, + "loss": 65.3888, + "step": 13200 + }, + { + "epoch": 2.1330615046123413, + "grad_norm": 1.2086169719696045, + "learning_rate": 0.0001, + "loss": 65.7214, + "step": 13210 + }, + { + "epoch": 2.1346763286974424, + "grad_norm": 0.6794751286506653, + "learning_rate": 0.0001, + "loss": 66.6761, + "step": 13220 + }, + { + "epoch": 2.136291152782544, + "grad_norm": 0.5939100384712219, + "learning_rate": 0.0001, + "loss": 64.1858, + "step": 13230 + }, + { + "epoch": 2.137905976867645, + "grad_norm": 0.581163763999939, + "learning_rate": 0.0001, + "loss": 65.9456, + "step": 13240 + }, + { + "epoch": 2.139520800952746, + "grad_norm": 0.676103949546814, + "learning_rate": 0.0001, + "loss": 65.1919, + "step": 13250 + }, + { + "epoch": 2.1411356250378475, + "grad_norm": 0.6052015423774719, + "learning_rate": 0.0001, + "loss": 67.9872, + "step": 13260 + }, + { + "epoch": 2.1427504491229485, + "grad_norm": 0.5938939452171326, + "learning_rate": 0.0001, + "loss": 64.7395, + "step": 13270 + }, + { + "epoch": 2.14436527320805, + "grad_norm": 0.563115656375885, + "learning_rate": 0.0001, + "loss": 63.9757, + "step": 13280 + }, + { + "epoch": 2.145980097293151, + "grad_norm": 0.5397975444793701, + "learning_rate": 0.0001, + "loss": 65.6596, + "step": 13290 + }, + { + "epoch": 2.1475949213782526, + "grad_norm": 0.6397805213928223, + "learning_rate": 0.0001, + "loss": 67.0618, + "step": 13300 + }, + { + "epoch": 2.1492097454633536, + "grad_norm": 0.678053081035614, + "learning_rate": 0.0001, + "loss": 65.8194, + "step": 13310 + }, + { + "epoch": 2.1508245695484547, + "grad_norm": 0.6067008972167969, + "learning_rate": 0.0001, + "loss": 65.9035, + "step": 13320 + }, + { + "epoch": 2.152439393633556, + "grad_norm": 0.6560337543487549, + "learning_rate": 0.0001, + "loss": 64.9907, + "step": 13330 + }, + { + "epoch": 2.1540542177186572, + "grad_norm": 0.5570183396339417, + "learning_rate": 0.0001, + "loss": 64.5903, + "step": 13340 + }, + { + "epoch": 2.1556690418037583, + "grad_norm": 0.6721981167793274, + "learning_rate": 0.0001, + "loss": 66.5618, + "step": 13350 + }, + { + "epoch": 2.15728386588886, + "grad_norm": 0.7251304388046265, + "learning_rate": 0.0001, + "loss": 66.724, + "step": 13360 + }, + { + "epoch": 2.158898689973961, + "grad_norm": 0.5928503274917603, + "learning_rate": 0.0001, + "loss": 66.6991, + "step": 13370 + }, + { + "epoch": 2.1605135140590623, + "grad_norm": 0.7684288620948792, + "learning_rate": 0.0001, + "loss": 66.4064, + "step": 13380 + }, + { + "epoch": 2.1621283381441634, + "grad_norm": 0.5499702095985413, + "learning_rate": 0.0001, + "loss": 66.7671, + "step": 13390 + }, + { + "epoch": 2.163743162229265, + "grad_norm": 0.5267577767372131, + "learning_rate": 0.0001, + "loss": 65.8112, + "step": 13400 + }, + { + "epoch": 2.165357986314366, + "grad_norm": 0.6214171648025513, + "learning_rate": 0.0001, + "loss": 65.851, + "step": 13410 + }, + { + "epoch": 2.166972810399467, + "grad_norm": 0.5952336192131042, + "learning_rate": 0.0001, + "loss": 65.4166, + "step": 13420 + }, + { + "epoch": 2.1685876344845685, + "grad_norm": 1.3454428911209106, + "learning_rate": 0.0001, + "loss": 64.7546, + "step": 13430 + }, + { + "epoch": 2.1702024585696695, + "grad_norm": 0.6586198210716248, + "learning_rate": 0.0001, + "loss": 65.6818, + "step": 13440 + }, + { + "epoch": 2.1718172826547706, + "grad_norm": 0.8028526306152344, + "learning_rate": 0.0001, + "loss": 67.2677, + "step": 13450 + }, + { + "epoch": 2.173432106739872, + "grad_norm": 0.5949039459228516, + "learning_rate": 0.0001, + "loss": 66.994, + "step": 13460 + }, + { + "epoch": 2.175046930824973, + "grad_norm": 0.6666626930236816, + "learning_rate": 0.0001, + "loss": 66.6509, + "step": 13470 + }, + { + "epoch": 2.1766617549100746, + "grad_norm": 0.5985739827156067, + "learning_rate": 0.0001, + "loss": 65.9211, + "step": 13480 + }, + { + "epoch": 2.1782765789951757, + "grad_norm": 0.6771738529205322, + "learning_rate": 0.0001, + "loss": 65.3477, + "step": 13490 + }, + { + "epoch": 2.1798914030802767, + "grad_norm": 0.6147419214248657, + "learning_rate": 0.0001, + "loss": 64.4142, + "step": 13500 + }, + { + "epoch": 2.1815062271653782, + "grad_norm": 0.6774442791938782, + "learning_rate": 0.0001, + "loss": 64.5831, + "step": 13510 + }, + { + "epoch": 2.1831210512504793, + "grad_norm": 0.5778859853744507, + "learning_rate": 0.0001, + "loss": 64.4449, + "step": 13520 + }, + { + "epoch": 2.184735875335581, + "grad_norm": 0.5697338581085205, + "learning_rate": 0.0001, + "loss": 67.1627, + "step": 13530 + }, + { + "epoch": 2.186350699420682, + "grad_norm": 1.2444179058074951, + "learning_rate": 0.0001, + "loss": 64.7959, + "step": 13540 + }, + { + "epoch": 2.187965523505783, + "grad_norm": 0.6489098072052002, + "learning_rate": 0.0001, + "loss": 65.9926, + "step": 13550 + }, + { + "epoch": 2.1895803475908844, + "grad_norm": 0.6871664524078369, + "learning_rate": 0.0001, + "loss": 67.681, + "step": 13560 + }, + { + "epoch": 2.1911951716759854, + "grad_norm": 0.5581620931625366, + "learning_rate": 0.0001, + "loss": 65.6707, + "step": 13570 + }, + { + "epoch": 2.192809995761087, + "grad_norm": 0.522402286529541, + "learning_rate": 0.0001, + "loss": 65.9014, + "step": 13580 + }, + { + "epoch": 2.194424819846188, + "grad_norm": 0.6149793267250061, + "learning_rate": 0.0001, + "loss": 64.6348, + "step": 13590 + }, + { + "epoch": 2.196039643931289, + "grad_norm": 0.5616180896759033, + "learning_rate": 0.0001, + "loss": 66.3635, + "step": 13600 + }, + { + "epoch": 2.1976544680163905, + "grad_norm": 1.024154543876648, + "learning_rate": 0.0001, + "loss": 66.7156, + "step": 13610 + }, + { + "epoch": 2.1992692921014916, + "grad_norm": 1.035205602645874, + "learning_rate": 0.0001, + "loss": 65.996, + "step": 13620 + }, + { + "epoch": 2.200884116186593, + "grad_norm": 0.5354496240615845, + "learning_rate": 0.0001, + "loss": 66.6107, + "step": 13630 + }, + { + "epoch": 2.202498940271694, + "grad_norm": 0.5648348331451416, + "learning_rate": 0.0001, + "loss": 66.473, + "step": 13640 + }, + { + "epoch": 2.204113764356795, + "grad_norm": 0.6247662901878357, + "learning_rate": 0.0001, + "loss": 66.576, + "step": 13650 + }, + { + "epoch": 2.2057285884418967, + "grad_norm": 0.5871408581733704, + "learning_rate": 0.0001, + "loss": 66.6254, + "step": 13660 + }, + { + "epoch": 2.2073434125269977, + "grad_norm": 0.6394441723823547, + "learning_rate": 0.0001, + "loss": 66.2924, + "step": 13670 + }, + { + "epoch": 2.2089582366120992, + "grad_norm": 0.6017318367958069, + "learning_rate": 0.0001, + "loss": 66.6313, + "step": 13680 + }, + { + "epoch": 2.2105730606972003, + "grad_norm": 0.5696585774421692, + "learning_rate": 0.0001, + "loss": 65.9198, + "step": 13690 + }, + { + "epoch": 2.2121878847823013, + "grad_norm": 0.5966867804527283, + "learning_rate": 0.0001, + "loss": 67.507, + "step": 13700 + }, + { + "epoch": 2.213802708867403, + "grad_norm": 0.6163555979728699, + "learning_rate": 0.0001, + "loss": 67.2444, + "step": 13710 + }, + { + "epoch": 2.215417532952504, + "grad_norm": 0.5670737624168396, + "learning_rate": 0.0001, + "loss": 66.5747, + "step": 13720 + }, + { + "epoch": 2.2170323570376054, + "grad_norm": 0.6289104223251343, + "learning_rate": 0.0001, + "loss": 65.6289, + "step": 13730 + }, + { + "epoch": 2.2186471811227064, + "grad_norm": 0.5649837255477905, + "learning_rate": 0.0001, + "loss": 66.0953, + "step": 13740 + }, + { + "epoch": 2.2202620052078075, + "grad_norm": 0.6066703200340271, + "learning_rate": 0.0001, + "loss": 65.9425, + "step": 13750 + }, + { + "epoch": 2.221876829292909, + "grad_norm": 0.6202117204666138, + "learning_rate": 0.0001, + "loss": 65.3268, + "step": 13760 + }, + { + "epoch": 2.22349165337801, + "grad_norm": 0.6883425116539001, + "learning_rate": 0.0001, + "loss": 64.1026, + "step": 13770 + }, + { + "epoch": 2.2251064774631115, + "grad_norm": 0.5564731955528259, + "learning_rate": 0.0001, + "loss": 65.1203, + "step": 13780 + }, + { + "epoch": 2.2267213015482126, + "grad_norm": 0.48229852318763733, + "learning_rate": 0.0001, + "loss": 65.2423, + "step": 13790 + }, + { + "epoch": 2.2283361256333136, + "grad_norm": 0.549240231513977, + "learning_rate": 0.0001, + "loss": 68.3811, + "step": 13800 + }, + { + "epoch": 2.229950949718415, + "grad_norm": 0.7217127084732056, + "learning_rate": 0.0001, + "loss": 64.9966, + "step": 13810 + }, + { + "epoch": 2.231565773803516, + "grad_norm": 0.8491377830505371, + "learning_rate": 0.0001, + "loss": 65.6468, + "step": 13820 + }, + { + "epoch": 2.2331805978886177, + "grad_norm": 0.5460603833198547, + "learning_rate": 0.0001, + "loss": 65.192, + "step": 13830 + }, + { + "epoch": 2.2347954219737187, + "grad_norm": 0.5784180164337158, + "learning_rate": 0.0001, + "loss": 67.3488, + "step": 13840 + }, + { + "epoch": 2.2364102460588198, + "grad_norm": 0.7575398087501526, + "learning_rate": 0.0001, + "loss": 65.6969, + "step": 13850 + }, + { + "epoch": 2.2380250701439213, + "grad_norm": 0.5111517310142517, + "learning_rate": 0.0001, + "loss": 66.482, + "step": 13860 + }, + { + "epoch": 2.2396398942290223, + "grad_norm": 0.6383832693099976, + "learning_rate": 0.0001, + "loss": 67.4894, + "step": 13870 + }, + { + "epoch": 2.241254718314124, + "grad_norm": 0.5922625064849854, + "learning_rate": 0.0001, + "loss": 66.1718, + "step": 13880 + }, + { + "epoch": 2.242869542399225, + "grad_norm": 5.395338535308838, + "learning_rate": 0.0001, + "loss": 64.7385, + "step": 13890 + }, + { + "epoch": 2.244484366484326, + "grad_norm": 1.4203481674194336, + "learning_rate": 0.0001, + "loss": 64.7676, + "step": 13900 + }, + { + "epoch": 2.2460991905694274, + "grad_norm": 0.6692256331443787, + "learning_rate": 0.0001, + "loss": 65.856, + "step": 13910 + }, + { + "epoch": 2.2477140146545285, + "grad_norm": 0.5868324041366577, + "learning_rate": 0.0001, + "loss": 66.1823, + "step": 13920 + }, + { + "epoch": 2.24932883873963, + "grad_norm": 0.6828433275222778, + "learning_rate": 0.0001, + "loss": 65.7601, + "step": 13930 + }, + { + "epoch": 2.250943662824731, + "grad_norm": 0.5108211040496826, + "learning_rate": 0.0001, + "loss": 63.413, + "step": 13940 + }, + { + "epoch": 2.252558486909832, + "grad_norm": 0.5538185238838196, + "learning_rate": 0.0001, + "loss": 64.9088, + "step": 13950 + }, + { + "epoch": 2.2541733109949336, + "grad_norm": 0.7211924195289612, + "learning_rate": 0.0001, + "loss": 66.0293, + "step": 13960 + }, + { + "epoch": 2.2557881350800346, + "grad_norm": 0.5632925033569336, + "learning_rate": 0.0001, + "loss": 64.8261, + "step": 13970 + }, + { + "epoch": 2.257402959165136, + "grad_norm": 0.574968159198761, + "learning_rate": 0.0001, + "loss": 65.2667, + "step": 13980 + }, + { + "epoch": 2.259017783250237, + "grad_norm": 0.5919519066810608, + "learning_rate": 0.0001, + "loss": 66.2643, + "step": 13990 + }, + { + "epoch": 2.260632607335338, + "grad_norm": 0.6732460260391235, + "learning_rate": 0.0001, + "loss": 64.7524, + "step": 14000 + }, + { + "epoch": 2.2622474314204397, + "grad_norm": 0.5927128791809082, + "learning_rate": 0.0001, + "loss": 65.653, + "step": 14010 + }, + { + "epoch": 2.2638622555055408, + "grad_norm": 0.5900641083717346, + "learning_rate": 0.0001, + "loss": 65.3171, + "step": 14020 + }, + { + "epoch": 2.2654770795906423, + "grad_norm": 0.5498414635658264, + "learning_rate": 0.0001, + "loss": 66.8886, + "step": 14030 + }, + { + "epoch": 2.2670919036757433, + "grad_norm": 0.7032901048660278, + "learning_rate": 0.0001, + "loss": 65.9949, + "step": 14040 + }, + { + "epoch": 2.2687067277608444, + "grad_norm": 0.6699029207229614, + "learning_rate": 0.0001, + "loss": 67.2374, + "step": 14050 + }, + { + "epoch": 2.270321551845946, + "grad_norm": 0.5847286581993103, + "learning_rate": 0.0001, + "loss": 63.996, + "step": 14060 + }, + { + "epoch": 2.271936375931047, + "grad_norm": 0.6173244714736938, + "learning_rate": 0.0001, + "loss": 66.927, + "step": 14070 + }, + { + "epoch": 2.2735512000161484, + "grad_norm": 0.6063929200172424, + "learning_rate": 0.0001, + "loss": 64.1213, + "step": 14080 + }, + { + "epoch": 2.2751660241012495, + "grad_norm": 0.5927698016166687, + "learning_rate": 0.0001, + "loss": 65.5379, + "step": 14090 + }, + { + "epoch": 2.2767808481863505, + "grad_norm": 0.6632462739944458, + "learning_rate": 0.0001, + "loss": 66.4461, + "step": 14100 + }, + { + "epoch": 2.278395672271452, + "grad_norm": 0.6032512187957764, + "learning_rate": 0.0001, + "loss": 65.3191, + "step": 14110 + }, + { + "epoch": 2.280010496356553, + "grad_norm": 0.5799509882926941, + "learning_rate": 0.0001, + "loss": 64.7406, + "step": 14120 + }, + { + "epoch": 2.2816253204416546, + "grad_norm": 0.6965101361274719, + "learning_rate": 0.0001, + "loss": 65.2813, + "step": 14130 + }, + { + "epoch": 2.2832401445267556, + "grad_norm": 0.6447696089744568, + "learning_rate": 0.0001, + "loss": 65.6274, + "step": 14140 + }, + { + "epoch": 2.2848549686118567, + "grad_norm": 0.7639263272285461, + "learning_rate": 0.0001, + "loss": 64.9972, + "step": 14150 + }, + { + "epoch": 2.286469792696958, + "grad_norm": 0.6040478944778442, + "learning_rate": 0.0001, + "loss": 66.1419, + "step": 14160 + }, + { + "epoch": 2.288084616782059, + "grad_norm": 0.6466631889343262, + "learning_rate": 0.0001, + "loss": 66.0022, + "step": 14170 + }, + { + "epoch": 2.2896994408671607, + "grad_norm": 0.6795923709869385, + "learning_rate": 0.0001, + "loss": 65.7562, + "step": 14180 + }, + { + "epoch": 2.2913142649522618, + "grad_norm": 0.5632458329200745, + "learning_rate": 0.0001, + "loss": 64.7196, + "step": 14190 + }, + { + "epoch": 2.292929089037363, + "grad_norm": 1.2942395210266113, + "learning_rate": 0.0001, + "loss": 65.3416, + "step": 14200 + }, + { + "epoch": 2.2945439131224643, + "grad_norm": 0.728187084197998, + "learning_rate": 0.0001, + "loss": 65.3743, + "step": 14210 + }, + { + "epoch": 2.2961587372075654, + "grad_norm": 0.5735196471214294, + "learning_rate": 0.0001, + "loss": 67.1841, + "step": 14220 + }, + { + "epoch": 2.297773561292667, + "grad_norm": 0.5804634690284729, + "learning_rate": 0.0001, + "loss": 65.856, + "step": 14230 + }, + { + "epoch": 2.299388385377768, + "grad_norm": 0.5995981693267822, + "learning_rate": 0.0001, + "loss": 65.7268, + "step": 14240 + }, + { + "epoch": 2.301003209462869, + "grad_norm": 0.4901711940765381, + "learning_rate": 0.0001, + "loss": 66.5355, + "step": 14250 + }, + { + "epoch": 2.3026180335479705, + "grad_norm": 0.6467465162277222, + "learning_rate": 0.0001, + "loss": 66.0457, + "step": 14260 + }, + { + "epoch": 2.3042328576330715, + "grad_norm": 0.6831316351890564, + "learning_rate": 0.0001, + "loss": 65.7363, + "step": 14270 + }, + { + "epoch": 2.305847681718173, + "grad_norm": 0.5308393836021423, + "learning_rate": 0.0001, + "loss": 64.5911, + "step": 14280 + }, + { + "epoch": 2.307462505803274, + "grad_norm": 0.6398120522499084, + "learning_rate": 0.0001, + "loss": 63.9705, + "step": 14290 + }, + { + "epoch": 2.309077329888375, + "grad_norm": 0.5657719969749451, + "learning_rate": 0.0001, + "loss": 65.9251, + "step": 14300 + }, + { + "epoch": 2.3106921539734766, + "grad_norm": 0.5824252367019653, + "learning_rate": 0.0001, + "loss": 64.7748, + "step": 14310 + }, + { + "epoch": 2.3123069780585777, + "grad_norm": 0.6470819115638733, + "learning_rate": 0.0001, + "loss": 65.2014, + "step": 14320 + }, + { + "epoch": 2.313921802143679, + "grad_norm": 0.643173336982727, + "learning_rate": 0.0001, + "loss": 65.9344, + "step": 14330 + }, + { + "epoch": 2.31553662622878, + "grad_norm": 0.6090084314346313, + "learning_rate": 0.0001, + "loss": 66.0823, + "step": 14340 + }, + { + "epoch": 2.3171514503138813, + "grad_norm": 0.627765953540802, + "learning_rate": 0.0001, + "loss": 66.2703, + "step": 14350 + }, + { + "epoch": 2.3187662743989828, + "grad_norm": 0.6484197974205017, + "learning_rate": 0.0001, + "loss": 65.928, + "step": 14360 + }, + { + "epoch": 2.320381098484084, + "grad_norm": 0.653978168964386, + "learning_rate": 0.0001, + "loss": 67.0193, + "step": 14370 + }, + { + "epoch": 2.3219959225691853, + "grad_norm": 0.6991199851036072, + "learning_rate": 0.0001, + "loss": 66.1559, + "step": 14380 + }, + { + "epoch": 2.3236107466542864, + "grad_norm": 0.938056468963623, + "learning_rate": 0.0001, + "loss": 66.6212, + "step": 14390 + }, + { + "epoch": 2.3252255707393874, + "grad_norm": 0.6154502630233765, + "learning_rate": 0.0001, + "loss": 64.5868, + "step": 14400 + }, + { + "epoch": 2.326840394824489, + "grad_norm": 0.6453447341918945, + "learning_rate": 0.0001, + "loss": 67.5199, + "step": 14410 + }, + { + "epoch": 2.32845521890959, + "grad_norm": 0.6678239107131958, + "learning_rate": 0.0001, + "loss": 65.3518, + "step": 14420 + }, + { + "epoch": 2.3300700429946914, + "grad_norm": 0.6608372330665588, + "learning_rate": 0.0001, + "loss": 65.1985, + "step": 14430 + }, + { + "epoch": 2.3316848670797925, + "grad_norm": 0.625511646270752, + "learning_rate": 0.0001, + "loss": 66.9224, + "step": 14440 + }, + { + "epoch": 2.3332996911648936, + "grad_norm": 0.6678557991981506, + "learning_rate": 0.0001, + "loss": 65.8618, + "step": 14450 + }, + { + "epoch": 2.334914515249995, + "grad_norm": 0.6184708476066589, + "learning_rate": 0.0001, + "loss": 66.5419, + "step": 14460 + }, + { + "epoch": 2.336529339335096, + "grad_norm": 0.6668255925178528, + "learning_rate": 0.0001, + "loss": 64.0567, + "step": 14470 + }, + { + "epoch": 2.3381441634201976, + "grad_norm": 0.5756039023399353, + "learning_rate": 0.0001, + "loss": 63.043, + "step": 14480 + }, + { + "epoch": 2.3397589875052986, + "grad_norm": 0.6090635657310486, + "learning_rate": 0.0001, + "loss": 65.8793, + "step": 14490 + }, + { + "epoch": 2.3413738115903997, + "grad_norm": 0.8366266489028931, + "learning_rate": 0.0001, + "loss": 65.1995, + "step": 14500 + }, + { + "epoch": 2.342988635675501, + "grad_norm": 0.6535093188285828, + "learning_rate": 0.0001, + "loss": 63.3831, + "step": 14510 + }, + { + "epoch": 2.3446034597606022, + "grad_norm": 0.9697316884994507, + "learning_rate": 0.0001, + "loss": 65.1754, + "step": 14520 + }, + { + "epoch": 2.3462182838457037, + "grad_norm": 0.7946458458900452, + "learning_rate": 0.0001, + "loss": 65.5404, + "step": 14530 + }, + { + "epoch": 2.347833107930805, + "grad_norm": 0.5621834397315979, + "learning_rate": 0.0001, + "loss": 65.8532, + "step": 14540 + }, + { + "epoch": 2.349447932015906, + "grad_norm": 0.6668163537979126, + "learning_rate": 0.0001, + "loss": 65.0884, + "step": 14550 + }, + { + "epoch": 2.3510627561010073, + "grad_norm": 0.5969575047492981, + "learning_rate": 0.0001, + "loss": 66.5821, + "step": 14560 + }, + { + "epoch": 2.3526775801861084, + "grad_norm": 0.6905517578125, + "learning_rate": 0.0001, + "loss": 65.0928, + "step": 14570 + }, + { + "epoch": 2.35429240427121, + "grad_norm": 0.5888230204582214, + "learning_rate": 0.0001, + "loss": 67.4596, + "step": 14580 + }, + { + "epoch": 2.355907228356311, + "grad_norm": 0.5115624666213989, + "learning_rate": 0.0001, + "loss": 65.771, + "step": 14590 + }, + { + "epoch": 2.357522052441412, + "grad_norm": 0.7888309359550476, + "learning_rate": 0.0001, + "loss": 63.5326, + "step": 14600 + }, + { + "epoch": 2.3591368765265135, + "grad_norm": 0.6653981804847717, + "learning_rate": 0.0001, + "loss": 64.8015, + "step": 14610 + }, + { + "epoch": 2.3607517006116145, + "grad_norm": 0.7898159027099609, + "learning_rate": 0.0001, + "loss": 66.398, + "step": 14620 + }, + { + "epoch": 2.362366524696716, + "grad_norm": 1.02628755569458, + "learning_rate": 0.0001, + "loss": 67.6004, + "step": 14630 + }, + { + "epoch": 2.363981348781817, + "grad_norm": 0.5777941942214966, + "learning_rate": 0.0001, + "loss": 64.9402, + "step": 14640 + }, + { + "epoch": 2.365596172866918, + "grad_norm": 0.7004761695861816, + "learning_rate": 0.0001, + "loss": 64.9344, + "step": 14650 + }, + { + "epoch": 2.3672109969520196, + "grad_norm": 0.6230147480964661, + "learning_rate": 0.0001, + "loss": 64.8412, + "step": 14660 + }, + { + "epoch": 2.3688258210371207, + "grad_norm": 0.7234131097793579, + "learning_rate": 0.0001, + "loss": 65.4454, + "step": 14670 + }, + { + "epoch": 2.370440645122222, + "grad_norm": 0.7054212093353271, + "learning_rate": 0.0001, + "loss": 62.2294, + "step": 14680 + }, + { + "epoch": 2.3720554692073232, + "grad_norm": 0.5377123355865479, + "learning_rate": 0.0001, + "loss": 63.1186, + "step": 14690 + }, + { + "epoch": 2.3736702932924243, + "grad_norm": 0.5648056864738464, + "learning_rate": 0.0001, + "loss": 66.1893, + "step": 14700 + }, + { + "epoch": 2.375285117377526, + "grad_norm": 0.6017388105392456, + "learning_rate": 0.0001, + "loss": 64.8003, + "step": 14710 + }, + { + "epoch": 2.376899941462627, + "grad_norm": 0.592069149017334, + "learning_rate": 0.0001, + "loss": 64.7885, + "step": 14720 + }, + { + "epoch": 2.3785147655477283, + "grad_norm": 0.5448320508003235, + "learning_rate": 0.0001, + "loss": 64.3022, + "step": 14730 + }, + { + "epoch": 2.3801295896328294, + "grad_norm": 0.7200132012367249, + "learning_rate": 0.0001, + "loss": 65.3915, + "step": 14740 + }, + { + "epoch": 2.3817444137179304, + "grad_norm": 0.6497626304626465, + "learning_rate": 0.0001, + "loss": 63.9914, + "step": 14750 + }, + { + "epoch": 2.383359237803032, + "grad_norm": 0.7847583293914795, + "learning_rate": 0.0001, + "loss": 65.7746, + "step": 14760 + }, + { + "epoch": 2.384974061888133, + "grad_norm": 0.6991321444511414, + "learning_rate": 0.0001, + "loss": 65.0386, + "step": 14770 + }, + { + "epoch": 2.386588885973234, + "grad_norm": 0.5715296268463135, + "learning_rate": 0.0001, + "loss": 64.2633, + "step": 14780 + }, + { + "epoch": 2.3882037100583355, + "grad_norm": 0.5266057848930359, + "learning_rate": 0.0001, + "loss": 63.0002, + "step": 14790 + }, + { + "epoch": 2.3898185341434366, + "grad_norm": 0.7040116786956787, + "learning_rate": 0.0001, + "loss": 66.2895, + "step": 14800 + }, + { + "epoch": 2.391433358228538, + "grad_norm": 0.6963793635368347, + "learning_rate": 0.0001, + "loss": 64.7487, + "step": 14810 + }, + { + "epoch": 2.393048182313639, + "grad_norm": 0.5361505746841431, + "learning_rate": 0.0001, + "loss": 66.6834, + "step": 14820 + }, + { + "epoch": 2.3946630063987406, + "grad_norm": 0.8016838431358337, + "learning_rate": 0.0001, + "loss": 64.3359, + "step": 14830 + }, + { + "epoch": 2.3962778304838417, + "grad_norm": 0.5636866092681885, + "learning_rate": 0.0001, + "loss": 65.2468, + "step": 14840 + }, + { + "epoch": 2.3978926545689427, + "grad_norm": 0.6926373839378357, + "learning_rate": 0.0001, + "loss": 66.3353, + "step": 14850 + }, + { + "epoch": 2.3995074786540442, + "grad_norm": 0.6361585855484009, + "learning_rate": 0.0001, + "loss": 64.8466, + "step": 14860 + }, + { + "epoch": 2.4011223027391453, + "grad_norm": 0.6232298612594604, + "learning_rate": 0.0001, + "loss": 65.1253, + "step": 14870 + }, + { + "epoch": 2.4027371268242463, + "grad_norm": 0.5644697546958923, + "learning_rate": 0.0001, + "loss": 67.1739, + "step": 14880 + }, + { + "epoch": 2.404351950909348, + "grad_norm": 0.5655877590179443, + "learning_rate": 0.0001, + "loss": 65.1823, + "step": 14890 + }, + { + "epoch": 2.405966774994449, + "grad_norm": 0.8255041837692261, + "learning_rate": 0.0001, + "loss": 64.6833, + "step": 14900 + }, + { + "epoch": 2.4075815990795504, + "grad_norm": 0.856689989566803, + "learning_rate": 0.0001, + "loss": 64.4978, + "step": 14910 + }, + { + "epoch": 2.4091964231646514, + "grad_norm": 0.6280230283737183, + "learning_rate": 0.0001, + "loss": 66.3989, + "step": 14920 + }, + { + "epoch": 2.410811247249753, + "grad_norm": 0.5739511251449585, + "learning_rate": 0.0001, + "loss": 64.6365, + "step": 14930 + }, + { + "epoch": 2.412426071334854, + "grad_norm": 0.6244319677352905, + "learning_rate": 0.0001, + "loss": 65.5078, + "step": 14940 + }, + { + "epoch": 2.414040895419955, + "grad_norm": 0.6932142376899719, + "learning_rate": 0.0001, + "loss": 65.1869, + "step": 14950 + }, + { + "epoch": 2.4156557195050565, + "grad_norm": 0.5393000245094299, + "learning_rate": 0.0001, + "loss": 65.553, + "step": 14960 + }, + { + "epoch": 2.4172705435901576, + "grad_norm": 0.6555871963500977, + "learning_rate": 0.0001, + "loss": 66.625, + "step": 14970 + }, + { + "epoch": 2.4188853676752586, + "grad_norm": 0.6515706181526184, + "learning_rate": 0.0001, + "loss": 64.876, + "step": 14980 + }, + { + "epoch": 2.42050019176036, + "grad_norm": 0.6380309462547302, + "learning_rate": 0.0001, + "loss": 63.1801, + "step": 14990 + }, + { + "epoch": 2.422115015845461, + "grad_norm": 0.8313520550727844, + "learning_rate": 0.0001, + "loss": 66.2692, + "step": 15000 + }, + { + "epoch": 2.4237298399305627, + "grad_norm": 0.6654364466667175, + "learning_rate": 0.0001, + "loss": 65.4955, + "step": 15010 + }, + { + "epoch": 2.4253446640156637, + "grad_norm": 0.6382327675819397, + "learning_rate": 0.0001, + "loss": 65.0912, + "step": 15020 + }, + { + "epoch": 2.4269594881007652, + "grad_norm": 0.5685209035873413, + "learning_rate": 0.0001, + "loss": 64.8058, + "step": 15030 + }, + { + "epoch": 2.4285743121858663, + "grad_norm": 0.6129642724990845, + "learning_rate": 0.0001, + "loss": 65.8475, + "step": 15040 + }, + { + "epoch": 2.4301891362709673, + "grad_norm": 0.6892847418785095, + "learning_rate": 0.0001, + "loss": 64.7247, + "step": 15050 + }, + { + "epoch": 2.431803960356069, + "grad_norm": 0.6307751536369324, + "learning_rate": 0.0001, + "loss": 64.0042, + "step": 15060 + }, + { + "epoch": 2.43341878444117, + "grad_norm": 0.5506501197814941, + "learning_rate": 0.0001, + "loss": 64.5555, + "step": 15070 + }, + { + "epoch": 2.435033608526271, + "grad_norm": 0.5580377578735352, + "learning_rate": 0.0001, + "loss": 65.0344, + "step": 15080 + }, + { + "epoch": 2.4366484326113724, + "grad_norm": 0.5712753534317017, + "learning_rate": 0.0001, + "loss": 66.6778, + "step": 15090 + }, + { + "epoch": 2.4382632566964735, + "grad_norm": 0.7153586745262146, + "learning_rate": 0.0001, + "loss": 65.893, + "step": 15100 + }, + { + "epoch": 2.439878080781575, + "grad_norm": 0.6946969628334045, + "learning_rate": 0.0001, + "loss": 64.1815, + "step": 15110 + }, + { + "epoch": 2.441492904866676, + "grad_norm": 0.608729898929596, + "learning_rate": 0.0001, + "loss": 64.8054, + "step": 15120 + }, + { + "epoch": 2.4431077289517775, + "grad_norm": 0.6703302264213562, + "learning_rate": 0.0001, + "loss": 65.3736, + "step": 15130 + }, + { + "epoch": 2.4447225530368786, + "grad_norm": 0.736193835735321, + "learning_rate": 0.0001, + "loss": 66.7051, + "step": 15140 + }, + { + "epoch": 2.4463373771219796, + "grad_norm": 0.48381003737449646, + "learning_rate": 0.0001, + "loss": 64.3723, + "step": 15150 + }, + { + "epoch": 2.447952201207081, + "grad_norm": 0.7487664222717285, + "learning_rate": 0.0001, + "loss": 64.4967, + "step": 15160 + }, + { + "epoch": 2.449567025292182, + "grad_norm": 0.7033581733703613, + "learning_rate": 0.0001, + "loss": 66.2185, + "step": 15170 + }, + { + "epoch": 2.4511818493772832, + "grad_norm": 0.5489983558654785, + "learning_rate": 0.0001, + "loss": 64.9698, + "step": 15180 + }, + { + "epoch": 2.4527966734623847, + "grad_norm": 0.6467112898826599, + "learning_rate": 0.0001, + "loss": 64.5987, + "step": 15190 + }, + { + "epoch": 2.4544114975474858, + "grad_norm": 0.6499149799346924, + "learning_rate": 0.0001, + "loss": 65.757, + "step": 15200 + }, + { + "epoch": 2.4560263216325873, + "grad_norm": 0.7622562050819397, + "learning_rate": 0.0001, + "loss": 66.4642, + "step": 15210 + }, + { + "epoch": 2.4576411457176883, + "grad_norm": 0.8985865712165833, + "learning_rate": 0.0001, + "loss": 64.5804, + "step": 15220 + }, + { + "epoch": 2.45925596980279, + "grad_norm": 0.6116635203361511, + "learning_rate": 0.0001, + "loss": 64.8022, + "step": 15230 + }, + { + "epoch": 2.460870793887891, + "grad_norm": 0.6313971281051636, + "learning_rate": 0.0001, + "loss": 63.4837, + "step": 15240 + }, + { + "epoch": 2.462485617972992, + "grad_norm": 0.7002763152122498, + "learning_rate": 0.0001, + "loss": 64.7201, + "step": 15250 + }, + { + "epoch": 2.4641004420580934, + "grad_norm": 0.6030148863792419, + "learning_rate": 0.0001, + "loss": 65.0987, + "step": 15260 + }, + { + "epoch": 2.4657152661431945, + "grad_norm": 0.5838733911514282, + "learning_rate": 0.0001, + "loss": 64.9182, + "step": 15270 + }, + { + "epoch": 2.4673300902282955, + "grad_norm": 1.1749393939971924, + "learning_rate": 0.0001, + "loss": 63.4387, + "step": 15280 + }, + { + "epoch": 2.468944914313397, + "grad_norm": 0.5215787291526794, + "learning_rate": 0.0001, + "loss": 64.7365, + "step": 15290 + }, + { + "epoch": 2.470559738398498, + "grad_norm": 0.5576916933059692, + "learning_rate": 0.0001, + "loss": 65.644, + "step": 15300 + }, + { + "epoch": 2.4721745624835996, + "grad_norm": 0.6760591864585876, + "learning_rate": 0.0001, + "loss": 64.729, + "step": 15310 + }, + { + "epoch": 2.4737893865687006, + "grad_norm": 0.7162330150604248, + "learning_rate": 0.0001, + "loss": 66.8289, + "step": 15320 + }, + { + "epoch": 2.475404210653802, + "grad_norm": 0.5687427520751953, + "learning_rate": 0.0001, + "loss": 63.9522, + "step": 15330 + }, + { + "epoch": 2.477019034738903, + "grad_norm": 0.5663572549819946, + "learning_rate": 0.0001, + "loss": 65.7151, + "step": 15340 + }, + { + "epoch": 2.478633858824004, + "grad_norm": 0.6018327474594116, + "learning_rate": 0.0001, + "loss": 64.6599, + "step": 15350 + }, + { + "epoch": 2.4802486829091057, + "grad_norm": 0.6908984184265137, + "learning_rate": 0.0001, + "loss": 65.3083, + "step": 15360 + }, + { + "epoch": 2.4818635069942068, + "grad_norm": 0.6582157015800476, + "learning_rate": 0.0001, + "loss": 64.6219, + "step": 15370 + }, + { + "epoch": 2.483478331079308, + "grad_norm": 0.7800308465957642, + "learning_rate": 0.0001, + "loss": 66.6917, + "step": 15380 + }, + { + "epoch": 2.4850931551644093, + "grad_norm": 0.5621150732040405, + "learning_rate": 0.0001, + "loss": 66.6579, + "step": 15390 + }, + { + "epoch": 2.4867079792495104, + "grad_norm": 0.6318013668060303, + "learning_rate": 0.0001, + "loss": 64.8888, + "step": 15400 + }, + { + "epoch": 2.488322803334612, + "grad_norm": 0.522668719291687, + "learning_rate": 0.0001, + "loss": 64.6468, + "step": 15410 + }, + { + "epoch": 2.489937627419713, + "grad_norm": 0.5940443873405457, + "learning_rate": 0.0001, + "loss": 65.0119, + "step": 15420 + }, + { + "epoch": 2.4915524515048144, + "grad_norm": 0.5932700037956238, + "learning_rate": 0.0001, + "loss": 66.2562, + "step": 15430 + }, + { + "epoch": 2.4931672755899155, + "grad_norm": 0.5753180980682373, + "learning_rate": 0.0001, + "loss": 65.6223, + "step": 15440 + }, + { + "epoch": 2.4947820996750165, + "grad_norm": 1.0136449337005615, + "learning_rate": 0.0001, + "loss": 65.4779, + "step": 15450 + }, + { + "epoch": 2.496396923760118, + "grad_norm": 0.6616164445877075, + "learning_rate": 0.0001, + "loss": 65.2368, + "step": 15460 + }, + { + "epoch": 2.498011747845219, + "grad_norm": 0.584385871887207, + "learning_rate": 0.0001, + "loss": 63.5195, + "step": 15470 + }, + { + "epoch": 2.49962657193032, + "grad_norm": 0.6892259120941162, + "learning_rate": 0.0001, + "loss": 63.3364, + "step": 15480 + }, + { + "epoch": 2.5012413960154216, + "grad_norm": 0.594386637210846, + "learning_rate": 0.0001, + "loss": 65.062, + "step": 15490 + }, + { + "epoch": 2.5028562201005227, + "grad_norm": 0.6604944467544556, + "learning_rate": 0.0001, + "loss": 65.3744, + "step": 15500 + }, + { + "epoch": 2.504471044185624, + "grad_norm": 0.5402690768241882, + "learning_rate": 0.0001, + "loss": 65.0583, + "step": 15510 + }, + { + "epoch": 2.506085868270725, + "grad_norm": 0.4574640989303589, + "learning_rate": 0.0001, + "loss": 64.3828, + "step": 15520 + }, + { + "epoch": 2.5077006923558267, + "grad_norm": 0.62662672996521, + "learning_rate": 0.0001, + "loss": 62.774, + "step": 15530 + }, + { + "epoch": 2.5093155164409278, + "grad_norm": 1.1336137056350708, + "learning_rate": 0.0001, + "loss": 63.9481, + "step": 15540 + }, + { + "epoch": 2.510930340526029, + "grad_norm": 0.7171993255615234, + "learning_rate": 0.0001, + "loss": 63.2949, + "step": 15550 + }, + { + "epoch": 2.5125451646111303, + "grad_norm": 0.6186808943748474, + "learning_rate": 0.0001, + "loss": 65.3805, + "step": 15560 + }, + { + "epoch": 2.5141599886962314, + "grad_norm": 0.5631883144378662, + "learning_rate": 0.0001, + "loss": 65.9464, + "step": 15570 + }, + { + "epoch": 2.5157748127813324, + "grad_norm": 0.6015043258666992, + "learning_rate": 0.0001, + "loss": 66.5938, + "step": 15580 + }, + { + "epoch": 2.517389636866434, + "grad_norm": 0.63025963306427, + "learning_rate": 0.0001, + "loss": 65.6977, + "step": 15590 + }, + { + "epoch": 2.519004460951535, + "grad_norm": 0.6715611815452576, + "learning_rate": 0.0001, + "loss": 66.1286, + "step": 15600 + }, + { + "epoch": 2.5206192850366365, + "grad_norm": 0.6191041469573975, + "learning_rate": 0.0001, + "loss": 64.257, + "step": 15610 + }, + { + "epoch": 2.5222341091217375, + "grad_norm": 0.5771703720092773, + "learning_rate": 0.0001, + "loss": 64.1293, + "step": 15620 + }, + { + "epoch": 2.523848933206839, + "grad_norm": 0.6421854496002197, + "learning_rate": 0.0001, + "loss": 64.27, + "step": 15630 + }, + { + "epoch": 2.52546375729194, + "grad_norm": 0.5963718295097351, + "learning_rate": 0.0001, + "loss": 66.5889, + "step": 15640 + }, + { + "epoch": 2.527078581377041, + "grad_norm": 0.5750302672386169, + "learning_rate": 0.0001, + "loss": 64.4224, + "step": 15650 + }, + { + "epoch": 2.5286934054621426, + "grad_norm": 0.558082640171051, + "learning_rate": 0.0001, + "loss": 64.3853, + "step": 15660 + }, + { + "epoch": 2.5303082295472437, + "grad_norm": 0.6801859736442566, + "learning_rate": 0.0001, + "loss": 65.5297, + "step": 15670 + }, + { + "epoch": 2.5319230536323447, + "grad_norm": 0.7200396656990051, + "learning_rate": 0.0001, + "loss": 66.7467, + "step": 15680 + }, + { + "epoch": 2.533537877717446, + "grad_norm": 0.5736697316169739, + "learning_rate": 0.0001, + "loss": 62.2177, + "step": 15690 + }, + { + "epoch": 2.5351527018025473, + "grad_norm": 1.6533524990081787, + "learning_rate": 0.0001, + "loss": 65.245, + "step": 15700 + }, + { + "epoch": 2.5367675258876488, + "grad_norm": 0.578233003616333, + "learning_rate": 0.0001, + "loss": 67.0796, + "step": 15710 + }, + { + "epoch": 2.53838234997275, + "grad_norm": 0.5405007600784302, + "learning_rate": 0.0001, + "loss": 65.4651, + "step": 15720 + }, + { + "epoch": 2.5399971740578513, + "grad_norm": 0.65571528673172, + "learning_rate": 0.0001, + "loss": 65.8385, + "step": 15730 + }, + { + "epoch": 2.5416119981429524, + "grad_norm": 0.5576966404914856, + "learning_rate": 0.0001, + "loss": 64.3478, + "step": 15740 + }, + { + "epoch": 2.5432268222280534, + "grad_norm": 0.7222445011138916, + "learning_rate": 0.0001, + "loss": 65.8258, + "step": 15750 + }, + { + "epoch": 2.544841646313155, + "grad_norm": 0.6464501619338989, + "learning_rate": 0.0001, + "loss": 63.535, + "step": 15760 + }, + { + "epoch": 2.546456470398256, + "grad_norm": 0.5374952554702759, + "learning_rate": 0.0001, + "loss": 63.7492, + "step": 15770 + }, + { + "epoch": 2.548071294483357, + "grad_norm": 0.6572895646095276, + "learning_rate": 0.0001, + "loss": 66.2192, + "step": 15780 + }, + { + "epoch": 2.5496861185684585, + "grad_norm": 0.578153133392334, + "learning_rate": 0.0001, + "loss": 64.311, + "step": 15790 + }, + { + "epoch": 2.5513009426535596, + "grad_norm": 0.6136942505836487, + "learning_rate": 0.0001, + "loss": 64.6734, + "step": 15800 + }, + { + "epoch": 2.552915766738661, + "grad_norm": 0.8686643242835999, + "learning_rate": 0.0001, + "loss": 64.5421, + "step": 15810 + }, + { + "epoch": 2.554530590823762, + "grad_norm": 0.8858036994934082, + "learning_rate": 0.0001, + "loss": 66.358, + "step": 15820 + }, + { + "epoch": 2.5561454149088636, + "grad_norm": 0.6373804807662964, + "learning_rate": 0.0001, + "loss": 63.7582, + "step": 15830 + }, + { + "epoch": 2.5577602389939647, + "grad_norm": 1.5556457042694092, + "learning_rate": 0.0001, + "loss": 63.8109, + "step": 15840 + }, + { + "epoch": 2.5593750630790657, + "grad_norm": 0.5945340394973755, + "learning_rate": 0.0001, + "loss": 64.7864, + "step": 15850 + }, + { + "epoch": 2.560989887164167, + "grad_norm": 0.6172908544540405, + "learning_rate": 0.0001, + "loss": 62.7738, + "step": 15860 + }, + { + "epoch": 2.5626047112492683, + "grad_norm": 0.6360666155815125, + "learning_rate": 0.0001, + "loss": 63.4814, + "step": 15870 + }, + { + "epoch": 2.5642195353343693, + "grad_norm": 0.7986658811569214, + "learning_rate": 0.0001, + "loss": 64.3178, + "step": 15880 + }, + { + "epoch": 2.565834359419471, + "grad_norm": 0.5716906785964966, + "learning_rate": 0.0001, + "loss": 64.371, + "step": 15890 + }, + { + "epoch": 2.567449183504572, + "grad_norm": 0.5859067440032959, + "learning_rate": 0.0001, + "loss": 64.0359, + "step": 15900 + }, + { + "epoch": 2.5690640075896733, + "grad_norm": 0.6288742423057556, + "learning_rate": 0.0001, + "loss": 64.4613, + "step": 15910 + }, + { + "epoch": 2.5706788316747744, + "grad_norm": 0.9052578210830688, + "learning_rate": 0.0001, + "loss": 66.3599, + "step": 15920 + }, + { + "epoch": 2.572293655759876, + "grad_norm": 0.5532401204109192, + "learning_rate": 0.0001, + "loss": 64.1753, + "step": 15930 + }, + { + "epoch": 2.573908479844977, + "grad_norm": 0.5749264359474182, + "learning_rate": 0.0001, + "loss": 63.4118, + "step": 15940 + }, + { + "epoch": 2.575523303930078, + "grad_norm": 2.064448356628418, + "learning_rate": 0.0001, + "loss": 63.2922, + "step": 15950 + }, + { + "epoch": 2.5771381280151795, + "grad_norm": 0.6260086297988892, + "learning_rate": 0.0001, + "loss": 65.2183, + "step": 15960 + }, + { + "epoch": 2.5787529521002805, + "grad_norm": 0.5910785794258118, + "learning_rate": 0.0001, + "loss": 63.3134, + "step": 15970 + }, + { + "epoch": 2.5803677761853816, + "grad_norm": 0.5972382426261902, + "learning_rate": 0.0001, + "loss": 64.3669, + "step": 15980 + }, + { + "epoch": 2.581982600270483, + "grad_norm": 0.5171772837638855, + "learning_rate": 0.0001, + "loss": 64.3819, + "step": 15990 + }, + { + "epoch": 2.583597424355584, + "grad_norm": 0.5792955160140991, + "learning_rate": 0.0001, + "loss": 65.8632, + "step": 16000 + }, + { + "epoch": 2.5852122484406856, + "grad_norm": 0.6603825092315674, + "learning_rate": 0.0001, + "loss": 63.7743, + "step": 16010 + }, + { + "epoch": 2.5868270725257867, + "grad_norm": 0.6616955399513245, + "learning_rate": 0.0001, + "loss": 64.5524, + "step": 16020 + }, + { + "epoch": 2.588441896610888, + "grad_norm": 0.6547766327857971, + "learning_rate": 0.0001, + "loss": 63.9554, + "step": 16030 + }, + { + "epoch": 2.5900567206959892, + "grad_norm": 0.6487435698509216, + "learning_rate": 0.0001, + "loss": 65.8675, + "step": 16040 + }, + { + "epoch": 2.5916715447810903, + "grad_norm": 0.7175359129905701, + "learning_rate": 0.0001, + "loss": 64.2535, + "step": 16050 + }, + { + "epoch": 2.593286368866192, + "grad_norm": 0.6271935701370239, + "learning_rate": 0.0001, + "loss": 65.0441, + "step": 16060 + }, + { + "epoch": 2.594901192951293, + "grad_norm": 0.5246796607971191, + "learning_rate": 0.0001, + "loss": 62.7952, + "step": 16070 + }, + { + "epoch": 2.596516017036394, + "grad_norm": 0.5847776532173157, + "learning_rate": 0.0001, + "loss": 65.7221, + "step": 16080 + }, + { + "epoch": 2.5981308411214954, + "grad_norm": 0.6071584224700928, + "learning_rate": 0.0001, + "loss": 65.3868, + "step": 16090 + }, + { + "epoch": 2.5997456652065964, + "grad_norm": 0.8454984426498413, + "learning_rate": 0.0001, + "loss": 62.8597, + "step": 16100 + }, + { + "epoch": 2.601360489291698, + "grad_norm": 0.7428585290908813, + "learning_rate": 0.0001, + "loss": 66.3603, + "step": 16110 + }, + { + "epoch": 2.602975313376799, + "grad_norm": 0.586359977722168, + "learning_rate": 0.0001, + "loss": 64.1264, + "step": 16120 + }, + { + "epoch": 2.6045901374619005, + "grad_norm": 0.6131327152252197, + "learning_rate": 0.0001, + "loss": 65.4991, + "step": 16130 + }, + { + "epoch": 2.6062049615470015, + "grad_norm": 0.5977164506912231, + "learning_rate": 0.0001, + "loss": 64.4486, + "step": 16140 + }, + { + "epoch": 2.6078197856321026, + "grad_norm": 0.5891970992088318, + "learning_rate": 0.0001, + "loss": 63.7493, + "step": 16150 + }, + { + "epoch": 2.609434609717204, + "grad_norm": 0.5650322437286377, + "learning_rate": 0.0001, + "loss": 63.5063, + "step": 16160 + }, + { + "epoch": 2.611049433802305, + "grad_norm": 0.6460058689117432, + "learning_rate": 0.0001, + "loss": 63.4877, + "step": 16170 + }, + { + "epoch": 2.612664257887406, + "grad_norm": 1.0753989219665527, + "learning_rate": 0.0001, + "loss": 63.8523, + "step": 16180 + }, + { + "epoch": 2.6142790819725077, + "grad_norm": 0.5751987099647522, + "learning_rate": 0.0001, + "loss": 66.5687, + "step": 16190 + }, + { + "epoch": 2.6158939060576087, + "grad_norm": 0.7001827955245972, + "learning_rate": 0.0001, + "loss": 65.8639, + "step": 16200 + }, + { + "epoch": 2.6175087301427102, + "grad_norm": 0.6699106693267822, + "learning_rate": 0.0001, + "loss": 64.5044, + "step": 16210 + }, + { + "epoch": 2.6191235542278113, + "grad_norm": 0.5610349774360657, + "learning_rate": 0.0001, + "loss": 64.5379, + "step": 16220 + }, + { + "epoch": 2.620738378312913, + "grad_norm": 0.5153414607048035, + "learning_rate": 0.0001, + "loss": 66.7831, + "step": 16230 + }, + { + "epoch": 2.622353202398014, + "grad_norm": 0.5742719769477844, + "learning_rate": 0.0001, + "loss": 64.6758, + "step": 16240 + }, + { + "epoch": 2.623968026483115, + "grad_norm": 0.5706104636192322, + "learning_rate": 0.0001, + "loss": 65.3165, + "step": 16250 + }, + { + "epoch": 2.6255828505682164, + "grad_norm": 0.6260847449302673, + "learning_rate": 0.0001, + "loss": 64.3578, + "step": 16260 + }, + { + "epoch": 2.6271976746533174, + "grad_norm": 0.5445175766944885, + "learning_rate": 0.0001, + "loss": 63.3265, + "step": 16270 + }, + { + "epoch": 2.6288124987384185, + "grad_norm": 0.7492052316665649, + "learning_rate": 0.0001, + "loss": 62.7638, + "step": 16280 + }, + { + "epoch": 2.63042732282352, + "grad_norm": 0.6429327130317688, + "learning_rate": 0.0001, + "loss": 64.2921, + "step": 16290 + }, + { + "epoch": 2.632042146908621, + "grad_norm": 0.6327224969863892, + "learning_rate": 0.0001, + "loss": 65.3948, + "step": 16300 + }, + { + "epoch": 2.6336569709937225, + "grad_norm": 0.5778248906135559, + "learning_rate": 0.0001, + "loss": 63.8617, + "step": 16310 + }, + { + "epoch": 2.6352717950788236, + "grad_norm": 0.5628714561462402, + "learning_rate": 0.0001, + "loss": 62.5165, + "step": 16320 + }, + { + "epoch": 2.636886619163925, + "grad_norm": 0.6409966945648193, + "learning_rate": 0.0001, + "loss": 64.2843, + "step": 16330 + }, + { + "epoch": 2.638501443249026, + "grad_norm": 0.7780113816261292, + "learning_rate": 0.0001, + "loss": 64.7801, + "step": 16340 + }, + { + "epoch": 2.640116267334127, + "grad_norm": 0.5766817331314087, + "learning_rate": 0.0001, + "loss": 65.0183, + "step": 16350 + }, + { + "epoch": 2.6417310914192287, + "grad_norm": 0.6962740421295166, + "learning_rate": 0.0001, + "loss": 65.9575, + "step": 16360 + }, + { + "epoch": 2.6433459155043297, + "grad_norm": 0.5021215081214905, + "learning_rate": 0.0001, + "loss": 66.8226, + "step": 16370 + }, + { + "epoch": 2.644960739589431, + "grad_norm": 0.6883698105812073, + "learning_rate": 0.0001, + "loss": 63.6297, + "step": 16380 + }, + { + "epoch": 2.6465755636745323, + "grad_norm": 0.7089389562606812, + "learning_rate": 0.0001, + "loss": 65.1165, + "step": 16390 + }, + { + "epoch": 2.6481903877596333, + "grad_norm": 0.5431233644485474, + "learning_rate": 0.0001, + "loss": 63.7242, + "step": 16400 + }, + { + "epoch": 2.6498052118447344, + "grad_norm": 0.6602722406387329, + "learning_rate": 0.0001, + "loss": 64.497, + "step": 16410 + }, + { + "epoch": 2.651420035929836, + "grad_norm": 0.5621533393859863, + "learning_rate": 0.0001, + "loss": 63.9001, + "step": 16420 + }, + { + "epoch": 2.6530348600149374, + "grad_norm": 0.612298846244812, + "learning_rate": 0.0001, + "loss": 65.2207, + "step": 16430 + }, + { + "epoch": 2.6546496841000384, + "grad_norm": 0.6222931146621704, + "learning_rate": 0.0001, + "loss": 64.5573, + "step": 16440 + }, + { + "epoch": 2.6562645081851395, + "grad_norm": 0.5502400994300842, + "learning_rate": 0.0001, + "loss": 63.9544, + "step": 16450 + }, + { + "epoch": 2.657879332270241, + "grad_norm": 0.5200631022453308, + "learning_rate": 0.0001, + "loss": 62.4098, + "step": 16460 + }, + { + "epoch": 2.659494156355342, + "grad_norm": 0.6516156196594238, + "learning_rate": 0.0001, + "loss": 63.4309, + "step": 16470 + }, + { + "epoch": 2.661108980440443, + "grad_norm": 0.5822672247886658, + "learning_rate": 0.0001, + "loss": 64.9704, + "step": 16480 + }, + { + "epoch": 2.6627238045255446, + "grad_norm": 0.7026363611221313, + "learning_rate": 0.0001, + "loss": 64.8368, + "step": 16490 + }, + { + "epoch": 2.6643386286106456, + "grad_norm": 0.5978866219520569, + "learning_rate": 0.0001, + "loss": 63.8391, + "step": 16500 + }, + { + "epoch": 2.6659534526957467, + "grad_norm": 0.6636821627616882, + "learning_rate": 0.0001, + "loss": 64.5218, + "step": 16510 + }, + { + "epoch": 2.667568276780848, + "grad_norm": 0.8163989782333374, + "learning_rate": 0.0001, + "loss": 64.7803, + "step": 16520 + }, + { + "epoch": 2.6691831008659497, + "grad_norm": 0.6390615701675415, + "learning_rate": 0.0001, + "loss": 65.8387, + "step": 16530 + }, + { + "epoch": 2.6707979249510507, + "grad_norm": 0.714682400226593, + "learning_rate": 0.0001, + "loss": 65.3721, + "step": 16540 + }, + { + "epoch": 2.672412749036152, + "grad_norm": 0.6131526827812195, + "learning_rate": 0.0001, + "loss": 65.3495, + "step": 16550 + }, + { + "epoch": 2.6740275731212533, + "grad_norm": 0.5448771715164185, + "learning_rate": 0.0001, + "loss": 63.439, + "step": 16560 + }, + { + "epoch": 2.6756423972063543, + "grad_norm": 0.6785935163497925, + "learning_rate": 0.0001, + "loss": 65.5655, + "step": 16570 + }, + { + "epoch": 2.6772572212914554, + "grad_norm": 0.6915177702903748, + "learning_rate": 0.0001, + "loss": 62.5622, + "step": 16580 + }, + { + "epoch": 2.678872045376557, + "grad_norm": 0.5327939987182617, + "learning_rate": 0.0001, + "loss": 62.3226, + "step": 16590 + }, + { + "epoch": 2.680486869461658, + "grad_norm": 0.6650441884994507, + "learning_rate": 0.0001, + "loss": 66.2329, + "step": 16600 + }, + { + "epoch": 2.682101693546759, + "grad_norm": 0.7371799945831299, + "learning_rate": 0.0001, + "loss": 64.45, + "step": 16610 + }, + { + "epoch": 2.6837165176318605, + "grad_norm": 0.6186871528625488, + "learning_rate": 0.0001, + "loss": 64.1364, + "step": 16620 + }, + { + "epoch": 2.685331341716962, + "grad_norm": 0.6977059245109558, + "learning_rate": 0.0001, + "loss": 62.1233, + "step": 16630 + }, + { + "epoch": 2.686946165802063, + "grad_norm": 0.580351710319519, + "learning_rate": 0.0001, + "loss": 64.0906, + "step": 16640 + }, + { + "epoch": 2.688560989887164, + "grad_norm": 0.6118156313896179, + "learning_rate": 0.0001, + "loss": 64.2748, + "step": 16650 + }, + { + "epoch": 2.6901758139722656, + "grad_norm": 0.5368832945823669, + "learning_rate": 0.0001, + "loss": 63.7136, + "step": 16660 + }, + { + "epoch": 2.6917906380573666, + "grad_norm": 0.6193193793296814, + "learning_rate": 0.0001, + "loss": 64.2116, + "step": 16670 + }, + { + "epoch": 2.6934054621424677, + "grad_norm": 0.6088812947273254, + "learning_rate": 0.0001, + "loss": 64.1221, + "step": 16680 + }, + { + "epoch": 2.695020286227569, + "grad_norm": 0.6514917612075806, + "learning_rate": 0.0001, + "loss": 63.764, + "step": 16690 + }, + { + "epoch": 2.6966351103126702, + "grad_norm": 0.5403677821159363, + "learning_rate": 0.0001, + "loss": 64.7421, + "step": 16700 + }, + { + "epoch": 2.6982499343977713, + "grad_norm": 0.7827529311180115, + "learning_rate": 0.0001, + "loss": 63.5076, + "step": 16710 + }, + { + "epoch": 2.6998647584828728, + "grad_norm": 0.6968997716903687, + "learning_rate": 0.0001, + "loss": 64.4448, + "step": 16720 + }, + { + "epoch": 2.7014795825679743, + "grad_norm": 0.6300367116928101, + "learning_rate": 0.0001, + "loss": 64.0245, + "step": 16730 + }, + { + "epoch": 2.7030944066530753, + "grad_norm": 0.6817230582237244, + "learning_rate": 0.0001, + "loss": 64.2338, + "step": 16740 + }, + { + "epoch": 2.7047092307381764, + "grad_norm": 0.6073988080024719, + "learning_rate": 0.0001, + "loss": 64.748, + "step": 16750 + }, + { + "epoch": 2.706324054823278, + "grad_norm": 0.5220157504081726, + "learning_rate": 0.0001, + "loss": 65.8302, + "step": 16760 + }, + { + "epoch": 2.707938878908379, + "grad_norm": 0.536228597164154, + "learning_rate": 0.0001, + "loss": 63.4033, + "step": 16770 + }, + { + "epoch": 2.70955370299348, + "grad_norm": 0.7102165818214417, + "learning_rate": 0.0001, + "loss": 64.862, + "step": 16780 + }, + { + "epoch": 2.7111685270785815, + "grad_norm": 0.6942351460456848, + "learning_rate": 0.0001, + "loss": 63.8927, + "step": 16790 + }, + { + "epoch": 2.7127833511636825, + "grad_norm": 0.7046493887901306, + "learning_rate": 0.0001, + "loss": 63.8942, + "step": 16800 + }, + { + "epoch": 2.7143981752487836, + "grad_norm": 0.5580130815505981, + "learning_rate": 0.0001, + "loss": 64.369, + "step": 16810 + }, + { + "epoch": 2.716012999333885, + "grad_norm": 0.7228151559829712, + "learning_rate": 0.0001, + "loss": 64.4892, + "step": 16820 + }, + { + "epoch": 2.7176278234189866, + "grad_norm": 0.5859993100166321, + "learning_rate": 0.0001, + "loss": 64.895, + "step": 16830 + }, + { + "epoch": 2.7192426475040876, + "grad_norm": 0.5286171436309814, + "learning_rate": 0.0001, + "loss": 64.4283, + "step": 16840 + }, + { + "epoch": 2.7208574715891887, + "grad_norm": 0.5749659538269043, + "learning_rate": 0.0001, + "loss": 65.7189, + "step": 16850 + }, + { + "epoch": 2.72247229567429, + "grad_norm": 0.8945223689079285, + "learning_rate": 0.0001, + "loss": 64.7374, + "step": 16860 + }, + { + "epoch": 2.724087119759391, + "grad_norm": 0.5596129298210144, + "learning_rate": 0.0001, + "loss": 64.7539, + "step": 16870 + }, + { + "epoch": 2.7257019438444923, + "grad_norm": 0.6479132771492004, + "learning_rate": 0.0001, + "loss": 63.6447, + "step": 16880 + }, + { + "epoch": 2.7273167679295938, + "grad_norm": 0.8423380255699158, + "learning_rate": 0.0001, + "loss": 63.7605, + "step": 16890 + }, + { + "epoch": 2.728931592014695, + "grad_norm": 1.0722684860229492, + "learning_rate": 0.0001, + "loss": 65.054, + "step": 16900 + }, + { + "epoch": 2.730546416099796, + "grad_norm": 0.5196233987808228, + "learning_rate": 0.0001, + "loss": 64.7511, + "step": 16910 + }, + { + "epoch": 2.7321612401848974, + "grad_norm": 0.7080458998680115, + "learning_rate": 0.0001, + "loss": 65.1224, + "step": 16920 + }, + { + "epoch": 2.733776064269999, + "grad_norm": 0.5816940665245056, + "learning_rate": 0.0001, + "loss": 64.8532, + "step": 16930 + }, + { + "epoch": 2.7353908883551, + "grad_norm": 0.5616808533668518, + "learning_rate": 0.0001, + "loss": 62.4221, + "step": 16940 + }, + { + "epoch": 2.737005712440201, + "grad_norm": 0.4750169515609741, + "learning_rate": 0.0001, + "loss": 60.3851, + "step": 16950 + }, + { + "epoch": 2.7386205365253025, + "grad_norm": 0.5957995653152466, + "learning_rate": 0.0001, + "loss": 62.907, + "step": 16960 + }, + { + "epoch": 2.7402353606104035, + "grad_norm": 0.60072261095047, + "learning_rate": 0.0001, + "loss": 66.446, + "step": 16970 + }, + { + "epoch": 2.7418501846955046, + "grad_norm": 0.5808581113815308, + "learning_rate": 0.0001, + "loss": 62.8852, + "step": 16980 + }, + { + "epoch": 2.743465008780606, + "grad_norm": 0.5243962407112122, + "learning_rate": 0.0001, + "loss": 64.3737, + "step": 16990 + }, + { + "epoch": 2.745079832865707, + "grad_norm": 0.6608695387840271, + "learning_rate": 0.0001, + "loss": 65.9526, + "step": 17000 + }, + { + "epoch": 2.746694656950808, + "grad_norm": 0.5109400153160095, + "learning_rate": 0.0001, + "loss": 63.5495, + "step": 17010 + }, + { + "epoch": 2.7483094810359097, + "grad_norm": 0.5007188320159912, + "learning_rate": 0.0001, + "loss": 64.3408, + "step": 17020 + }, + { + "epoch": 2.7499243051210107, + "grad_norm": 0.7058080434799194, + "learning_rate": 0.0001, + "loss": 63.1929, + "step": 17030 + }, + { + "epoch": 2.751539129206112, + "grad_norm": 0.5310391783714294, + "learning_rate": 0.0001, + "loss": 64.9107, + "step": 17040 + }, + { + "epoch": 2.7531539532912133, + "grad_norm": 0.5303440093994141, + "learning_rate": 0.0001, + "loss": 64.9961, + "step": 17050 + }, + { + "epoch": 2.7547687773763148, + "grad_norm": 0.5952566266059875, + "learning_rate": 0.0001, + "loss": 64.7917, + "step": 17060 + }, + { + "epoch": 2.756383601461416, + "grad_norm": 0.5863012075424194, + "learning_rate": 0.0001, + "loss": 65.2902, + "step": 17070 + }, + { + "epoch": 2.757998425546517, + "grad_norm": 0.8943840265274048, + "learning_rate": 0.0001, + "loss": 64.9076, + "step": 17080 + }, + { + "epoch": 2.7596132496316184, + "grad_norm": 0.7045338153839111, + "learning_rate": 0.0001, + "loss": 63.2516, + "step": 17090 + }, + { + "epoch": 2.7612280737167194, + "grad_norm": 0.8543638586997986, + "learning_rate": 0.0001, + "loss": 64.5437, + "step": 17100 + }, + { + "epoch": 2.7628428978018205, + "grad_norm": 0.6041001081466675, + "learning_rate": 0.0001, + "loss": 64.6194, + "step": 17110 + }, + { + "epoch": 2.764457721886922, + "grad_norm": 0.7917356491088867, + "learning_rate": 0.0001, + "loss": 64.5536, + "step": 17120 + }, + { + "epoch": 2.766072545972023, + "grad_norm": 0.6502277851104736, + "learning_rate": 0.0001, + "loss": 64.603, + "step": 17130 + }, + { + "epoch": 2.7676873700571245, + "grad_norm": 0.6309056282043457, + "learning_rate": 0.0001, + "loss": 64.757, + "step": 17140 + }, + { + "epoch": 2.7693021941422256, + "grad_norm": 0.6861969828605652, + "learning_rate": 0.0001, + "loss": 64.3539, + "step": 17150 + }, + { + "epoch": 2.770917018227327, + "grad_norm": 0.7183892130851746, + "learning_rate": 0.0001, + "loss": 63.9602, + "step": 17160 + }, + { + "epoch": 2.772531842312428, + "grad_norm": 0.5584074854850769, + "learning_rate": 0.0001, + "loss": 63.5869, + "step": 17170 + }, + { + "epoch": 2.774146666397529, + "grad_norm": 0.6466682553291321, + "learning_rate": 0.0001, + "loss": 62.8788, + "step": 17180 + }, + { + "epoch": 2.7757614904826307, + "grad_norm": 0.5255836844444275, + "learning_rate": 0.0001, + "loss": 62.0409, + "step": 17190 + }, + { + "epoch": 2.7773763145677317, + "grad_norm": 0.5032522678375244, + "learning_rate": 0.0001, + "loss": 64.8646, + "step": 17200 + }, + { + "epoch": 2.7789911386528328, + "grad_norm": 0.6183506846427917, + "learning_rate": 0.0001, + "loss": 63.2399, + "step": 17210 + }, + { + "epoch": 2.7806059627379343, + "grad_norm": 0.788912296295166, + "learning_rate": 0.0001, + "loss": 63.245, + "step": 17220 + }, + { + "epoch": 2.7822207868230353, + "grad_norm": 0.6142173409461975, + "learning_rate": 0.0001, + "loss": 65.9609, + "step": 17230 + }, + { + "epoch": 2.783835610908137, + "grad_norm": 0.5913170576095581, + "learning_rate": 0.0001, + "loss": 62.1876, + "step": 17240 + }, + { + "epoch": 2.785450434993238, + "grad_norm": 0.6448681354522705, + "learning_rate": 0.0001, + "loss": 63.1455, + "step": 17250 + }, + { + "epoch": 2.7870652590783394, + "grad_norm": 0.5871738195419312, + "learning_rate": 0.0001, + "loss": 63.6158, + "step": 17260 + }, + { + "epoch": 2.7886800831634404, + "grad_norm": 0.5945441722869873, + "learning_rate": 0.0001, + "loss": 63.7733, + "step": 17270 + }, + { + "epoch": 2.7902949072485415, + "grad_norm": 0.5735594630241394, + "learning_rate": 0.0001, + "loss": 64.6803, + "step": 17280 + }, + { + "epoch": 2.791909731333643, + "grad_norm": 0.6030424237251282, + "learning_rate": 0.0001, + "loss": 64.5242, + "step": 17290 + }, + { + "epoch": 2.793524555418744, + "grad_norm": 0.6867285966873169, + "learning_rate": 0.0001, + "loss": 64.8689, + "step": 17300 + }, + { + "epoch": 2.795139379503845, + "grad_norm": 0.5623561143875122, + "learning_rate": 0.0001, + "loss": 63.6502, + "step": 17310 + }, + { + "epoch": 2.7967542035889466, + "grad_norm": 0.6703289151191711, + "learning_rate": 0.0001, + "loss": 63.3221, + "step": 17320 + }, + { + "epoch": 2.7983690276740476, + "grad_norm": 0.6562789082527161, + "learning_rate": 0.0001, + "loss": 63.9181, + "step": 17330 + }, + { + "epoch": 2.799983851759149, + "grad_norm": 0.729101836681366, + "learning_rate": 0.0001, + "loss": 65.6384, + "step": 17340 + }, + { + "epoch": 2.80159867584425, + "grad_norm": 0.6310204267501831, + "learning_rate": 0.0001, + "loss": 63.5784, + "step": 17350 + }, + { + "epoch": 2.8032134999293516, + "grad_norm": 0.6090773344039917, + "learning_rate": 0.0001, + "loss": 64.3578, + "step": 17360 + }, + { + "epoch": 2.8048283240144527, + "grad_norm": 0.6332791447639465, + "learning_rate": 0.0001, + "loss": 66.4391, + "step": 17370 + }, + { + "epoch": 2.8064431480995538, + "grad_norm": 0.6627421379089355, + "learning_rate": 0.0001, + "loss": 65.8978, + "step": 17380 + }, + { + "epoch": 2.8080579721846552, + "grad_norm": 0.4875560402870178, + "learning_rate": 0.0001, + "loss": 64.1927, + "step": 17390 + }, + { + "epoch": 2.8096727962697563, + "grad_norm": 0.5367199778556824, + "learning_rate": 0.0001, + "loss": 65.9923, + "step": 17400 + }, + { + "epoch": 2.8112876203548574, + "grad_norm": 0.7587525844573975, + "learning_rate": 0.0001, + "loss": 64.5313, + "step": 17410 + }, + { + "epoch": 2.812902444439959, + "grad_norm": 0.5619229674339294, + "learning_rate": 0.0001, + "loss": 65.6483, + "step": 17420 + }, + { + "epoch": 2.81451726852506, + "grad_norm": 0.6196359992027283, + "learning_rate": 0.0001, + "loss": 63.1428, + "step": 17430 + }, + { + "epoch": 2.8161320926101614, + "grad_norm": 0.7516114115715027, + "learning_rate": 0.0001, + "loss": 63.6484, + "step": 17440 + }, + { + "epoch": 2.8177469166952624, + "grad_norm": 0.5730687379837036, + "learning_rate": 0.0001, + "loss": 63.0208, + "step": 17450 + }, + { + "epoch": 2.819361740780364, + "grad_norm": 0.5426228046417236, + "learning_rate": 0.0001, + "loss": 64.9543, + "step": 17460 + }, + { + "epoch": 2.820976564865465, + "grad_norm": 0.6501995921134949, + "learning_rate": 0.0001, + "loss": 64.8474, + "step": 17470 + }, + { + "epoch": 2.822591388950566, + "grad_norm": 0.7911072373390198, + "learning_rate": 0.0001, + "loss": 64.6111, + "step": 17480 + }, + { + "epoch": 2.8242062130356675, + "grad_norm": 0.5159533023834229, + "learning_rate": 0.0001, + "loss": 64.7333, + "step": 17490 + }, + { + "epoch": 2.8258210371207686, + "grad_norm": 0.6600466966629028, + "learning_rate": 0.0001, + "loss": 63.9058, + "step": 17500 + }, + { + "epoch": 2.8274358612058696, + "grad_norm": 0.9110124707221985, + "learning_rate": 0.0001, + "loss": 62.7915, + "step": 17510 + }, + { + "epoch": 2.829050685290971, + "grad_norm": 0.6339550018310547, + "learning_rate": 0.0001, + "loss": 62.6357, + "step": 17520 + }, + { + "epoch": 2.830665509376072, + "grad_norm": 0.5881263613700867, + "learning_rate": 0.0001, + "loss": 65.1601, + "step": 17530 + }, + { + "epoch": 2.8322803334611737, + "grad_norm": 0.5641292333602905, + "learning_rate": 0.0001, + "loss": 64.0657, + "step": 17540 + }, + { + "epoch": 2.8338951575462747, + "grad_norm": 0.5120996236801147, + "learning_rate": 0.0001, + "loss": 65.4236, + "step": 17550 + }, + { + "epoch": 2.8355099816313762, + "grad_norm": 0.5632758736610413, + "learning_rate": 0.0001, + "loss": 63.6252, + "step": 17560 + }, + { + "epoch": 2.8371248057164773, + "grad_norm": 0.507538378238678, + "learning_rate": 0.0001, + "loss": 63.9376, + "step": 17570 + }, + { + "epoch": 2.8387396298015783, + "grad_norm": 0.6714527010917664, + "learning_rate": 0.0001, + "loss": 63.7478, + "step": 17580 + }, + { + "epoch": 2.84035445388668, + "grad_norm": 0.5168405175209045, + "learning_rate": 0.0001, + "loss": 63.2652, + "step": 17590 + }, + { + "epoch": 2.841969277971781, + "grad_norm": 0.597294270992279, + "learning_rate": 0.0001, + "loss": 61.7012, + "step": 17600 + }, + { + "epoch": 2.843584102056882, + "grad_norm": 0.5981683135032654, + "learning_rate": 0.0001, + "loss": 63.5361, + "step": 17610 + }, + { + "epoch": 2.8451989261419834, + "grad_norm": 1.3800987005233765, + "learning_rate": 0.0001, + "loss": 64.9525, + "step": 17620 + }, + { + "epoch": 2.8468137502270845, + "grad_norm": 0.7254458069801331, + "learning_rate": 0.0001, + "loss": 65.9079, + "step": 17630 + }, + { + "epoch": 2.848428574312186, + "grad_norm": 0.7237294316291809, + "learning_rate": 0.0001, + "loss": 63.8467, + "step": 17640 + }, + { + "epoch": 2.850043398397287, + "grad_norm": 0.6137713193893433, + "learning_rate": 0.0001, + "loss": 64.088, + "step": 17650 + }, + { + "epoch": 2.8516582224823885, + "grad_norm": 0.5131928324699402, + "learning_rate": 0.0001, + "loss": 66.4916, + "step": 17660 + }, + { + "epoch": 2.8532730465674896, + "grad_norm": 0.6695879697799683, + "learning_rate": 0.0001, + "loss": 64.9182, + "step": 17670 + }, + { + "epoch": 2.8548878706525906, + "grad_norm": 0.9055695533752441, + "learning_rate": 0.0001, + "loss": 64.84, + "step": 17680 + }, + { + "epoch": 2.856502694737692, + "grad_norm": 0.9126661419868469, + "learning_rate": 0.0001, + "loss": 63.657, + "step": 17690 + }, + { + "epoch": 2.858117518822793, + "grad_norm": 0.8321142792701721, + "learning_rate": 0.0001, + "loss": 62.86, + "step": 17700 + }, + { + "epoch": 2.8597323429078942, + "grad_norm": 0.6571202278137207, + "learning_rate": 0.0001, + "loss": 66.1002, + "step": 17710 + }, + { + "epoch": 2.8613471669929957, + "grad_norm": 0.7111368775367737, + "learning_rate": 0.0001, + "loss": 64.3513, + "step": 17720 + }, + { + "epoch": 2.862961991078097, + "grad_norm": 0.5026522874832153, + "learning_rate": 0.0001, + "loss": 61.8481, + "step": 17730 + }, + { + "epoch": 2.8645768151631983, + "grad_norm": 0.5820951461791992, + "learning_rate": 0.0001, + "loss": 65.2891, + "step": 17740 + }, + { + "epoch": 2.8661916392482993, + "grad_norm": 0.5339917540550232, + "learning_rate": 0.0001, + "loss": 66.4793, + "step": 17750 + }, + { + "epoch": 2.867806463333401, + "grad_norm": 0.7611919045448303, + "learning_rate": 0.0001, + "loss": 63.3336, + "step": 17760 + }, + { + "epoch": 2.869421287418502, + "grad_norm": 0.7337769865989685, + "learning_rate": 0.0001, + "loss": 64.0139, + "step": 17770 + }, + { + "epoch": 2.871036111503603, + "grad_norm": 0.631750226020813, + "learning_rate": 0.0001, + "loss": 64.7222, + "step": 17780 + }, + { + "epoch": 2.8726509355887044, + "grad_norm": 0.6601754426956177, + "learning_rate": 0.0001, + "loss": 63.5139, + "step": 17790 + }, + { + "epoch": 2.8742657596738055, + "grad_norm": 0.567677915096283, + "learning_rate": 0.0001, + "loss": 64.9689, + "step": 17800 + }, + { + "epoch": 2.8758805837589065, + "grad_norm": 0.6223359704017639, + "learning_rate": 0.0001, + "loss": 64.0291, + "step": 17810 + }, + { + "epoch": 2.877495407844008, + "grad_norm": 0.5666190981864929, + "learning_rate": 0.0001, + "loss": 65.1292, + "step": 17820 + }, + { + "epoch": 2.879110231929109, + "grad_norm": 0.5870898365974426, + "learning_rate": 0.0001, + "loss": 62.9442, + "step": 17830 + }, + { + "epoch": 2.8807250560142106, + "grad_norm": 0.7579465508460999, + "learning_rate": 0.0001, + "loss": 63.997, + "step": 17840 + }, + { + "epoch": 2.8823398800993116, + "grad_norm": 0.674678385257721, + "learning_rate": 0.0001, + "loss": 62.7068, + "step": 17850 + }, + { + "epoch": 2.883954704184413, + "grad_norm": 0.7183076739311218, + "learning_rate": 0.0001, + "loss": 63.2744, + "step": 17860 + }, + { + "epoch": 2.885569528269514, + "grad_norm": 0.8441048264503479, + "learning_rate": 0.0001, + "loss": 64.587, + "step": 17870 + }, + { + "epoch": 2.8871843523546152, + "grad_norm": 0.5780750513076782, + "learning_rate": 0.0001, + "loss": 63.2929, + "step": 17880 + }, + { + "epoch": 2.8887991764397167, + "grad_norm": 0.5931091904640198, + "learning_rate": 0.0001, + "loss": 63.0084, + "step": 17890 + }, + { + "epoch": 2.890414000524818, + "grad_norm": 0.5232470631599426, + "learning_rate": 0.0001, + "loss": 64.4947, + "step": 17900 + }, + { + "epoch": 2.892028824609919, + "grad_norm": 0.5742111206054688, + "learning_rate": 0.0001, + "loss": 63.544, + "step": 17910 + }, + { + "epoch": 2.8936436486950203, + "grad_norm": 0.531182587146759, + "learning_rate": 0.0001, + "loss": 64.5583, + "step": 17920 + }, + { + "epoch": 2.8952584727801214, + "grad_norm": 0.5353103876113892, + "learning_rate": 0.0001, + "loss": 61.9807, + "step": 17930 + }, + { + "epoch": 2.896873296865223, + "grad_norm": 0.53264981508255, + "learning_rate": 0.0001, + "loss": 62.3831, + "step": 17940 + }, + { + "epoch": 2.898488120950324, + "grad_norm": 0.5475999712944031, + "learning_rate": 0.0001, + "loss": 64.9383, + "step": 17950 + }, + { + "epoch": 2.9001029450354254, + "grad_norm": 0.876865804195404, + "learning_rate": 0.0001, + "loss": 64.955, + "step": 17960 + }, + { + "epoch": 2.9017177691205265, + "grad_norm": 0.5121579170227051, + "learning_rate": 0.0001, + "loss": 62.8306, + "step": 17970 + }, + { + "epoch": 2.9033325932056275, + "grad_norm": 0.6591141819953918, + "learning_rate": 0.0001, + "loss": 61.6768, + "step": 17980 + }, + { + "epoch": 2.904947417290729, + "grad_norm": 0.5192371010780334, + "learning_rate": 0.0001, + "loss": 63.5609, + "step": 17990 + }, + { + "epoch": 2.90656224137583, + "grad_norm": 0.5635988116264343, + "learning_rate": 0.0001, + "loss": 61.4911, + "step": 18000 + }, + { + "epoch": 2.908177065460931, + "grad_norm": 0.6950483322143555, + "learning_rate": 0.0001, + "loss": 65.1385, + "step": 18010 + }, + { + "epoch": 2.9097918895460326, + "grad_norm": 0.5985620021820068, + "learning_rate": 0.0001, + "loss": 63.0768, + "step": 18020 + }, + { + "epoch": 2.9114067136311337, + "grad_norm": 0.650201141834259, + "learning_rate": 0.0001, + "loss": 63.7908, + "step": 18030 + }, + { + "epoch": 2.913021537716235, + "grad_norm": 1.6501675844192505, + "learning_rate": 0.0001, + "loss": 63.8961, + "step": 18040 + }, + { + "epoch": 2.9146363618013362, + "grad_norm": 0.6359374523162842, + "learning_rate": 0.0001, + "loss": 64.4169, + "step": 18050 + }, + { + "epoch": 2.9162511858864377, + "grad_norm": 0.578163743019104, + "learning_rate": 0.0001, + "loss": 64.1323, + "step": 18060 + }, + { + "epoch": 2.9178660099715388, + "grad_norm": 0.6043027639389038, + "learning_rate": 0.0001, + "loss": 65.7538, + "step": 18070 + }, + { + "epoch": 2.91948083405664, + "grad_norm": 0.6983678936958313, + "learning_rate": 0.0001, + "loss": 64.5066, + "step": 18080 + }, + { + "epoch": 2.9210956581417413, + "grad_norm": 0.7991185784339905, + "learning_rate": 0.0001, + "loss": 63.6101, + "step": 18090 + }, + { + "epoch": 2.9227104822268424, + "grad_norm": 0.6259360909461975, + "learning_rate": 0.0001, + "loss": 64.3639, + "step": 18100 + }, + { + "epoch": 2.9243253063119434, + "grad_norm": 0.6313489675521851, + "learning_rate": 0.0001, + "loss": 64.6103, + "step": 18110 + }, + { + "epoch": 2.925940130397045, + "grad_norm": 0.5663475394248962, + "learning_rate": 0.0001, + "loss": 63.626, + "step": 18120 + }, + { + "epoch": 2.927554954482146, + "grad_norm": 0.5612998008728027, + "learning_rate": 0.0001, + "loss": 63.1078, + "step": 18130 + }, + { + "epoch": 2.9291697785672475, + "grad_norm": 0.6059394478797913, + "learning_rate": 0.0001, + "loss": 64.2389, + "step": 18140 + }, + { + "epoch": 2.9307846026523485, + "grad_norm": 0.5778524279594421, + "learning_rate": 0.0001, + "loss": 62.6661, + "step": 18150 + }, + { + "epoch": 2.93239942673745, + "grad_norm": 0.6518381834030151, + "learning_rate": 0.0001, + "loss": 64.6579, + "step": 18160 + }, + { + "epoch": 2.934014250822551, + "grad_norm": 0.6941561102867126, + "learning_rate": 0.0001, + "loss": 63.4389, + "step": 18170 + }, + { + "epoch": 2.935629074907652, + "grad_norm": 0.5611969828605652, + "learning_rate": 0.0001, + "loss": 64.0737, + "step": 18180 + }, + { + "epoch": 2.9372438989927536, + "grad_norm": 0.6152001619338989, + "learning_rate": 0.0001, + "loss": 65.4391, + "step": 18190 + }, + { + "epoch": 2.9388587230778547, + "grad_norm": 0.5341830253601074, + "learning_rate": 0.0001, + "loss": 61.6958, + "step": 18200 + }, + { + "epoch": 2.9404735471629557, + "grad_norm": 0.5216324329376221, + "learning_rate": 0.0001, + "loss": 62.1087, + "step": 18210 + }, + { + "epoch": 2.942088371248057, + "grad_norm": 0.5688874125480652, + "learning_rate": 0.0001, + "loss": 63.5409, + "step": 18220 + }, + { + "epoch": 2.9437031953331583, + "grad_norm": 0.6242631673812866, + "learning_rate": 0.0001, + "loss": 63.28, + "step": 18230 + }, + { + "epoch": 2.9453180194182598, + "grad_norm": 0.5243523716926575, + "learning_rate": 0.0001, + "loss": 62.0036, + "step": 18240 + }, + { + "epoch": 2.946932843503361, + "grad_norm": 0.5922643542289734, + "learning_rate": 0.0001, + "loss": 64.2824, + "step": 18250 + }, + { + "epoch": 2.9485476675884623, + "grad_norm": 0.5020294189453125, + "learning_rate": 0.0001, + "loss": 63.3895, + "step": 18260 + }, + { + "epoch": 2.9501624916735634, + "grad_norm": 0.6435392498970032, + "learning_rate": 0.0001, + "loss": 63.596, + "step": 18270 + }, + { + "epoch": 2.9517773157586644, + "grad_norm": 0.5490348935127258, + "learning_rate": 0.0001, + "loss": 63.4475, + "step": 18280 + }, + { + "epoch": 2.953392139843766, + "grad_norm": 0.7996026873588562, + "learning_rate": 0.0001, + "loss": 63.1103, + "step": 18290 + }, + { + "epoch": 2.955006963928867, + "grad_norm": 0.5791653990745544, + "learning_rate": 0.0001, + "loss": 64.7261, + "step": 18300 + }, + { + "epoch": 2.956621788013968, + "grad_norm": 0.7814281582832336, + "learning_rate": 0.0001, + "loss": 63.0434, + "step": 18310 + }, + { + "epoch": 2.9582366120990695, + "grad_norm": 0.6304134130477905, + "learning_rate": 0.0001, + "loss": 62.331, + "step": 18320 + }, + { + "epoch": 2.9598514361841706, + "grad_norm": 0.6132511496543884, + "learning_rate": 0.0001, + "loss": 64.5359, + "step": 18330 + }, + { + "epoch": 2.961466260269272, + "grad_norm": 0.5530602335929871, + "learning_rate": 0.0001, + "loss": 63.5784, + "step": 18340 + }, + { + "epoch": 2.963081084354373, + "grad_norm": 0.5595644116401672, + "learning_rate": 0.0001, + "loss": 65.5288, + "step": 18350 + }, + { + "epoch": 2.9646959084394746, + "grad_norm": 0.6270143985748291, + "learning_rate": 0.0001, + "loss": 62.7798, + "step": 18360 + }, + { + "epoch": 2.9663107325245757, + "grad_norm": 0.5871894359588623, + "learning_rate": 0.0001, + "loss": 65.2275, + "step": 18370 + }, + { + "epoch": 2.9679255566096767, + "grad_norm": 0.5413428544998169, + "learning_rate": 0.0001, + "loss": 64.3245, + "step": 18380 + }, + { + "epoch": 2.969540380694778, + "grad_norm": 0.6115475296974182, + "learning_rate": 0.0001, + "loss": 62.7083, + "step": 18390 + }, + { + "epoch": 2.9711552047798793, + "grad_norm": 0.5501357913017273, + "learning_rate": 0.0001, + "loss": 65.2912, + "step": 18400 + }, + { + "epoch": 2.9727700288649803, + "grad_norm": 0.8732792735099792, + "learning_rate": 0.0001, + "loss": 63.4837, + "step": 18410 + }, + { + "epoch": 2.974384852950082, + "grad_norm": 0.693257749080658, + "learning_rate": 0.0001, + "loss": 62.1404, + "step": 18420 + }, + { + "epoch": 2.975999677035183, + "grad_norm": 0.615731954574585, + "learning_rate": 0.0001, + "loss": 65.1973, + "step": 18430 + }, + { + "epoch": 2.9776145011202844, + "grad_norm": 0.6412269473075867, + "learning_rate": 0.0001, + "loss": 64.523, + "step": 18440 + }, + { + "epoch": 2.9792293252053854, + "grad_norm": 0.5384656190872192, + "learning_rate": 0.0001, + "loss": 64.9669, + "step": 18450 + }, + { + "epoch": 2.980844149290487, + "grad_norm": 0.5612038969993591, + "learning_rate": 0.0001, + "loss": 63.7712, + "step": 18460 + }, + { + "epoch": 2.982458973375588, + "grad_norm": 0.481273889541626, + "learning_rate": 0.0001, + "loss": 64.3953, + "step": 18470 + }, + { + "epoch": 2.984073797460689, + "grad_norm": 0.6603255867958069, + "learning_rate": 0.0001, + "loss": 62.7127, + "step": 18480 + }, + { + "epoch": 2.9856886215457905, + "grad_norm": 0.590406596660614, + "learning_rate": 0.0001, + "loss": 65.0984, + "step": 18490 + }, + { + "epoch": 2.9873034456308916, + "grad_norm": 0.4916887879371643, + "learning_rate": 0.0001, + "loss": 63.5379, + "step": 18500 + }, + { + "epoch": 2.9889182697159926, + "grad_norm": 0.6201387643814087, + "learning_rate": 0.0001, + "loss": 62.5165, + "step": 18510 + }, + { + "epoch": 2.990533093801094, + "grad_norm": 0.514717161655426, + "learning_rate": 0.0001, + "loss": 63.6199, + "step": 18520 + }, + { + "epoch": 2.992147917886195, + "grad_norm": 0.9358267188072205, + "learning_rate": 0.0001, + "loss": 62.7973, + "step": 18530 + }, + { + "epoch": 2.9937627419712967, + "grad_norm": 0.592309296131134, + "learning_rate": 0.0001, + "loss": 63.9463, + "step": 18540 + }, + { + "epoch": 2.9953775660563977, + "grad_norm": 0.6352376937866211, + "learning_rate": 0.0001, + "loss": 64.2489, + "step": 18550 + }, + { + "epoch": 2.996992390141499, + "grad_norm": 0.6198953986167908, + "learning_rate": 0.0001, + "loss": 64.4858, + "step": 18560 + }, + { + "epoch": 2.9986072142266003, + "grad_norm": 0.7788981795310974, + "learning_rate": 0.0001, + "loss": 64.3219, + "step": 18570 + }, + { + "epoch": 3.0001614824085103, + "grad_norm": 0.7411819100379944, + "learning_rate": 0.0001, + "loss": 60.7997, + "step": 18580 + }, + { + "epoch": 3.0017763064936114, + "grad_norm": 0.5716884732246399, + "learning_rate": 0.0001, + "loss": 64.2044, + "step": 18590 + }, + { + "epoch": 3.0033911305787124, + "grad_norm": 0.6183570623397827, + "learning_rate": 0.0001, + "loss": 62.9673, + "step": 18600 + }, + { + "epoch": 3.005005954663814, + "grad_norm": 0.8126595616340637, + "learning_rate": 0.0001, + "loss": 64.4, + "step": 18610 + }, + { + "epoch": 3.006620778748915, + "grad_norm": 0.5606074333190918, + "learning_rate": 0.0001, + "loss": 63.5389, + "step": 18620 + }, + { + "epoch": 3.0082356028340165, + "grad_norm": 0.5653588771820068, + "learning_rate": 0.0001, + "loss": 64.0225, + "step": 18630 + }, + { + "epoch": 3.0098504269191175, + "grad_norm": 0.6049582362174988, + "learning_rate": 0.0001, + "loss": 62.0674, + "step": 18640 + }, + { + "epoch": 3.0114652510042186, + "grad_norm": 0.5483401417732239, + "learning_rate": 0.0001, + "loss": 62.3744, + "step": 18650 + }, + { + "epoch": 3.01308007508932, + "grad_norm": 1.7736371755599976, + "learning_rate": 0.0001, + "loss": 62.8774, + "step": 18660 + }, + { + "epoch": 3.014694899174421, + "grad_norm": 0.5802493095397949, + "learning_rate": 0.0001, + "loss": 63.8731, + "step": 18670 + }, + { + "epoch": 3.0163097232595226, + "grad_norm": 0.636654794216156, + "learning_rate": 0.0001, + "loss": 64.4445, + "step": 18680 + }, + { + "epoch": 3.0179245473446237, + "grad_norm": 0.51685631275177, + "learning_rate": 0.0001, + "loss": 63.8634, + "step": 18690 + }, + { + "epoch": 3.0195393714297247, + "grad_norm": 0.5670599341392517, + "learning_rate": 0.0001, + "loss": 63.9522, + "step": 18700 + }, + { + "epoch": 3.0211541955148262, + "grad_norm": 0.5583446621894836, + "learning_rate": 0.0001, + "loss": 64.2944, + "step": 18710 + }, + { + "epoch": 3.0227690195999273, + "grad_norm": 0.6011738777160645, + "learning_rate": 0.0001, + "loss": 65.7547, + "step": 18720 + }, + { + "epoch": 3.0243838436850288, + "grad_norm": 0.7925758957862854, + "learning_rate": 0.0001, + "loss": 63.2283, + "step": 18730 + }, + { + "epoch": 3.02599866777013, + "grad_norm": 0.4886399507522583, + "learning_rate": 0.0001, + "loss": 62.4337, + "step": 18740 + }, + { + "epoch": 3.027613491855231, + "grad_norm": 0.5633055567741394, + "learning_rate": 0.0001, + "loss": 64.3412, + "step": 18750 + }, + { + "epoch": 3.0292283159403324, + "grad_norm": 0.5860275626182556, + "learning_rate": 0.0001, + "loss": 65.0562, + "step": 18760 + }, + { + "epoch": 3.0308431400254334, + "grad_norm": 0.6726853251457214, + "learning_rate": 0.0001, + "loss": 62.7725, + "step": 18770 + }, + { + "epoch": 3.032457964110535, + "grad_norm": 0.5257040858268738, + "learning_rate": 0.0001, + "loss": 62.6034, + "step": 18780 + }, + { + "epoch": 3.034072788195636, + "grad_norm": 0.6232852339744568, + "learning_rate": 0.0001, + "loss": 65.4976, + "step": 18790 + }, + { + "epoch": 3.035687612280737, + "grad_norm": 0.45132556557655334, + "learning_rate": 0.0001, + "loss": 64.2146, + "step": 18800 + }, + { + "epoch": 3.0373024363658385, + "grad_norm": 0.5963318943977356, + "learning_rate": 0.0001, + "loss": 62.4738, + "step": 18810 + }, + { + "epoch": 3.0389172604509396, + "grad_norm": 0.5632551312446594, + "learning_rate": 0.0001, + "loss": 64.2321, + "step": 18820 + }, + { + "epoch": 3.040532084536041, + "grad_norm": 0.517217218875885, + "learning_rate": 0.0001, + "loss": 62.3571, + "step": 18830 + }, + { + "epoch": 3.042146908621142, + "grad_norm": 0.5927183628082275, + "learning_rate": 0.0001, + "loss": 62.337, + "step": 18840 + }, + { + "epoch": 3.043761732706243, + "grad_norm": 0.5111663341522217, + "learning_rate": 0.0001, + "loss": 64.1993, + "step": 18850 + }, + { + "epoch": 3.0453765567913447, + "grad_norm": 0.46675893664360046, + "learning_rate": 0.0001, + "loss": 62.4584, + "step": 18860 + }, + { + "epoch": 3.0469913808764457, + "grad_norm": 0.5513922572135925, + "learning_rate": 0.0001, + "loss": 61.1837, + "step": 18870 + }, + { + "epoch": 3.048606204961547, + "grad_norm": 0.6328930258750916, + "learning_rate": 0.0001, + "loss": 62.5237, + "step": 18880 + }, + { + "epoch": 3.0502210290466483, + "grad_norm": 0.49861156940460205, + "learning_rate": 0.0001, + "loss": 64.9669, + "step": 18890 + }, + { + "epoch": 3.0518358531317493, + "grad_norm": 0.5221716165542603, + "learning_rate": 0.0001, + "loss": 61.8127, + "step": 18900 + }, + { + "epoch": 3.053450677216851, + "grad_norm": 0.5635599493980408, + "learning_rate": 0.0001, + "loss": 62.917, + "step": 18910 + }, + { + "epoch": 3.055065501301952, + "grad_norm": 0.5035967230796814, + "learning_rate": 0.0001, + "loss": 63.3813, + "step": 18920 + }, + { + "epoch": 3.0566803253870534, + "grad_norm": 0.7660979628562927, + "learning_rate": 0.0001, + "loss": 63.0427, + "step": 18930 + }, + { + "epoch": 3.0582951494721544, + "grad_norm": 0.5173986554145813, + "learning_rate": 0.0001, + "loss": 63.9719, + "step": 18940 + }, + { + "epoch": 3.0599099735572555, + "grad_norm": 0.5582230687141418, + "learning_rate": 0.0001, + "loss": 63.1522, + "step": 18950 + }, + { + "epoch": 3.061524797642357, + "grad_norm": 0.5525544881820679, + "learning_rate": 0.0001, + "loss": 63.3151, + "step": 18960 + }, + { + "epoch": 3.063139621727458, + "grad_norm": 0.5501163005828857, + "learning_rate": 0.0001, + "loss": 64.3584, + "step": 18970 + }, + { + "epoch": 3.064754445812559, + "grad_norm": 0.7555058598518372, + "learning_rate": 0.0001, + "loss": 63.8739, + "step": 18980 + }, + { + "epoch": 3.0663692698976606, + "grad_norm": 0.6666732430458069, + "learning_rate": 0.0001, + "loss": 62.8489, + "step": 18990 + }, + { + "epoch": 3.0679840939827616, + "grad_norm": 0.7918298840522766, + "learning_rate": 0.0001, + "loss": 64.5336, + "step": 19000 + }, + { + "epoch": 3.069598918067863, + "grad_norm": 0.5280627608299255, + "learning_rate": 0.0001, + "loss": 64.5066, + "step": 19010 + }, + { + "epoch": 3.071213742152964, + "grad_norm": 0.5357632040977478, + "learning_rate": 0.0001, + "loss": 63.0134, + "step": 19020 + }, + { + "epoch": 3.0728285662380657, + "grad_norm": 0.5770837068557739, + "learning_rate": 0.0001, + "loss": 63.3923, + "step": 19030 + }, + { + "epoch": 3.0744433903231667, + "grad_norm": 0.5545732378959656, + "learning_rate": 0.0001, + "loss": 62.2578, + "step": 19040 + }, + { + "epoch": 3.0760582144082678, + "grad_norm": 0.5522381067276001, + "learning_rate": 0.0001, + "loss": 65.1748, + "step": 19050 + }, + { + "epoch": 3.0776730384933693, + "grad_norm": 0.6130033135414124, + "learning_rate": 0.0001, + "loss": 61.7126, + "step": 19060 + }, + { + "epoch": 3.0792878625784703, + "grad_norm": 0.6037668585777283, + "learning_rate": 0.0001, + "loss": 64.2351, + "step": 19070 + }, + { + "epoch": 3.0809026866635714, + "grad_norm": 0.6568130254745483, + "learning_rate": 0.0001, + "loss": 63.9457, + "step": 19080 + }, + { + "epoch": 3.082517510748673, + "grad_norm": 0.5877203345298767, + "learning_rate": 0.0001, + "loss": 63.7195, + "step": 19090 + }, + { + "epoch": 3.084132334833774, + "grad_norm": 0.7538934350013733, + "learning_rate": 0.0001, + "loss": 63.776, + "step": 19100 + }, + { + "epoch": 3.0857471589188754, + "grad_norm": 0.5769422054290771, + "learning_rate": 0.0001, + "loss": 64.5028, + "step": 19110 + }, + { + "epoch": 3.0873619830039765, + "grad_norm": 0.5494438409805298, + "learning_rate": 0.0001, + "loss": 61.8753, + "step": 19120 + }, + { + "epoch": 3.088976807089078, + "grad_norm": 0.5360962748527527, + "learning_rate": 0.0001, + "loss": 62.6862, + "step": 19130 + }, + { + "epoch": 3.090591631174179, + "grad_norm": 0.5352891087532043, + "learning_rate": 0.0001, + "loss": 62.5873, + "step": 19140 + }, + { + "epoch": 3.09220645525928, + "grad_norm": 0.680666983127594, + "learning_rate": 0.0001, + "loss": 62.5556, + "step": 19150 + }, + { + "epoch": 3.0938212793443816, + "grad_norm": 0.6260353326797485, + "learning_rate": 0.0001, + "loss": 61.9883, + "step": 19160 + }, + { + "epoch": 3.0954361034294826, + "grad_norm": 0.590894877910614, + "learning_rate": 0.0001, + "loss": 63.2063, + "step": 19170 + }, + { + "epoch": 3.0970509275145837, + "grad_norm": 0.751380205154419, + "learning_rate": 0.0001, + "loss": 62.6803, + "step": 19180 + }, + { + "epoch": 3.098665751599685, + "grad_norm": 0.5277641415596008, + "learning_rate": 0.0001, + "loss": 63.1831, + "step": 19190 + }, + { + "epoch": 3.100280575684786, + "grad_norm": 0.6478848457336426, + "learning_rate": 0.0001, + "loss": 64.1269, + "step": 19200 + }, + { + "epoch": 3.1018953997698877, + "grad_norm": 0.5327169299125671, + "learning_rate": 0.0001, + "loss": 63.1666, + "step": 19210 + }, + { + "epoch": 3.1035102238549888, + "grad_norm": 0.5020601153373718, + "learning_rate": 0.0001, + "loss": 64.8419, + "step": 19220 + }, + { + "epoch": 3.1051250479400903, + "grad_norm": 0.6309834718704224, + "learning_rate": 0.0001, + "loss": 63.9949, + "step": 19230 + }, + { + "epoch": 3.1067398720251913, + "grad_norm": 0.48635050654411316, + "learning_rate": 0.0001, + "loss": 61.6099, + "step": 19240 + }, + { + "epoch": 3.1083546961102924, + "grad_norm": 0.570539653301239, + "learning_rate": 0.0001, + "loss": 62.2674, + "step": 19250 + }, + { + "epoch": 3.109969520195394, + "grad_norm": 0.8218621611595154, + "learning_rate": 0.0001, + "loss": 63.6042, + "step": 19260 + }, + { + "epoch": 3.111584344280495, + "grad_norm": 0.5695828199386597, + "learning_rate": 0.0001, + "loss": 63.7523, + "step": 19270 + }, + { + "epoch": 3.113199168365596, + "grad_norm": 0.5348001718521118, + "learning_rate": 0.0001, + "loss": 64.5132, + "step": 19280 + }, + { + "epoch": 3.1148139924506975, + "grad_norm": 0.6444787979125977, + "learning_rate": 0.0001, + "loss": 62.26, + "step": 19290 + }, + { + "epoch": 3.1164288165357985, + "grad_norm": 0.6655979752540588, + "learning_rate": 0.0001, + "loss": 62.6194, + "step": 19300 + }, + { + "epoch": 3.1180436406209, + "grad_norm": 0.6016690135002136, + "learning_rate": 0.0001, + "loss": 63.9382, + "step": 19310 + }, + { + "epoch": 3.119658464706001, + "grad_norm": 0.908967137336731, + "learning_rate": 0.0001, + "loss": 61.7616, + "step": 19320 + }, + { + "epoch": 3.121273288791102, + "grad_norm": 0.5426335334777832, + "learning_rate": 0.0001, + "loss": 64.0678, + "step": 19330 + }, + { + "epoch": 3.1228881128762036, + "grad_norm": 0.5484811067581177, + "learning_rate": 0.0001, + "loss": 61.9463, + "step": 19340 + }, + { + "epoch": 3.1245029369613047, + "grad_norm": 0.5296673774719238, + "learning_rate": 0.0001, + "loss": 63.3108, + "step": 19350 + }, + { + "epoch": 3.126117761046406, + "grad_norm": 0.9080342054367065, + "learning_rate": 0.0001, + "loss": 62.1432, + "step": 19360 + }, + { + "epoch": 3.127732585131507, + "grad_norm": 0.5220646858215332, + "learning_rate": 0.0001, + "loss": 64.3117, + "step": 19370 + }, + { + "epoch": 3.1293474092166083, + "grad_norm": 0.4961260259151459, + "learning_rate": 0.0001, + "loss": 63.8505, + "step": 19380 + }, + { + "epoch": 3.1309622333017098, + "grad_norm": 0.9542286992073059, + "learning_rate": 0.0001, + "loss": 62.3382, + "step": 19390 + }, + { + "epoch": 3.132577057386811, + "grad_norm": 0.6965755224227905, + "learning_rate": 0.0001, + "loss": 63.1273, + "step": 19400 + }, + { + "epoch": 3.1341918814719123, + "grad_norm": 0.49719423055648804, + "learning_rate": 0.0001, + "loss": 63.9202, + "step": 19410 + }, + { + "epoch": 3.1358067055570134, + "grad_norm": 0.5940988063812256, + "learning_rate": 0.0001, + "loss": 64.705, + "step": 19420 + }, + { + "epoch": 3.137421529642115, + "grad_norm": 0.577771008014679, + "learning_rate": 0.0001, + "loss": 66.561, + "step": 19430 + }, + { + "epoch": 3.139036353727216, + "grad_norm": 0.6199527978897095, + "learning_rate": 0.0001, + "loss": 63.8797, + "step": 19440 + }, + { + "epoch": 3.140651177812317, + "grad_norm": 0.5772822499275208, + "learning_rate": 0.0001, + "loss": 63.8714, + "step": 19450 + }, + { + "epoch": 3.1422660018974184, + "grad_norm": 0.515441358089447, + "learning_rate": 0.0001, + "loss": 65.4966, + "step": 19460 + }, + { + "epoch": 3.1438808259825195, + "grad_norm": 0.6144456267356873, + "learning_rate": 0.0001, + "loss": 61.4128, + "step": 19470 + }, + { + "epoch": 3.1454956500676206, + "grad_norm": 0.5595372915267944, + "learning_rate": 0.0001, + "loss": 63.1169, + "step": 19480 + }, + { + "epoch": 3.147110474152722, + "grad_norm": 0.5817322731018066, + "learning_rate": 0.0001, + "loss": 61.5595, + "step": 19490 + }, + { + "epoch": 3.148725298237823, + "grad_norm": 0.8475374579429626, + "learning_rate": 0.0001, + "loss": 62.8805, + "step": 19500 + }, + { + "epoch": 3.1503401223229246, + "grad_norm": 0.71738201379776, + "learning_rate": 0.0001, + "loss": 63.9747, + "step": 19510 + }, + { + "epoch": 3.1519549464080256, + "grad_norm": 0.5480318069458008, + "learning_rate": 0.0001, + "loss": 63.3571, + "step": 19520 + }, + { + "epoch": 3.1535697704931267, + "grad_norm": 0.6449233889579773, + "learning_rate": 0.0001, + "loss": 64.9267, + "step": 19530 + }, + { + "epoch": 3.155184594578228, + "grad_norm": 0.5388614535331726, + "learning_rate": 0.0001, + "loss": 62.1875, + "step": 19540 + }, + { + "epoch": 3.1567994186633292, + "grad_norm": 0.7202700972557068, + "learning_rate": 0.0001, + "loss": 61.9189, + "step": 19550 + }, + { + "epoch": 3.1584142427484307, + "grad_norm": 0.6596958041191101, + "learning_rate": 0.0001, + "loss": 62.2087, + "step": 19560 + }, + { + "epoch": 3.160029066833532, + "grad_norm": 0.591659426689148, + "learning_rate": 0.0001, + "loss": 62.5925, + "step": 19570 + }, + { + "epoch": 3.161643890918633, + "grad_norm": 0.7457690238952637, + "learning_rate": 0.0001, + "loss": 64.4736, + "step": 19580 + }, + { + "epoch": 3.1632587150037343, + "grad_norm": 1.7162328958511353, + "learning_rate": 0.0001, + "loss": 63.3156, + "step": 19590 + }, + { + "epoch": 3.1648735390888354, + "grad_norm": 0.525978684425354, + "learning_rate": 0.0001, + "loss": 63.2241, + "step": 19600 + }, + { + "epoch": 3.166488363173937, + "grad_norm": 0.6228973865509033, + "learning_rate": 0.0001, + "loss": 61.8464, + "step": 19610 + }, + { + "epoch": 3.168103187259038, + "grad_norm": 0.5232539772987366, + "learning_rate": 0.0001, + "loss": 63.8573, + "step": 19620 + }, + { + "epoch": 3.169718011344139, + "grad_norm": 0.5182655453681946, + "learning_rate": 0.0001, + "loss": 63.2282, + "step": 19630 + }, + { + "epoch": 3.1713328354292405, + "grad_norm": 0.5557834506034851, + "learning_rate": 0.0001, + "loss": 63.8266, + "step": 19640 + }, + { + "epoch": 3.1729476595143415, + "grad_norm": 0.6693798899650574, + "learning_rate": 0.0001, + "loss": 63.4246, + "step": 19650 + }, + { + "epoch": 3.174562483599443, + "grad_norm": 0.8174669742584229, + "learning_rate": 0.0001, + "loss": 65.1013, + "step": 19660 + }, + { + "epoch": 3.176177307684544, + "grad_norm": 0.550268828868866, + "learning_rate": 0.0001, + "loss": 63.2053, + "step": 19670 + }, + { + "epoch": 3.177792131769645, + "grad_norm": 0.5820688009262085, + "learning_rate": 0.0001, + "loss": 65.1428, + "step": 19680 + }, + { + "epoch": 3.1794069558547466, + "grad_norm": 0.65394526720047, + "learning_rate": 0.0001, + "loss": 62.2917, + "step": 19690 + }, + { + "epoch": 3.1810217799398477, + "grad_norm": 0.520659863948822, + "learning_rate": 0.0001, + "loss": 61.6482, + "step": 19700 + }, + { + "epoch": 3.182636604024949, + "grad_norm": 0.6293434500694275, + "learning_rate": 0.0001, + "loss": 61.8904, + "step": 19710 + }, + { + "epoch": 3.1842514281100502, + "grad_norm": 0.5458523035049438, + "learning_rate": 0.0001, + "loss": 62.6965, + "step": 19720 + }, + { + "epoch": 3.1858662521951513, + "grad_norm": 0.530017077922821, + "learning_rate": 0.0001, + "loss": 63.1896, + "step": 19730 + }, + { + "epoch": 3.187481076280253, + "grad_norm": 0.5272435545921326, + "learning_rate": 0.0001, + "loss": 62.3113, + "step": 19740 + }, + { + "epoch": 3.189095900365354, + "grad_norm": 0.4784429669380188, + "learning_rate": 0.0001, + "loss": 61.1635, + "step": 19750 + }, + { + "epoch": 3.1907107244504553, + "grad_norm": 0.6120122671127319, + "learning_rate": 0.0001, + "loss": 62.3033, + "step": 19760 + }, + { + "epoch": 3.1923255485355564, + "grad_norm": 0.6047812700271606, + "learning_rate": 0.0001, + "loss": 63.4115, + "step": 19770 + }, + { + "epoch": 3.1939403726206574, + "grad_norm": 0.5897378325462341, + "learning_rate": 0.0001, + "loss": 62.7949, + "step": 19780 + }, + { + "epoch": 3.195555196705759, + "grad_norm": 0.5099884867668152, + "learning_rate": 0.0001, + "loss": 62.9608, + "step": 19790 + }, + { + "epoch": 3.19717002079086, + "grad_norm": 0.6914068460464478, + "learning_rate": 0.0001, + "loss": 61.8816, + "step": 19800 + }, + { + "epoch": 3.1987848448759615, + "grad_norm": 0.6614701151847839, + "learning_rate": 0.0001, + "loss": 65.2322, + "step": 19810 + }, + { + "epoch": 3.2003996689610625, + "grad_norm": 0.6105468273162842, + "learning_rate": 0.0001, + "loss": 62.8061, + "step": 19820 + }, + { + "epoch": 3.2020144930461636, + "grad_norm": 0.7444026470184326, + "learning_rate": 0.0001, + "loss": 63.4653, + "step": 19830 + }, + { + "epoch": 3.203629317131265, + "grad_norm": 0.4747075140476227, + "learning_rate": 0.0001, + "loss": 64.4906, + "step": 19840 + }, + { + "epoch": 3.205244141216366, + "grad_norm": 0.46989893913269043, + "learning_rate": 0.0001, + "loss": 63.7824, + "step": 19850 + }, + { + "epoch": 3.2068589653014676, + "grad_norm": 0.5924081802368164, + "learning_rate": 0.0001, + "loss": 61.8988, + "step": 19860 + }, + { + "epoch": 3.2084737893865687, + "grad_norm": 0.5401791334152222, + "learning_rate": 0.0001, + "loss": 63.9228, + "step": 19870 + }, + { + "epoch": 3.2100886134716697, + "grad_norm": 0.7052827477455139, + "learning_rate": 0.0001, + "loss": 65.8733, + "step": 19880 + }, + { + "epoch": 3.2117034375567712, + "grad_norm": 0.5406411290168762, + "learning_rate": 0.0001, + "loss": 63.0229, + "step": 19890 + }, + { + "epoch": 3.2133182616418723, + "grad_norm": 0.6213598251342773, + "learning_rate": 0.0001, + "loss": 63.5284, + "step": 19900 + }, + { + "epoch": 3.214933085726974, + "grad_norm": 0.5287862420082092, + "learning_rate": 0.0001, + "loss": 62.953, + "step": 19910 + }, + { + "epoch": 3.216547909812075, + "grad_norm": 0.613235354423523, + "learning_rate": 0.0001, + "loss": 62.8383, + "step": 19920 + }, + { + "epoch": 3.218162733897176, + "grad_norm": 0.49906688928604126, + "learning_rate": 0.0001, + "loss": 62.9626, + "step": 19930 + }, + { + "epoch": 3.2197775579822774, + "grad_norm": 0.4975186586380005, + "learning_rate": 0.0001, + "loss": 63.4374, + "step": 19940 + }, + { + "epoch": 3.2213923820673784, + "grad_norm": 0.5689376592636108, + "learning_rate": 0.0001, + "loss": 64.8217, + "step": 19950 + }, + { + "epoch": 3.22300720615248, + "grad_norm": 0.5206901431083679, + "learning_rate": 0.0001, + "loss": 62.8481, + "step": 19960 + }, + { + "epoch": 3.224622030237581, + "grad_norm": 0.6074235439300537, + "learning_rate": 0.0001, + "loss": 63.0375, + "step": 19970 + }, + { + "epoch": 3.226236854322682, + "grad_norm": 1.0357732772827148, + "learning_rate": 0.0001, + "loss": 61.4771, + "step": 19980 + }, + { + "epoch": 3.2278516784077835, + "grad_norm": 1.633543848991394, + "learning_rate": 0.0001, + "loss": 62.4537, + "step": 19990 + }, + { + "epoch": 3.2294665024928846, + "grad_norm": 0.5143811106681824, + "learning_rate": 0.0001, + "loss": 62.9348, + "step": 20000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.4488646217360736e+20, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6a151f546555c78ebc502dd7b703b665f0672d6f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35235df8408202e91954a585851d27187bafaf4e1d75c4e63c51fbf721136ff5 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-20000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/latest b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/latest new file mode 100644 index 0000000000000000000000000000000000000000..641c40ce8d5e05e04a0840d69bcc5fac045a3fbd --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/latest @@ -0,0 +1 @@ +global_step4000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ec96aa07a553a278c2c22f8c0af76443e50478d9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f12738aef75b5d0cf864d0a6aa9f89f36455f8a64cd8df565c2e8ede9fedaa55 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..765bcb7cf6d6331f95d77c80e1a65822f7c0c3fc --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ab13d9cfb97366c96a5ad023857ce1bbffea5c5598ee3881ffb979c69dbd8c8 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..c75f766e5902d18e610541a7661993483794ef9a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:855e3088785591f18babb0ba72ad8a27091cad126b30e9a41247938ac41fdfed +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..a50d08703d130201c8690c7e27a1c878be07fb17 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4234e4484fa523a283df8055cf2abdbc0ff99bc398a703ebbc57954e9e2132a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..835252b7f33f90244b668130a01b1e90aa3e98c5 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93b18d8f276e0c1c8ec4071bda5c12112ed1ab96ae2efb8adfa487eb7544a9f6 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..db38b62a7fc51c4871b01588434ac2196ab197cc --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ff041494625f96812a3abda4179831f953bdce189434c51ee2f62dc8c5bf88f +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..e71fdd4437afdd8ab35fb54477f20909c7e7506d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d3bd1114ea5e6574480177baad92a073fc17d416dd45cfe5a9107344bf6c506 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..203eecd60a0539b07b7791e83ffeadf2a9b62e9a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:581c811778a0a36646ec155f3466eddb0c6dbe873f817972f07cf61e5590eff6 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..ab568f76571e2a01cbb41b44d5796e2aa3dcc3ce --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5cb77542d6dd1898d12a9c9d3acd3529beca0eb106a039539e75d1f989d99f0 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7c33095320ec6a7f585e68e4e02ed022aa80652 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e88d5675c3e7af2f5bef04cb9622f8a630e6e25147b9f39eb466dd8ba603a69a +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a87d2804da39d05f795cd017f86e267a35021353 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/trainer_state.json @@ -0,0 +1,2834 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6459296340404918, + "eval_steps": 2000, + "global_step": 4000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0016148240851012292, + "grad_norm": 43.90101623535156, + "learning_rate": 0.0001, + "loss": 307.2084, + "step": 10 + }, + { + "epoch": 0.0032296481702024584, + "grad_norm": 19.106313705444336, + "learning_rate": 0.0001, + "loss": 320.2794, + "step": 20 + }, + { + "epoch": 0.004844472255303688, + "grad_norm": 5.771847248077393, + "learning_rate": 0.0001, + "loss": 247.4977, + "step": 30 + }, + { + "epoch": 0.006459296340404917, + "grad_norm": 2.630659580230713, + "learning_rate": 0.0001, + "loss": 229.9593, + "step": 40 + }, + { + "epoch": 0.008074120425506147, + "grad_norm": 2.487499475479126, + "learning_rate": 0.0001, + "loss": 228.65, + "step": 50 + }, + { + "epoch": 0.009688944510607376, + "grad_norm": 2.078263521194458, + "learning_rate": 0.0001, + "loss": 227.648, + "step": 60 + }, + { + "epoch": 0.011303768595708604, + "grad_norm": 3.105417490005493, + "learning_rate": 0.0001, + "loss": 227.9345, + "step": 70 + }, + { + "epoch": 0.012918592680809833, + "grad_norm": 2.2145848274230957, + "learning_rate": 0.0001, + "loss": 227.3662, + "step": 80 + }, + { + "epoch": 0.014533416765911064, + "grad_norm": 1.9644615650177002, + "learning_rate": 0.0001, + "loss": 226.9145, + "step": 90 + }, + { + "epoch": 0.016148240851012293, + "grad_norm": 1.867382287979126, + "learning_rate": 0.0001, + "loss": 227.5703, + "step": 100 + }, + { + "epoch": 0.017763064936113522, + "grad_norm": 1.4613710641860962, + "learning_rate": 0.0001, + "loss": 227.1107, + "step": 110 + }, + { + "epoch": 0.01937788902121475, + "grad_norm": 2.8953869342803955, + "learning_rate": 0.0001, + "loss": 226.8185, + "step": 120 + }, + { + "epoch": 0.02099271310631598, + "grad_norm": 1.8446482419967651, + "learning_rate": 0.0001, + "loss": 226.9624, + "step": 130 + }, + { + "epoch": 0.02260753719141721, + "grad_norm": 1.3048362731933594, + "learning_rate": 0.0001, + "loss": 226.8404, + "step": 140 + }, + { + "epoch": 0.024222361276518438, + "grad_norm": 1.7100074291229248, + "learning_rate": 0.0001, + "loss": 226.1071, + "step": 150 + }, + { + "epoch": 0.025837185361619667, + "grad_norm": 1.7875330448150635, + "learning_rate": 0.0001, + "loss": 226.0764, + "step": 160 + }, + { + "epoch": 0.0274520094467209, + "grad_norm": 1.7503769397735596, + "learning_rate": 0.0001, + "loss": 226.0772, + "step": 170 + }, + { + "epoch": 0.029066833531822128, + "grad_norm": 2.7083983421325684, + "learning_rate": 0.0001, + "loss": 224.7667, + "step": 180 + }, + { + "epoch": 0.030681657616923357, + "grad_norm": 1.8192371129989624, + "learning_rate": 0.0001, + "loss": 223.6494, + "step": 190 + }, + { + "epoch": 0.032296481702024586, + "grad_norm": 3.210358142852783, + "learning_rate": 0.0001, + "loss": 223.3545, + "step": 200 + }, + { + "epoch": 0.03391130578712581, + "grad_norm": 5.819669246673584, + "learning_rate": 0.0001, + "loss": 222.1732, + "step": 210 + }, + { + "epoch": 0.035526129872227044, + "grad_norm": 4.185793399810791, + "learning_rate": 0.0001, + "loss": 221.6866, + "step": 220 + }, + { + "epoch": 0.03714095395732828, + "grad_norm": 2.7514216899871826, + "learning_rate": 0.0001, + "loss": 221.0769, + "step": 230 + }, + { + "epoch": 0.0387557780424295, + "grad_norm": 1.5478988885879517, + "learning_rate": 0.0001, + "loss": 219.7232, + "step": 240 + }, + { + "epoch": 0.040370602127530734, + "grad_norm": 1.4958223104476929, + "learning_rate": 0.0001, + "loss": 218.5454, + "step": 250 + }, + { + "epoch": 0.04198542621263196, + "grad_norm": 6.36635684967041, + "learning_rate": 0.0001, + "loss": 218.5061, + "step": 260 + }, + { + "epoch": 0.04360025029773319, + "grad_norm": 3.029601812362671, + "learning_rate": 0.0001, + "loss": 216.3466, + "step": 270 + }, + { + "epoch": 0.04521507438283442, + "grad_norm": 2.6436100006103516, + "learning_rate": 0.0001, + "loss": 214.0003, + "step": 280 + }, + { + "epoch": 0.04682989846793565, + "grad_norm": 5.329655647277832, + "learning_rate": 0.0001, + "loss": 212.9169, + "step": 290 + }, + { + "epoch": 0.048444722553036876, + "grad_norm": 4.930875301361084, + "learning_rate": 0.0001, + "loss": 209.7737, + "step": 300 + }, + { + "epoch": 0.05005954663813811, + "grad_norm": 18.902585983276367, + "learning_rate": 0.0001, + "loss": 209.4267, + "step": 310 + }, + { + "epoch": 0.051674370723239334, + "grad_norm": 5.364975929260254, + "learning_rate": 0.0001, + "loss": 206.49, + "step": 320 + }, + { + "epoch": 0.053289194808340566, + "grad_norm": 3.1421146392822266, + "learning_rate": 0.0001, + "loss": 204.8132, + "step": 330 + }, + { + "epoch": 0.0549040188934418, + "grad_norm": 3.399320602416992, + "learning_rate": 0.0001, + "loss": 202.4785, + "step": 340 + }, + { + "epoch": 0.056518842978543024, + "grad_norm": 5.137377738952637, + "learning_rate": 0.0001, + "loss": 199.5143, + "step": 350 + }, + { + "epoch": 0.058133667063644257, + "grad_norm": 5.4967522621154785, + "learning_rate": 0.0001, + "loss": 196.2809, + "step": 360 + }, + { + "epoch": 0.05974849114874548, + "grad_norm": 6.435070037841797, + "learning_rate": 0.0001, + "loss": 196.8831, + "step": 370 + }, + { + "epoch": 0.061363315233846714, + "grad_norm": 5.7334370613098145, + "learning_rate": 0.0001, + "loss": 193.141, + "step": 380 + }, + { + "epoch": 0.06297813931894794, + "grad_norm": 3.6522979736328125, + "learning_rate": 0.0001, + "loss": 191.5819, + "step": 390 + }, + { + "epoch": 0.06459296340404917, + "grad_norm": 2.614051103591919, + "learning_rate": 0.0001, + "loss": 190.046, + "step": 400 + }, + { + "epoch": 0.0662077874891504, + "grad_norm": 4.845924377441406, + "learning_rate": 0.0001, + "loss": 188.7121, + "step": 410 + }, + { + "epoch": 0.06782261157425162, + "grad_norm": 4.319681644439697, + "learning_rate": 0.0001, + "loss": 185.3697, + "step": 420 + }, + { + "epoch": 0.06943743565935286, + "grad_norm": 2.7018792629241943, + "learning_rate": 0.0001, + "loss": 183.5171, + "step": 430 + }, + { + "epoch": 0.07105225974445409, + "grad_norm": 2.6407113075256348, + "learning_rate": 0.0001, + "loss": 181.245, + "step": 440 + }, + { + "epoch": 0.07266708382955532, + "grad_norm": 3.314251184463501, + "learning_rate": 0.0001, + "loss": 179.3874, + "step": 450 + }, + { + "epoch": 0.07428190791465655, + "grad_norm": 2.635136365890503, + "learning_rate": 0.0001, + "loss": 177.8257, + "step": 460 + }, + { + "epoch": 0.07589673199975777, + "grad_norm": 5.227738857269287, + "learning_rate": 0.0001, + "loss": 176.4699, + "step": 470 + }, + { + "epoch": 0.077511556084859, + "grad_norm": 4.998415470123291, + "learning_rate": 0.0001, + "loss": 175.7601, + "step": 480 + }, + { + "epoch": 0.07912638016996024, + "grad_norm": 2.946892261505127, + "learning_rate": 0.0001, + "loss": 173.936, + "step": 490 + }, + { + "epoch": 0.08074120425506147, + "grad_norm": 4.20358943939209, + "learning_rate": 0.0001, + "loss": 172.065, + "step": 500 + }, + { + "epoch": 0.08235602834016269, + "grad_norm": 3.125765562057495, + "learning_rate": 0.0001, + "loss": 171.3727, + "step": 510 + }, + { + "epoch": 0.08397085242526392, + "grad_norm": 2.4870612621307373, + "learning_rate": 0.0001, + "loss": 169.9, + "step": 520 + }, + { + "epoch": 0.08558567651036515, + "grad_norm": 3.917384147644043, + "learning_rate": 0.0001, + "loss": 168.5049, + "step": 530 + }, + { + "epoch": 0.08720050059546638, + "grad_norm": 2.302142858505249, + "learning_rate": 0.0001, + "loss": 167.2027, + "step": 540 + }, + { + "epoch": 0.08881532468056762, + "grad_norm": 4.794841289520264, + "learning_rate": 0.0001, + "loss": 166.0887, + "step": 550 + }, + { + "epoch": 0.09043014876566884, + "grad_norm": 2.200291395187378, + "learning_rate": 0.0001, + "loss": 164.5882, + "step": 560 + }, + { + "epoch": 0.09204497285077007, + "grad_norm": 1.9572436809539795, + "learning_rate": 0.0001, + "loss": 163.3007, + "step": 570 + }, + { + "epoch": 0.0936597969358713, + "grad_norm": 3.443633556365967, + "learning_rate": 0.0001, + "loss": 160.6805, + "step": 580 + }, + { + "epoch": 0.09527462102097253, + "grad_norm": 2.6079020500183105, + "learning_rate": 0.0001, + "loss": 160.3372, + "step": 590 + }, + { + "epoch": 0.09688944510607375, + "grad_norm": 2.4401192665100098, + "learning_rate": 0.0001, + "loss": 160.0956, + "step": 600 + }, + { + "epoch": 0.09850426919117498, + "grad_norm": 2.686101198196411, + "learning_rate": 0.0001, + "loss": 157.6138, + "step": 610 + }, + { + "epoch": 0.10011909327627622, + "grad_norm": 2.594151496887207, + "learning_rate": 0.0001, + "loss": 156.0726, + "step": 620 + }, + { + "epoch": 0.10173391736137745, + "grad_norm": 2.509141683578491, + "learning_rate": 0.0001, + "loss": 155.9923, + "step": 630 + }, + { + "epoch": 0.10334874144647867, + "grad_norm": 2.4077377319335938, + "learning_rate": 0.0001, + "loss": 155.214, + "step": 640 + }, + { + "epoch": 0.1049635655315799, + "grad_norm": 4.083552837371826, + "learning_rate": 0.0001, + "loss": 152.8257, + "step": 650 + }, + { + "epoch": 0.10657838961668113, + "grad_norm": 2.7188713550567627, + "learning_rate": 0.0001, + "loss": 151.875, + "step": 660 + }, + { + "epoch": 0.10819321370178236, + "grad_norm": 2.41287899017334, + "learning_rate": 0.0001, + "loss": 151.935, + "step": 670 + }, + { + "epoch": 0.1098080377868836, + "grad_norm": 3.995708465576172, + "learning_rate": 0.0001, + "loss": 151.0069, + "step": 680 + }, + { + "epoch": 0.11142286187198482, + "grad_norm": 3.217705488204956, + "learning_rate": 0.0001, + "loss": 149.5922, + "step": 690 + }, + { + "epoch": 0.11303768595708605, + "grad_norm": 3.4349842071533203, + "learning_rate": 0.0001, + "loss": 148.0044, + "step": 700 + }, + { + "epoch": 0.11465251004218728, + "grad_norm": 2.106865167617798, + "learning_rate": 0.0001, + "loss": 147.8404, + "step": 710 + }, + { + "epoch": 0.11626733412728851, + "grad_norm": 1.8729510307312012, + "learning_rate": 0.0001, + "loss": 145.1299, + "step": 720 + }, + { + "epoch": 0.11788215821238973, + "grad_norm": 4.14620304107666, + "learning_rate": 0.0001, + "loss": 144.3928, + "step": 730 + }, + { + "epoch": 0.11949698229749096, + "grad_norm": 2.202505350112915, + "learning_rate": 0.0001, + "loss": 144.8357, + "step": 740 + }, + { + "epoch": 0.1211118063825922, + "grad_norm": 2.201470375061035, + "learning_rate": 0.0001, + "loss": 143.2202, + "step": 750 + }, + { + "epoch": 0.12272663046769343, + "grad_norm": 1.5977853536605835, + "learning_rate": 0.0001, + "loss": 143.3537, + "step": 760 + }, + { + "epoch": 0.12434145455279466, + "grad_norm": 2.4403927326202393, + "learning_rate": 0.0001, + "loss": 141.2914, + "step": 770 + }, + { + "epoch": 0.12595627863789588, + "grad_norm": 1.6895846128463745, + "learning_rate": 0.0001, + "loss": 139.9639, + "step": 780 + }, + { + "epoch": 0.12757110272299713, + "grad_norm": 3.4370665550231934, + "learning_rate": 0.0001, + "loss": 140.715, + "step": 790 + }, + { + "epoch": 0.12918592680809834, + "grad_norm": 2.1449077129364014, + "learning_rate": 0.0001, + "loss": 139.2094, + "step": 800 + }, + { + "epoch": 0.13080075089319956, + "grad_norm": 2.4578912258148193, + "learning_rate": 0.0001, + "loss": 137.6724, + "step": 810 + }, + { + "epoch": 0.1324155749783008, + "grad_norm": 1.8994814157485962, + "learning_rate": 0.0001, + "loss": 136.9551, + "step": 820 + }, + { + "epoch": 0.13403039906340203, + "grad_norm": 2.6531031131744385, + "learning_rate": 0.0001, + "loss": 136.5812, + "step": 830 + }, + { + "epoch": 0.13564522314850325, + "grad_norm": 3.8680083751678467, + "learning_rate": 0.0001, + "loss": 134.3655, + "step": 840 + }, + { + "epoch": 0.1372600472336045, + "grad_norm": 1.9594208002090454, + "learning_rate": 0.0001, + "loss": 135.4958, + "step": 850 + }, + { + "epoch": 0.1388748713187057, + "grad_norm": 1.5475000143051147, + "learning_rate": 0.0001, + "loss": 135.2142, + "step": 860 + }, + { + "epoch": 0.14048969540380696, + "grad_norm": 1.6569288969039917, + "learning_rate": 0.0001, + "loss": 133.4162, + "step": 870 + }, + { + "epoch": 0.14210451948890818, + "grad_norm": 1.6724224090576172, + "learning_rate": 0.0001, + "loss": 133.7519, + "step": 880 + }, + { + "epoch": 0.1437193435740094, + "grad_norm": 1.7299500703811646, + "learning_rate": 0.0001, + "loss": 131.471, + "step": 890 + }, + { + "epoch": 0.14533416765911064, + "grad_norm": 1.9444470405578613, + "learning_rate": 0.0001, + "loss": 128.5966, + "step": 900 + }, + { + "epoch": 0.14694899174421186, + "grad_norm": 2.404723644256592, + "learning_rate": 0.0001, + "loss": 129.9824, + "step": 910 + }, + { + "epoch": 0.1485638158293131, + "grad_norm": 2.1493897438049316, + "learning_rate": 0.0001, + "loss": 128.0219, + "step": 920 + }, + { + "epoch": 0.15017863991441432, + "grad_norm": 2.8861117362976074, + "learning_rate": 0.0001, + "loss": 128.5297, + "step": 930 + }, + { + "epoch": 0.15179346399951554, + "grad_norm": 2.546415090560913, + "learning_rate": 0.0001, + "loss": 126.6817, + "step": 940 + }, + { + "epoch": 0.1534082880846168, + "grad_norm": 1.727663278579712, + "learning_rate": 0.0001, + "loss": 125.0413, + "step": 950 + }, + { + "epoch": 0.155023112169718, + "grad_norm": 3.7156920433044434, + "learning_rate": 0.0001, + "loss": 124.8085, + "step": 960 + }, + { + "epoch": 0.15663793625481925, + "grad_norm": 3.0576517581939697, + "learning_rate": 0.0001, + "loss": 125.9107, + "step": 970 + }, + { + "epoch": 0.15825276033992047, + "grad_norm": 1.9951574802398682, + "learning_rate": 0.0001, + "loss": 123.5359, + "step": 980 + }, + { + "epoch": 0.1598675844250217, + "grad_norm": 1.4620885848999023, + "learning_rate": 0.0001, + "loss": 121.9363, + "step": 990 + }, + { + "epoch": 0.16148240851012294, + "grad_norm": 1.7640169858932495, + "learning_rate": 0.0001, + "loss": 121.1764, + "step": 1000 + }, + { + "epoch": 0.16309723259522416, + "grad_norm": 2.921750545501709, + "learning_rate": 0.0001, + "loss": 119.6931, + "step": 1010 + }, + { + "epoch": 0.16471205668032537, + "grad_norm": 1.5623588562011719, + "learning_rate": 0.0001, + "loss": 119.097, + "step": 1020 + }, + { + "epoch": 0.16632688076542662, + "grad_norm": 2.2386627197265625, + "learning_rate": 0.0001, + "loss": 120.9224, + "step": 1030 + }, + { + "epoch": 0.16794170485052784, + "grad_norm": 1.3213791847229004, + "learning_rate": 0.0001, + "loss": 119.5649, + "step": 1040 + }, + { + "epoch": 0.16955652893562909, + "grad_norm": 1.5255440473556519, + "learning_rate": 0.0001, + "loss": 118.1524, + "step": 1050 + }, + { + "epoch": 0.1711713530207303, + "grad_norm": 1.6038514375686646, + "learning_rate": 0.0001, + "loss": 118.6307, + "step": 1060 + }, + { + "epoch": 0.17278617710583152, + "grad_norm": 2.356739044189453, + "learning_rate": 0.0001, + "loss": 116.8472, + "step": 1070 + }, + { + "epoch": 0.17440100119093277, + "grad_norm": 2.5326743125915527, + "learning_rate": 0.0001, + "loss": 115.6625, + "step": 1080 + }, + { + "epoch": 0.176015825276034, + "grad_norm": 1.2823396921157837, + "learning_rate": 0.0001, + "loss": 115.3312, + "step": 1090 + }, + { + "epoch": 0.17763064936113523, + "grad_norm": 1.981788158416748, + "learning_rate": 0.0001, + "loss": 116.2257, + "step": 1100 + }, + { + "epoch": 0.17924547344623645, + "grad_norm": 1.540440559387207, + "learning_rate": 0.0001, + "loss": 114.5031, + "step": 1110 + }, + { + "epoch": 0.18086029753133767, + "grad_norm": 1.7667800188064575, + "learning_rate": 0.0001, + "loss": 113.7663, + "step": 1120 + }, + { + "epoch": 0.18247512161643892, + "grad_norm": 2.2793102264404297, + "learning_rate": 0.0001, + "loss": 115.219, + "step": 1130 + }, + { + "epoch": 0.18408994570154014, + "grad_norm": 2.132046699523926, + "learning_rate": 0.0001, + "loss": 112.8482, + "step": 1140 + }, + { + "epoch": 0.18570476978664135, + "grad_norm": 1.3153988122940063, + "learning_rate": 0.0001, + "loss": 111.4546, + "step": 1150 + }, + { + "epoch": 0.1873195938717426, + "grad_norm": 1.5246292352676392, + "learning_rate": 0.0001, + "loss": 112.4348, + "step": 1160 + }, + { + "epoch": 0.18893441795684382, + "grad_norm": 1.365096092224121, + "learning_rate": 0.0001, + "loss": 109.755, + "step": 1170 + }, + { + "epoch": 0.19054924204194507, + "grad_norm": 1.2777953147888184, + "learning_rate": 0.0001, + "loss": 111.0318, + "step": 1180 + }, + { + "epoch": 0.19216406612704628, + "grad_norm": 1.8672829866409302, + "learning_rate": 0.0001, + "loss": 109.1648, + "step": 1190 + }, + { + "epoch": 0.1937788902121475, + "grad_norm": 1.709987759590149, + "learning_rate": 0.0001, + "loss": 108.1616, + "step": 1200 + }, + { + "epoch": 0.19539371429724875, + "grad_norm": 1.176277995109558, + "learning_rate": 0.0001, + "loss": 109.3591, + "step": 1210 + }, + { + "epoch": 0.19700853838234997, + "grad_norm": 1.4181180000305176, + "learning_rate": 0.0001, + "loss": 110.0216, + "step": 1220 + }, + { + "epoch": 0.19862336246745121, + "grad_norm": 1.3246116638183594, + "learning_rate": 0.0001, + "loss": 107.9769, + "step": 1230 + }, + { + "epoch": 0.20023818655255243, + "grad_norm": 1.2501498460769653, + "learning_rate": 0.0001, + "loss": 108.2601, + "step": 1240 + }, + { + "epoch": 0.20185301063765365, + "grad_norm": 1.2277404069900513, + "learning_rate": 0.0001, + "loss": 107.1258, + "step": 1250 + }, + { + "epoch": 0.2034678347227549, + "grad_norm": 1.8064894676208496, + "learning_rate": 0.0001, + "loss": 105.6278, + "step": 1260 + }, + { + "epoch": 0.20508265880785612, + "grad_norm": 1.708897352218628, + "learning_rate": 0.0001, + "loss": 106.9269, + "step": 1270 + }, + { + "epoch": 0.20669748289295733, + "grad_norm": 1.4507478475570679, + "learning_rate": 0.0001, + "loss": 106.9236, + "step": 1280 + }, + { + "epoch": 0.20831230697805858, + "grad_norm": 1.4904310703277588, + "learning_rate": 0.0001, + "loss": 106.0653, + "step": 1290 + }, + { + "epoch": 0.2099271310631598, + "grad_norm": 1.2637165784835815, + "learning_rate": 0.0001, + "loss": 104.8665, + "step": 1300 + }, + { + "epoch": 0.21154195514826105, + "grad_norm": 1.4085133075714111, + "learning_rate": 0.0001, + "loss": 104.8891, + "step": 1310 + }, + { + "epoch": 0.21315677923336226, + "grad_norm": 1.5710629224777222, + "learning_rate": 0.0001, + "loss": 104.3009, + "step": 1320 + }, + { + "epoch": 0.21477160331846348, + "grad_norm": 1.3634663820266724, + "learning_rate": 0.0001, + "loss": 104.4545, + "step": 1330 + }, + { + "epoch": 0.21638642740356473, + "grad_norm": 1.5565859079360962, + "learning_rate": 0.0001, + "loss": 103.6205, + "step": 1340 + }, + { + "epoch": 0.21800125148866595, + "grad_norm": 1.4491888284683228, + "learning_rate": 0.0001, + "loss": 104.0892, + "step": 1350 + }, + { + "epoch": 0.2196160755737672, + "grad_norm": 1.7609063386917114, + "learning_rate": 0.0001, + "loss": 104.5177, + "step": 1360 + }, + { + "epoch": 0.2212308996588684, + "grad_norm": 1.6723753213882446, + "learning_rate": 0.0001, + "loss": 104.5552, + "step": 1370 + }, + { + "epoch": 0.22284572374396963, + "grad_norm": 1.3781665563583374, + "learning_rate": 0.0001, + "loss": 102.1022, + "step": 1380 + }, + { + "epoch": 0.22446054782907088, + "grad_norm": 1.5397366285324097, + "learning_rate": 0.0001, + "loss": 102.5882, + "step": 1390 + }, + { + "epoch": 0.2260753719141721, + "grad_norm": 1.4567879438400269, + "learning_rate": 0.0001, + "loss": 101.3781, + "step": 1400 + }, + { + "epoch": 0.22769019599927334, + "grad_norm": 1.2882301807403564, + "learning_rate": 0.0001, + "loss": 100.9378, + "step": 1410 + }, + { + "epoch": 0.22930502008437456, + "grad_norm": 1.4510424137115479, + "learning_rate": 0.0001, + "loss": 101.5383, + "step": 1420 + }, + { + "epoch": 0.23091984416947578, + "grad_norm": 2.8331449031829834, + "learning_rate": 0.0001, + "loss": 100.2773, + "step": 1430 + }, + { + "epoch": 0.23253466825457703, + "grad_norm": 1.363616943359375, + "learning_rate": 0.0001, + "loss": 101.646, + "step": 1440 + }, + { + "epoch": 0.23414949233967824, + "grad_norm": 1.2184548377990723, + "learning_rate": 0.0001, + "loss": 101.1133, + "step": 1450 + }, + { + "epoch": 0.23576431642477946, + "grad_norm": 2.235656976699829, + "learning_rate": 0.0001, + "loss": 100.2044, + "step": 1460 + }, + { + "epoch": 0.2373791405098807, + "grad_norm": 1.2334418296813965, + "learning_rate": 0.0001, + "loss": 99.4031, + "step": 1470 + }, + { + "epoch": 0.23899396459498193, + "grad_norm": 1.2394684553146362, + "learning_rate": 0.0001, + "loss": 101.1247, + "step": 1480 + }, + { + "epoch": 0.24060878868008317, + "grad_norm": 1.5218186378479004, + "learning_rate": 0.0001, + "loss": 100.4413, + "step": 1490 + }, + { + "epoch": 0.2422236127651844, + "grad_norm": 1.2112364768981934, + "learning_rate": 0.0001, + "loss": 99.8722, + "step": 1500 + }, + { + "epoch": 0.2438384368502856, + "grad_norm": 1.2277610301971436, + "learning_rate": 0.0001, + "loss": 99.0276, + "step": 1510 + }, + { + "epoch": 0.24545326093538686, + "grad_norm": 1.1005053520202637, + "learning_rate": 0.0001, + "loss": 98.6764, + "step": 1520 + }, + { + "epoch": 0.24706808502048808, + "grad_norm": 1.0754456520080566, + "learning_rate": 0.0001, + "loss": 97.8968, + "step": 1530 + }, + { + "epoch": 0.24868290910558932, + "grad_norm": 1.0695950984954834, + "learning_rate": 0.0001, + "loss": 98.4032, + "step": 1540 + }, + { + "epoch": 0.25029773319069054, + "grad_norm": 1.5098627805709839, + "learning_rate": 0.0001, + "loss": 97.9391, + "step": 1550 + }, + { + "epoch": 0.25191255727579176, + "grad_norm": 1.006681203842163, + "learning_rate": 0.0001, + "loss": 96.5525, + "step": 1560 + }, + { + "epoch": 0.253527381360893, + "grad_norm": 1.3987332582473755, + "learning_rate": 0.0001, + "loss": 98.383, + "step": 1570 + }, + { + "epoch": 0.25514220544599425, + "grad_norm": 1.290887713432312, + "learning_rate": 0.0001, + "loss": 99.0901, + "step": 1580 + }, + { + "epoch": 0.25675702953109547, + "grad_norm": 0.991177499294281, + "learning_rate": 0.0001, + "loss": 97.7093, + "step": 1590 + }, + { + "epoch": 0.2583718536161967, + "grad_norm": 1.147647738456726, + "learning_rate": 0.0001, + "loss": 96.9181, + "step": 1600 + }, + { + "epoch": 0.2599866777012979, + "grad_norm": 1.1756514310836792, + "learning_rate": 0.0001, + "loss": 98.3935, + "step": 1610 + }, + { + "epoch": 0.2616015017863991, + "grad_norm": 1.1708602905273438, + "learning_rate": 0.0001, + "loss": 96.715, + "step": 1620 + }, + { + "epoch": 0.2632163258715004, + "grad_norm": 1.4577715396881104, + "learning_rate": 0.0001, + "loss": 97.2971, + "step": 1630 + }, + { + "epoch": 0.2648311499566016, + "grad_norm": 1.1172800064086914, + "learning_rate": 0.0001, + "loss": 97.6494, + "step": 1640 + }, + { + "epoch": 0.26644597404170284, + "grad_norm": 1.1491258144378662, + "learning_rate": 0.0001, + "loss": 97.5503, + "step": 1650 + }, + { + "epoch": 0.26806079812680406, + "grad_norm": 1.2188555002212524, + "learning_rate": 0.0001, + "loss": 95.3423, + "step": 1660 + }, + { + "epoch": 0.2696756222119053, + "grad_norm": 1.8595614433288574, + "learning_rate": 0.0001, + "loss": 96.0112, + "step": 1670 + }, + { + "epoch": 0.2712904462970065, + "grad_norm": 1.1887950897216797, + "learning_rate": 0.0001, + "loss": 95.4939, + "step": 1680 + }, + { + "epoch": 0.27290527038210777, + "grad_norm": 1.1777839660644531, + "learning_rate": 0.0001, + "loss": 95.9619, + "step": 1690 + }, + { + "epoch": 0.274520094467209, + "grad_norm": 1.1241233348846436, + "learning_rate": 0.0001, + "loss": 96.2834, + "step": 1700 + }, + { + "epoch": 0.2761349185523102, + "grad_norm": 1.0900121927261353, + "learning_rate": 0.0001, + "loss": 94.7518, + "step": 1710 + }, + { + "epoch": 0.2777497426374114, + "grad_norm": 1.0615205764770508, + "learning_rate": 0.0001, + "loss": 95.8032, + "step": 1720 + }, + { + "epoch": 0.27936456672251264, + "grad_norm": 1.2626168727874756, + "learning_rate": 0.0001, + "loss": 93.9454, + "step": 1730 + }, + { + "epoch": 0.2809793908076139, + "grad_norm": 0.8869941830635071, + "learning_rate": 0.0001, + "loss": 95.4085, + "step": 1740 + }, + { + "epoch": 0.28259421489271513, + "grad_norm": 1.0350773334503174, + "learning_rate": 0.0001, + "loss": 95.7954, + "step": 1750 + }, + { + "epoch": 0.28420903897781635, + "grad_norm": 1.364768147468567, + "learning_rate": 0.0001, + "loss": 93.6447, + "step": 1760 + }, + { + "epoch": 0.28582386306291757, + "grad_norm": 1.0115883350372314, + "learning_rate": 0.0001, + "loss": 93.7905, + "step": 1770 + }, + { + "epoch": 0.2874386871480188, + "grad_norm": 0.9393435120582581, + "learning_rate": 0.0001, + "loss": 94.3108, + "step": 1780 + }, + { + "epoch": 0.28905351123312006, + "grad_norm": 1.0769627094268799, + "learning_rate": 0.0001, + "loss": 92.566, + "step": 1790 + }, + { + "epoch": 0.2906683353182213, + "grad_norm": 2.526201009750366, + "learning_rate": 0.0001, + "loss": 92.8161, + "step": 1800 + }, + { + "epoch": 0.2922831594033225, + "grad_norm": 1.3066468238830566, + "learning_rate": 0.0001, + "loss": 92.0459, + "step": 1810 + }, + { + "epoch": 0.2938979834884237, + "grad_norm": 1.121713638305664, + "learning_rate": 0.0001, + "loss": 93.5032, + "step": 1820 + }, + { + "epoch": 0.29551280757352494, + "grad_norm": 1.3744316101074219, + "learning_rate": 0.0001, + "loss": 94.2623, + "step": 1830 + }, + { + "epoch": 0.2971276316586262, + "grad_norm": 1.1561540365219116, + "learning_rate": 0.0001, + "loss": 94.1611, + "step": 1840 + }, + { + "epoch": 0.29874245574372743, + "grad_norm": 1.3947464227676392, + "learning_rate": 0.0001, + "loss": 93.5962, + "step": 1850 + }, + { + "epoch": 0.30035727982882865, + "grad_norm": 1.0011566877365112, + "learning_rate": 0.0001, + "loss": 92.0199, + "step": 1860 + }, + { + "epoch": 0.30197210391392987, + "grad_norm": 0.9978747963905334, + "learning_rate": 0.0001, + "loss": 91.0945, + "step": 1870 + }, + { + "epoch": 0.3035869279990311, + "grad_norm": 0.92522132396698, + "learning_rate": 0.0001, + "loss": 93.274, + "step": 1880 + }, + { + "epoch": 0.30520175208413236, + "grad_norm": 1.0089120864868164, + "learning_rate": 0.0001, + "loss": 91.9308, + "step": 1890 + }, + { + "epoch": 0.3068165761692336, + "grad_norm": 1.0535134077072144, + "learning_rate": 0.0001, + "loss": 92.563, + "step": 1900 + }, + { + "epoch": 0.3084314002543348, + "grad_norm": 0.9782275557518005, + "learning_rate": 0.0001, + "loss": 90.3486, + "step": 1910 + }, + { + "epoch": 0.310046224339436, + "grad_norm": 1.1433162689208984, + "learning_rate": 0.0001, + "loss": 90.4977, + "step": 1920 + }, + { + "epoch": 0.31166104842453723, + "grad_norm": 0.98392653465271, + "learning_rate": 0.0001, + "loss": 92.6529, + "step": 1930 + }, + { + "epoch": 0.3132758725096385, + "grad_norm": 0.8538148403167725, + "learning_rate": 0.0001, + "loss": 90.6859, + "step": 1940 + }, + { + "epoch": 0.3148906965947397, + "grad_norm": 1.019405484199524, + "learning_rate": 0.0001, + "loss": 91.0028, + "step": 1950 + }, + { + "epoch": 0.31650552067984095, + "grad_norm": 0.9828067421913147, + "learning_rate": 0.0001, + "loss": 89.9826, + "step": 1960 + }, + { + "epoch": 0.31812034476494216, + "grad_norm": 0.9725450277328491, + "learning_rate": 0.0001, + "loss": 91.7342, + "step": 1970 + }, + { + "epoch": 0.3197351688500434, + "grad_norm": 1.0139408111572266, + "learning_rate": 0.0001, + "loss": 90.5362, + "step": 1980 + }, + { + "epoch": 0.3213499929351446, + "grad_norm": 1.2493853569030762, + "learning_rate": 0.0001, + "loss": 92.7222, + "step": 1990 + }, + { + "epoch": 0.3229648170202459, + "grad_norm": 1.0152915716171265, + "learning_rate": 0.0001, + "loss": 92.2271, + "step": 2000 + }, + { + "epoch": 0.3245796411053471, + "grad_norm": 0.9745819568634033, + "learning_rate": 0.0001, + "loss": 88.7034, + "step": 2010 + }, + { + "epoch": 0.3261944651904483, + "grad_norm": 1.009624719619751, + "learning_rate": 0.0001, + "loss": 90.7428, + "step": 2020 + }, + { + "epoch": 0.32780928927554953, + "grad_norm": 0.910858690738678, + "learning_rate": 0.0001, + "loss": 89.8199, + "step": 2030 + }, + { + "epoch": 0.32942411336065075, + "grad_norm": 1.1745878458023071, + "learning_rate": 0.0001, + "loss": 90.8267, + "step": 2040 + }, + { + "epoch": 0.331038937445752, + "grad_norm": 0.9409273266792297, + "learning_rate": 0.0001, + "loss": 90.2937, + "step": 2050 + }, + { + "epoch": 0.33265376153085324, + "grad_norm": 1.4152942895889282, + "learning_rate": 0.0001, + "loss": 90.3965, + "step": 2060 + }, + { + "epoch": 0.33426858561595446, + "grad_norm": 0.9529693722724915, + "learning_rate": 0.0001, + "loss": 90.4739, + "step": 2070 + }, + { + "epoch": 0.3358834097010557, + "grad_norm": 0.8531096577644348, + "learning_rate": 0.0001, + "loss": 90.9301, + "step": 2080 + }, + { + "epoch": 0.3374982337861569, + "grad_norm": 0.9314167499542236, + "learning_rate": 0.0001, + "loss": 90.4235, + "step": 2090 + }, + { + "epoch": 0.33911305787125817, + "grad_norm": 0.9253250360488892, + "learning_rate": 0.0001, + "loss": 88.1893, + "step": 2100 + }, + { + "epoch": 0.3407278819563594, + "grad_norm": 1.773053526878357, + "learning_rate": 0.0001, + "loss": 88.3417, + "step": 2110 + }, + { + "epoch": 0.3423427060414606, + "grad_norm": 0.9432951211929321, + "learning_rate": 0.0001, + "loss": 89.7141, + "step": 2120 + }, + { + "epoch": 0.34395753012656183, + "grad_norm": 3.8847527503967285, + "learning_rate": 0.0001, + "loss": 89.1009, + "step": 2130 + }, + { + "epoch": 0.34557235421166305, + "grad_norm": 1.0309821367263794, + "learning_rate": 0.0001, + "loss": 89.2965, + "step": 2140 + }, + { + "epoch": 0.3471871782967643, + "grad_norm": 0.9590495824813843, + "learning_rate": 0.0001, + "loss": 89.1734, + "step": 2150 + }, + { + "epoch": 0.34880200238186554, + "grad_norm": 0.9943560361862183, + "learning_rate": 0.0001, + "loss": 85.425, + "step": 2160 + }, + { + "epoch": 0.35041682646696676, + "grad_norm": 1.3322339057922363, + "learning_rate": 0.0001, + "loss": 89.7791, + "step": 2170 + }, + { + "epoch": 0.352031650552068, + "grad_norm": 1.0212253332138062, + "learning_rate": 0.0001, + "loss": 89.3771, + "step": 2180 + }, + { + "epoch": 0.3536464746371692, + "grad_norm": 1.0446089506149292, + "learning_rate": 0.0001, + "loss": 88.6894, + "step": 2190 + }, + { + "epoch": 0.35526129872227047, + "grad_norm": 1.05767822265625, + "learning_rate": 0.0001, + "loss": 89.5341, + "step": 2200 + }, + { + "epoch": 0.3568761228073717, + "grad_norm": 1.0373681783676147, + "learning_rate": 0.0001, + "loss": 89.7173, + "step": 2210 + }, + { + "epoch": 0.3584909468924729, + "grad_norm": 0.9098039269447327, + "learning_rate": 0.0001, + "loss": 89.2116, + "step": 2220 + }, + { + "epoch": 0.3601057709775741, + "grad_norm": 1.028130292892456, + "learning_rate": 0.0001, + "loss": 88.6627, + "step": 2230 + }, + { + "epoch": 0.36172059506267534, + "grad_norm": 1.0295171737670898, + "learning_rate": 0.0001, + "loss": 89.608, + "step": 2240 + }, + { + "epoch": 0.36333541914777656, + "grad_norm": 0.9720722436904907, + "learning_rate": 0.0001, + "loss": 88.3624, + "step": 2250 + }, + { + "epoch": 0.36495024323287784, + "grad_norm": 0.839654803276062, + "learning_rate": 0.0001, + "loss": 87.2983, + "step": 2260 + }, + { + "epoch": 0.36656506731797905, + "grad_norm": 1.1600219011306763, + "learning_rate": 0.0001, + "loss": 86.6004, + "step": 2270 + }, + { + "epoch": 0.3681798914030803, + "grad_norm": 1.0513924360275269, + "learning_rate": 0.0001, + "loss": 86.6062, + "step": 2280 + }, + { + "epoch": 0.3697947154881815, + "grad_norm": 0.9728474617004395, + "learning_rate": 0.0001, + "loss": 86.1856, + "step": 2290 + }, + { + "epoch": 0.3714095395732827, + "grad_norm": 1.0196386575698853, + "learning_rate": 0.0001, + "loss": 87.6088, + "step": 2300 + }, + { + "epoch": 0.373024363658384, + "grad_norm": 1.0439444780349731, + "learning_rate": 0.0001, + "loss": 85.2374, + "step": 2310 + }, + { + "epoch": 0.3746391877434852, + "grad_norm": 1.0020275115966797, + "learning_rate": 0.0001, + "loss": 86.0596, + "step": 2320 + }, + { + "epoch": 0.3762540118285864, + "grad_norm": 1.0107691287994385, + "learning_rate": 0.0001, + "loss": 86.9513, + "step": 2330 + }, + { + "epoch": 0.37786883591368764, + "grad_norm": 1.680018663406372, + "learning_rate": 0.0001, + "loss": 87.8885, + "step": 2340 + }, + { + "epoch": 0.37948365999878886, + "grad_norm": 0.9181709885597229, + "learning_rate": 0.0001, + "loss": 86.7662, + "step": 2350 + }, + { + "epoch": 0.38109848408389013, + "grad_norm": 0.9598278403282166, + "learning_rate": 0.0001, + "loss": 88.3775, + "step": 2360 + }, + { + "epoch": 0.38271330816899135, + "grad_norm": 1.0707905292510986, + "learning_rate": 0.0001, + "loss": 86.0997, + "step": 2370 + }, + { + "epoch": 0.38432813225409257, + "grad_norm": 1.125797152519226, + "learning_rate": 0.0001, + "loss": 85.8117, + "step": 2380 + }, + { + "epoch": 0.3859429563391938, + "grad_norm": 1.0251439809799194, + "learning_rate": 0.0001, + "loss": 86.0806, + "step": 2390 + }, + { + "epoch": 0.387557780424295, + "grad_norm": 1.0193125009536743, + "learning_rate": 0.0001, + "loss": 87.264, + "step": 2400 + }, + { + "epoch": 0.3891726045093963, + "grad_norm": 0.9309751391410828, + "learning_rate": 0.0001, + "loss": 85.9187, + "step": 2410 + }, + { + "epoch": 0.3907874285944975, + "grad_norm": 1.1025906801223755, + "learning_rate": 0.0001, + "loss": 86.4365, + "step": 2420 + }, + { + "epoch": 0.3924022526795987, + "grad_norm": 0.9880627393722534, + "learning_rate": 0.0001, + "loss": 86.2303, + "step": 2430 + }, + { + "epoch": 0.39401707676469994, + "grad_norm": 0.9520611763000488, + "learning_rate": 0.0001, + "loss": 84.9035, + "step": 2440 + }, + { + "epoch": 0.39563190084980115, + "grad_norm": 0.8641896843910217, + "learning_rate": 0.0001, + "loss": 85.4754, + "step": 2450 + }, + { + "epoch": 0.39724672493490243, + "grad_norm": 1.1162904500961304, + "learning_rate": 0.0001, + "loss": 85.6767, + "step": 2460 + }, + { + "epoch": 0.39886154902000365, + "grad_norm": 0.9073636531829834, + "learning_rate": 0.0001, + "loss": 85.4198, + "step": 2470 + }, + { + "epoch": 0.40047637310510487, + "grad_norm": 1.1135166883468628, + "learning_rate": 0.0001, + "loss": 86.0884, + "step": 2480 + }, + { + "epoch": 0.4020911971902061, + "grad_norm": 0.9273980259895325, + "learning_rate": 0.0001, + "loss": 85.5473, + "step": 2490 + }, + { + "epoch": 0.4037060212753073, + "grad_norm": 0.8653831481933594, + "learning_rate": 0.0001, + "loss": 87.279, + "step": 2500 + }, + { + "epoch": 0.4053208453604086, + "grad_norm": 0.8535159826278687, + "learning_rate": 0.0001, + "loss": 86.7072, + "step": 2510 + }, + { + "epoch": 0.4069356694455098, + "grad_norm": 0.8864684700965881, + "learning_rate": 0.0001, + "loss": 86.441, + "step": 2520 + }, + { + "epoch": 0.408550493530611, + "grad_norm": 0.8042080998420715, + "learning_rate": 0.0001, + "loss": 83.1706, + "step": 2530 + }, + { + "epoch": 0.41016531761571223, + "grad_norm": 1.0486373901367188, + "learning_rate": 0.0001, + "loss": 83.54, + "step": 2540 + }, + { + "epoch": 0.41178014170081345, + "grad_norm": 0.8087959289550781, + "learning_rate": 0.0001, + "loss": 85.7615, + "step": 2550 + }, + { + "epoch": 0.41339496578591467, + "grad_norm": 1.5032018423080444, + "learning_rate": 0.0001, + "loss": 85.7368, + "step": 2560 + }, + { + "epoch": 0.41500978987101594, + "grad_norm": 1.0786182880401611, + "learning_rate": 0.0001, + "loss": 86.3484, + "step": 2570 + }, + { + "epoch": 0.41662461395611716, + "grad_norm": 0.8744294047355652, + "learning_rate": 0.0001, + "loss": 83.5945, + "step": 2580 + }, + { + "epoch": 0.4182394380412184, + "grad_norm": 0.7840721607208252, + "learning_rate": 0.0001, + "loss": 86.3831, + "step": 2590 + }, + { + "epoch": 0.4198542621263196, + "grad_norm": 0.7977836728096008, + "learning_rate": 0.0001, + "loss": 83.6314, + "step": 2600 + }, + { + "epoch": 0.4214690862114208, + "grad_norm": 0.892484188079834, + "learning_rate": 0.0001, + "loss": 86.4223, + "step": 2610 + }, + { + "epoch": 0.4230839102965221, + "grad_norm": 0.912995457649231, + "learning_rate": 0.0001, + "loss": 83.8971, + "step": 2620 + }, + { + "epoch": 0.4246987343816233, + "grad_norm": 0.9566861987113953, + "learning_rate": 0.0001, + "loss": 85.291, + "step": 2630 + }, + { + "epoch": 0.42631355846672453, + "grad_norm": 0.8324154615402222, + "learning_rate": 0.0001, + "loss": 84.6233, + "step": 2640 + }, + { + "epoch": 0.42792838255182575, + "grad_norm": 0.9490360617637634, + "learning_rate": 0.0001, + "loss": 81.799, + "step": 2650 + }, + { + "epoch": 0.42954320663692697, + "grad_norm": 0.9744566679000854, + "learning_rate": 0.0001, + "loss": 85.4837, + "step": 2660 + }, + { + "epoch": 0.43115803072202824, + "grad_norm": 0.941614031791687, + "learning_rate": 0.0001, + "loss": 82.838, + "step": 2670 + }, + { + "epoch": 0.43277285480712946, + "grad_norm": 1.338364601135254, + "learning_rate": 0.0001, + "loss": 84.4224, + "step": 2680 + }, + { + "epoch": 0.4343876788922307, + "grad_norm": 0.8643816113471985, + "learning_rate": 0.0001, + "loss": 84.3959, + "step": 2690 + }, + { + "epoch": 0.4360025029773319, + "grad_norm": 1.0231473445892334, + "learning_rate": 0.0001, + "loss": 82.3144, + "step": 2700 + }, + { + "epoch": 0.4376173270624331, + "grad_norm": 0.9089650511741638, + "learning_rate": 0.0001, + "loss": 83.0326, + "step": 2710 + }, + { + "epoch": 0.4392321511475344, + "grad_norm": 0.893628716468811, + "learning_rate": 0.0001, + "loss": 83.0981, + "step": 2720 + }, + { + "epoch": 0.4408469752326356, + "grad_norm": 0.869076132774353, + "learning_rate": 0.0001, + "loss": 84.2571, + "step": 2730 + }, + { + "epoch": 0.4424617993177368, + "grad_norm": 0.7964235544204712, + "learning_rate": 0.0001, + "loss": 83.3329, + "step": 2740 + }, + { + "epoch": 0.44407662340283804, + "grad_norm": 3.1645138263702393, + "learning_rate": 0.0001, + "loss": 83.7232, + "step": 2750 + }, + { + "epoch": 0.44569144748793926, + "grad_norm": 0.934644877910614, + "learning_rate": 0.0001, + "loss": 85.0688, + "step": 2760 + }, + { + "epoch": 0.44730627157304054, + "grad_norm": 0.8120711445808411, + "learning_rate": 0.0001, + "loss": 82.7706, + "step": 2770 + }, + { + "epoch": 0.44892109565814176, + "grad_norm": 0.8161435127258301, + "learning_rate": 0.0001, + "loss": 84.4826, + "step": 2780 + }, + { + "epoch": 0.450535919743243, + "grad_norm": 0.8659493327140808, + "learning_rate": 0.0001, + "loss": 82.6222, + "step": 2790 + }, + { + "epoch": 0.4521507438283442, + "grad_norm": 0.9306702613830566, + "learning_rate": 0.0001, + "loss": 83.5897, + "step": 2800 + }, + { + "epoch": 0.4537655679134454, + "grad_norm": 2.2544431686401367, + "learning_rate": 0.0001, + "loss": 82.9603, + "step": 2810 + }, + { + "epoch": 0.4553803919985467, + "grad_norm": 0.8868395686149597, + "learning_rate": 0.0001, + "loss": 82.6674, + "step": 2820 + }, + { + "epoch": 0.4569952160836479, + "grad_norm": 0.9336455464363098, + "learning_rate": 0.0001, + "loss": 84.6974, + "step": 2830 + }, + { + "epoch": 0.4586100401687491, + "grad_norm": 0.8814165592193604, + "learning_rate": 0.0001, + "loss": 83.9329, + "step": 2840 + }, + { + "epoch": 0.46022486425385034, + "grad_norm": 0.8125660419464111, + "learning_rate": 0.0001, + "loss": 84.6168, + "step": 2850 + }, + { + "epoch": 0.46183968833895156, + "grad_norm": 0.9447190165519714, + "learning_rate": 0.0001, + "loss": 82.4098, + "step": 2860 + }, + { + "epoch": 0.4634545124240528, + "grad_norm": 0.8080969452857971, + "learning_rate": 0.0001, + "loss": 83.4076, + "step": 2870 + }, + { + "epoch": 0.46506933650915405, + "grad_norm": 1.0586694478988647, + "learning_rate": 0.0001, + "loss": 83.3295, + "step": 2880 + }, + { + "epoch": 0.46668416059425527, + "grad_norm": 0.8207664489746094, + "learning_rate": 0.0001, + "loss": 84.5056, + "step": 2890 + }, + { + "epoch": 0.4682989846793565, + "grad_norm": 1.0647571086883545, + "learning_rate": 0.0001, + "loss": 83.0766, + "step": 2900 + }, + { + "epoch": 0.4699138087644577, + "grad_norm": 0.9416915774345398, + "learning_rate": 0.0001, + "loss": 83.8166, + "step": 2910 + }, + { + "epoch": 0.4715286328495589, + "grad_norm": 0.7479362487792969, + "learning_rate": 0.0001, + "loss": 82.7507, + "step": 2920 + }, + { + "epoch": 0.4731434569346602, + "grad_norm": 1.0775514841079712, + "learning_rate": 0.0001, + "loss": 84.4641, + "step": 2930 + }, + { + "epoch": 0.4747582810197614, + "grad_norm": 0.8736841082572937, + "learning_rate": 0.0001, + "loss": 82.0451, + "step": 2940 + }, + { + "epoch": 0.47637310510486264, + "grad_norm": 0.9500495791435242, + "learning_rate": 0.0001, + "loss": 82.6425, + "step": 2950 + }, + { + "epoch": 0.47798792918996386, + "grad_norm": 0.8551210165023804, + "learning_rate": 0.0001, + "loss": 81.1971, + "step": 2960 + }, + { + "epoch": 0.4796027532750651, + "grad_norm": 0.8149814605712891, + "learning_rate": 0.0001, + "loss": 82.7919, + "step": 2970 + }, + { + "epoch": 0.48121757736016635, + "grad_norm": 0.9012249112129211, + "learning_rate": 0.0001, + "loss": 81.975, + "step": 2980 + }, + { + "epoch": 0.48283240144526757, + "grad_norm": 1.3784620761871338, + "learning_rate": 0.0001, + "loss": 81.541, + "step": 2990 + }, + { + "epoch": 0.4844472255303688, + "grad_norm": 0.7921801209449768, + "learning_rate": 0.0001, + "loss": 83.6215, + "step": 3000 + }, + { + "epoch": 0.48606204961547, + "grad_norm": 0.9051580429077148, + "learning_rate": 0.0001, + "loss": 82.4048, + "step": 3010 + }, + { + "epoch": 0.4876768737005712, + "grad_norm": 1.0717192888259888, + "learning_rate": 0.0001, + "loss": 82.4079, + "step": 3020 + }, + { + "epoch": 0.4892916977856725, + "grad_norm": 0.8971781730651855, + "learning_rate": 0.0001, + "loss": 83.3668, + "step": 3030 + }, + { + "epoch": 0.4909065218707737, + "grad_norm": 0.9635320901870728, + "learning_rate": 0.0001, + "loss": 82.8244, + "step": 3040 + }, + { + "epoch": 0.49252134595587493, + "grad_norm": 1.1009949445724487, + "learning_rate": 0.0001, + "loss": 82.2174, + "step": 3050 + }, + { + "epoch": 0.49413617004097615, + "grad_norm": 0.9805473685264587, + "learning_rate": 0.0001, + "loss": 83.7897, + "step": 3060 + }, + { + "epoch": 0.49575099412607737, + "grad_norm": 1.2107361555099487, + "learning_rate": 0.0001, + "loss": 82.5108, + "step": 3070 + }, + { + "epoch": 0.49736581821117865, + "grad_norm": 0.9171829223632812, + "learning_rate": 0.0001, + "loss": 80.9126, + "step": 3080 + }, + { + "epoch": 0.49898064229627986, + "grad_norm": 0.8093295097351074, + "learning_rate": 0.0001, + "loss": 83.8147, + "step": 3090 + }, + { + "epoch": 0.5005954663813811, + "grad_norm": 1.2166653871536255, + "learning_rate": 0.0001, + "loss": 82.0288, + "step": 3100 + }, + { + "epoch": 0.5022102904664824, + "grad_norm": 1.0471502542495728, + "learning_rate": 0.0001, + "loss": 82.0706, + "step": 3110 + }, + { + "epoch": 0.5038251145515835, + "grad_norm": 1.007645845413208, + "learning_rate": 0.0001, + "loss": 81.7387, + "step": 3120 + }, + { + "epoch": 0.5054399386366848, + "grad_norm": 0.7252638339996338, + "learning_rate": 0.0001, + "loss": 82.9107, + "step": 3130 + }, + { + "epoch": 0.507054762721786, + "grad_norm": 0.8565683364868164, + "learning_rate": 0.0001, + "loss": 80.2935, + "step": 3140 + }, + { + "epoch": 0.5086695868068872, + "grad_norm": 0.9905404448509216, + "learning_rate": 0.0001, + "loss": 82.5201, + "step": 3150 + }, + { + "epoch": 0.5102844108919885, + "grad_norm": 0.887428343296051, + "learning_rate": 0.0001, + "loss": 80.8528, + "step": 3160 + }, + { + "epoch": 0.5118992349770897, + "grad_norm": 1.062942624092102, + "learning_rate": 0.0001, + "loss": 80.7035, + "step": 3170 + }, + { + "epoch": 0.5135140590621909, + "grad_norm": 1.203599214553833, + "learning_rate": 0.0001, + "loss": 81.2521, + "step": 3180 + }, + { + "epoch": 0.5151288831472921, + "grad_norm": 0.8452383875846863, + "learning_rate": 0.0001, + "loss": 81.6079, + "step": 3190 + }, + { + "epoch": 0.5167437072323934, + "grad_norm": 1.2530587911605835, + "learning_rate": 0.0001, + "loss": 82.2148, + "step": 3200 + }, + { + "epoch": 0.5183585313174947, + "grad_norm": 0.9351627826690674, + "learning_rate": 0.0001, + "loss": 82.7857, + "step": 3210 + }, + { + "epoch": 0.5199733554025958, + "grad_norm": 1.0128077268600464, + "learning_rate": 0.0001, + "loss": 81.0532, + "step": 3220 + }, + { + "epoch": 0.5215881794876971, + "grad_norm": 0.9316917061805725, + "learning_rate": 0.0001, + "loss": 80.7443, + "step": 3230 + }, + { + "epoch": 0.5232030035727983, + "grad_norm": 1.502705693244934, + "learning_rate": 0.0001, + "loss": 80.0608, + "step": 3240 + }, + { + "epoch": 0.5248178276578995, + "grad_norm": 0.8166837692260742, + "learning_rate": 0.0001, + "loss": 81.2013, + "step": 3250 + }, + { + "epoch": 0.5264326517430008, + "grad_norm": 0.8210042119026184, + "learning_rate": 0.0001, + "loss": 80.2491, + "step": 3260 + }, + { + "epoch": 0.528047475828102, + "grad_norm": 0.7854771018028259, + "learning_rate": 0.0001, + "loss": 81.1017, + "step": 3270 + }, + { + "epoch": 0.5296622999132032, + "grad_norm": 0.8530565500259399, + "learning_rate": 0.0001, + "loss": 80.8123, + "step": 3280 + }, + { + "epoch": 0.5312771239983044, + "grad_norm": 0.9446498155593872, + "learning_rate": 0.0001, + "loss": 80.0029, + "step": 3290 + }, + { + "epoch": 0.5328919480834057, + "grad_norm": 0.9646537899971008, + "learning_rate": 0.0001, + "loss": 81.5413, + "step": 3300 + }, + { + "epoch": 0.5345067721685068, + "grad_norm": 0.9262019395828247, + "learning_rate": 0.0001, + "loss": 80.9399, + "step": 3310 + }, + { + "epoch": 0.5361215962536081, + "grad_norm": 0.8515486121177673, + "learning_rate": 0.0001, + "loss": 80.9421, + "step": 3320 + }, + { + "epoch": 0.5377364203387094, + "grad_norm": 1.679321050643921, + "learning_rate": 0.0001, + "loss": 82.4851, + "step": 3330 + }, + { + "epoch": 0.5393512444238105, + "grad_norm": 0.8289812207221985, + "learning_rate": 0.0001, + "loss": 80.5664, + "step": 3340 + }, + { + "epoch": 0.5409660685089118, + "grad_norm": 0.7463789582252502, + "learning_rate": 0.0001, + "loss": 81.5193, + "step": 3350 + }, + { + "epoch": 0.542580892594013, + "grad_norm": 0.8332446217536926, + "learning_rate": 0.0001, + "loss": 80.8091, + "step": 3360 + }, + { + "epoch": 0.5441957166791143, + "grad_norm": 0.7612847089767456, + "learning_rate": 0.0001, + "loss": 79.8893, + "step": 3370 + }, + { + "epoch": 0.5458105407642155, + "grad_norm": 1.0557948350906372, + "learning_rate": 0.0001, + "loss": 81.4305, + "step": 3380 + }, + { + "epoch": 0.5474253648493167, + "grad_norm": 0.7975788116455078, + "learning_rate": 0.0001, + "loss": 80.0105, + "step": 3390 + }, + { + "epoch": 0.549040188934418, + "grad_norm": 0.9049227237701416, + "learning_rate": 0.0001, + "loss": 81.2689, + "step": 3400 + }, + { + "epoch": 0.5506550130195191, + "grad_norm": 1.1195895671844482, + "learning_rate": 0.0001, + "loss": 80.7163, + "step": 3410 + }, + { + "epoch": 0.5522698371046204, + "grad_norm": 0.7506540417671204, + "learning_rate": 0.0001, + "loss": 79.0855, + "step": 3420 + }, + { + "epoch": 0.5538846611897217, + "grad_norm": 0.73989337682724, + "learning_rate": 0.0001, + "loss": 81.1865, + "step": 3430 + }, + { + "epoch": 0.5554994852748228, + "grad_norm": 0.8418401479721069, + "learning_rate": 0.0001, + "loss": 79.5714, + "step": 3440 + }, + { + "epoch": 0.5571143093599241, + "grad_norm": 0.9807226061820984, + "learning_rate": 0.0001, + "loss": 78.5699, + "step": 3450 + }, + { + "epoch": 0.5587291334450253, + "grad_norm": 1.2890545129776, + "learning_rate": 0.0001, + "loss": 81.0128, + "step": 3460 + }, + { + "epoch": 0.5603439575301266, + "grad_norm": 0.8593361973762512, + "learning_rate": 0.0001, + "loss": 80.1638, + "step": 3470 + }, + { + "epoch": 0.5619587816152278, + "grad_norm": 0.7766354084014893, + "learning_rate": 0.0001, + "loss": 81.5597, + "step": 3480 + }, + { + "epoch": 0.563573605700329, + "grad_norm": 0.7785626649856567, + "learning_rate": 0.0001, + "loss": 79.3862, + "step": 3490 + }, + { + "epoch": 0.5651884297854303, + "grad_norm": 0.8429279327392578, + "learning_rate": 0.0001, + "loss": 76.6294, + "step": 3500 + }, + { + "epoch": 0.5668032538705314, + "grad_norm": 0.8141765594482422, + "learning_rate": 0.0001, + "loss": 80.2584, + "step": 3510 + }, + { + "epoch": 0.5684180779556327, + "grad_norm": 0.7065490484237671, + "learning_rate": 0.0001, + "loss": 79.3626, + "step": 3520 + }, + { + "epoch": 0.570032902040734, + "grad_norm": 0.7002031803131104, + "learning_rate": 0.0001, + "loss": 79.283, + "step": 3530 + }, + { + "epoch": 0.5716477261258351, + "grad_norm": 0.8715317845344543, + "learning_rate": 0.0001, + "loss": 80.3145, + "step": 3540 + }, + { + "epoch": 0.5732625502109364, + "grad_norm": 0.9584882855415344, + "learning_rate": 0.0001, + "loss": 78.7443, + "step": 3550 + }, + { + "epoch": 0.5748773742960376, + "grad_norm": 0.7908946871757507, + "learning_rate": 0.0001, + "loss": 79.1906, + "step": 3560 + }, + { + "epoch": 0.5764921983811389, + "grad_norm": 0.8416746258735657, + "learning_rate": 0.0001, + "loss": 77.8172, + "step": 3570 + }, + { + "epoch": 0.5781070224662401, + "grad_norm": 0.823667049407959, + "learning_rate": 0.0001, + "loss": 81.864, + "step": 3580 + }, + { + "epoch": 0.5797218465513413, + "grad_norm": 1.4282617568969727, + "learning_rate": 0.0001, + "loss": 79.9052, + "step": 3590 + }, + { + "epoch": 0.5813366706364426, + "grad_norm": 0.8747813701629639, + "learning_rate": 0.0001, + "loss": 78.8234, + "step": 3600 + }, + { + "epoch": 0.5829514947215437, + "grad_norm": 0.8415511846542358, + "learning_rate": 0.0001, + "loss": 78.8171, + "step": 3610 + }, + { + "epoch": 0.584566318806645, + "grad_norm": 0.9014492034912109, + "learning_rate": 0.0001, + "loss": 80.5011, + "step": 3620 + }, + { + "epoch": 0.5861811428917463, + "grad_norm": 0.7535204887390137, + "learning_rate": 0.0001, + "loss": 79.4895, + "step": 3630 + }, + { + "epoch": 0.5877959669768474, + "grad_norm": 0.8196508884429932, + "learning_rate": 0.0001, + "loss": 79.7101, + "step": 3640 + }, + { + "epoch": 0.5894107910619487, + "grad_norm": 1.5975492000579834, + "learning_rate": 0.0001, + "loss": 79.0687, + "step": 3650 + }, + { + "epoch": 0.5910256151470499, + "grad_norm": 0.9251692295074463, + "learning_rate": 0.0001, + "loss": 80.2377, + "step": 3660 + }, + { + "epoch": 0.5926404392321512, + "grad_norm": 0.7143557071685791, + "learning_rate": 0.0001, + "loss": 78.0671, + "step": 3670 + }, + { + "epoch": 0.5942552633172524, + "grad_norm": 0.7876290678977966, + "learning_rate": 0.0001, + "loss": 79.1455, + "step": 3680 + }, + { + "epoch": 0.5958700874023536, + "grad_norm": 1.5361895561218262, + "learning_rate": 0.0001, + "loss": 78.1253, + "step": 3690 + }, + { + "epoch": 0.5974849114874549, + "grad_norm": 0.7637040019035339, + "learning_rate": 0.0001, + "loss": 77.4658, + "step": 3700 + }, + { + "epoch": 0.599099735572556, + "grad_norm": 0.8166250586509705, + "learning_rate": 0.0001, + "loss": 79.5806, + "step": 3710 + }, + { + "epoch": 0.6007145596576573, + "grad_norm": 1.2767380475997925, + "learning_rate": 0.0001, + "loss": 79.1894, + "step": 3720 + }, + { + "epoch": 0.6023293837427586, + "grad_norm": 1.3048640489578247, + "learning_rate": 0.0001, + "loss": 78.0593, + "step": 3730 + }, + { + "epoch": 0.6039442078278597, + "grad_norm": 0.7945445775985718, + "learning_rate": 0.0001, + "loss": 79.7623, + "step": 3740 + }, + { + "epoch": 0.605559031912961, + "grad_norm": 0.7366836667060852, + "learning_rate": 0.0001, + "loss": 77.9571, + "step": 3750 + }, + { + "epoch": 0.6071738559980622, + "grad_norm": 0.9126115441322327, + "learning_rate": 0.0001, + "loss": 79.9502, + "step": 3760 + }, + { + "epoch": 0.6087886800831634, + "grad_norm": 0.800355851650238, + "learning_rate": 0.0001, + "loss": 78.7244, + "step": 3770 + }, + { + "epoch": 0.6104035041682647, + "grad_norm": 0.8290980458259583, + "learning_rate": 0.0001, + "loss": 78.7724, + "step": 3780 + }, + { + "epoch": 0.6120183282533659, + "grad_norm": 0.8512316942214966, + "learning_rate": 0.0001, + "loss": 78.6277, + "step": 3790 + }, + { + "epoch": 0.6136331523384672, + "grad_norm": 0.8909353613853455, + "learning_rate": 0.0001, + "loss": 78.5956, + "step": 3800 + }, + { + "epoch": 0.6152479764235683, + "grad_norm": 0.8328800797462463, + "learning_rate": 0.0001, + "loss": 78.1323, + "step": 3810 + }, + { + "epoch": 0.6168628005086696, + "grad_norm": 0.7950423359870911, + "learning_rate": 0.0001, + "loss": 78.7529, + "step": 3820 + }, + { + "epoch": 0.6184776245937709, + "grad_norm": 0.9988545179367065, + "learning_rate": 0.0001, + "loss": 80.7727, + "step": 3830 + }, + { + "epoch": 0.620092448678872, + "grad_norm": 0.7243085503578186, + "learning_rate": 0.0001, + "loss": 76.7616, + "step": 3840 + }, + { + "epoch": 0.6217072727639733, + "grad_norm": 0.7674855589866638, + "learning_rate": 0.0001, + "loss": 79.1938, + "step": 3850 + }, + { + "epoch": 0.6233220968490745, + "grad_norm": 0.8979483842849731, + "learning_rate": 0.0001, + "loss": 79.1522, + "step": 3860 + }, + { + "epoch": 0.6249369209341757, + "grad_norm": 0.7759745717048645, + "learning_rate": 0.0001, + "loss": 77.7001, + "step": 3870 + }, + { + "epoch": 0.626551745019277, + "grad_norm": 0.9940371513366699, + "learning_rate": 0.0001, + "loss": 79.8712, + "step": 3880 + }, + { + "epoch": 0.6281665691043782, + "grad_norm": 0.8103354573249817, + "learning_rate": 0.0001, + "loss": 77.5229, + "step": 3890 + }, + { + "epoch": 0.6297813931894795, + "grad_norm": 0.9188411831855774, + "learning_rate": 0.0001, + "loss": 78.2933, + "step": 3900 + }, + { + "epoch": 0.6313962172745806, + "grad_norm": 0.823017954826355, + "learning_rate": 0.0001, + "loss": 77.8692, + "step": 3910 + }, + { + "epoch": 0.6330110413596819, + "grad_norm": 0.8833358287811279, + "learning_rate": 0.0001, + "loss": 78.4465, + "step": 3920 + }, + { + "epoch": 0.634625865444783, + "grad_norm": 0.6880685687065125, + "learning_rate": 0.0001, + "loss": 78.8881, + "step": 3930 + }, + { + "epoch": 0.6362406895298843, + "grad_norm": 0.778455913066864, + "learning_rate": 0.0001, + "loss": 76.0277, + "step": 3940 + }, + { + "epoch": 0.6378555136149856, + "grad_norm": 0.8132772445678711, + "learning_rate": 0.0001, + "loss": 79.1284, + "step": 3950 + }, + { + "epoch": 0.6394703377000868, + "grad_norm": 0.7703788876533508, + "learning_rate": 0.0001, + "loss": 76.9013, + "step": 3960 + }, + { + "epoch": 0.641085161785188, + "grad_norm": 0.7876151204109192, + "learning_rate": 0.0001, + "loss": 77.1354, + "step": 3970 + }, + { + "epoch": 0.6426999858702892, + "grad_norm": 0.8027064800262451, + "learning_rate": 0.0001, + "loss": 77.4936, + "step": 3980 + }, + { + "epoch": 0.6443148099553905, + "grad_norm": 0.8261818885803223, + "learning_rate": 0.0001, + "loss": 76.7634, + "step": 3990 + }, + { + "epoch": 0.6459296340404918, + "grad_norm": 0.7024852633476257, + "learning_rate": 0.0001, + "loss": 79.951, + "step": 4000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.898004756175258e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6a151f546555c78ebc502dd7b703b665f0672d6f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35235df8408202e91954a585851d27187bafaf4e1d75c4e63c51fbf721136ff5 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-4000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/latest b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/latest new file mode 100644 index 0000000000000000000000000000000000000000..3ccb5110452b4a1eef79a1c432a18944dc25a985 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/latest @@ -0,0 +1 @@ +global_step6000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fd5ff0bfff426e726e4b87c5e55e9ea59039e6f4 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0288f1457b0ca2f0dc1b35a2ff19811272d60a9474015d4533abbf42b625b57b +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..aa8dac603840980686dc11eb6d1c8132191643e9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58427bc21238099267373e6fd46e2c534c2dc5082b23a46fc9cbf54233a6d8a2 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..5784a0bcd7786e60dc11af7005e3055cdd8425d7 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bac1a700fa7f3dbd4728ab9a8f0e21dd657aeae013cd0bf56b2ad30af57fe3a +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..c4b0e3c8e757a5eaecc00fed0bb5e895240b3732 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6b1ef3d238eafd1af06bbd498e334bcf8e4f3298e2622f19b4343b31c09ada5 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..6c17b8ad8c6855d79bc449b1e1bd2e8a78aa94b0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6976f3937958369c6adf3bc59c384f881ea76fb974e7653af6331831b9f3e3ee +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..0e7b3b8235c63d72ec7ec025752e908bc919289d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:738c6738c533b49aef16ca9a8ce5ba88f9dac77e98172f08e72d1ee67ef46b3e +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..01032428dd630e29b0a85867b6f8d399fd56fd3f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26c189241a062d02c81d1f737d24b56b778fa1e493084947f7323c295f53e08b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..1feece2d2e87541d8a1db5e8eabdf1ca2e6317ce --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fada4f4f34e5bcef1cca8dee4ce512a5cfb51d619b9acd52b23f6e60553bb13c +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..bafaf1cfdc84004755e0a6571e80d5de33ce59a4 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9067d9bec3afb7610bb1a6c3e931f718d3fe32fe96f8fa9eaf048061dd0cead7 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..a76cd8e8b9ba139f39ea2186e82134fa532ba2e8 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e7682127f80cd25a98dfdc8b12ba353464983fd8dca3440f4fbf60a3e51c0a5 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2c39a6a409f7d6614c407f4f0fc40d2d21eb4838 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/trainer_state.json @@ -0,0 +1,4234 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9688944510607376, + "eval_steps": 2000, + "global_step": 6000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0016148240851012292, + "grad_norm": 43.90101623535156, + "learning_rate": 0.0001, + "loss": 307.2084, + "step": 10 + }, + { + "epoch": 0.0032296481702024584, + "grad_norm": 19.106313705444336, + "learning_rate": 0.0001, + "loss": 320.2794, + "step": 20 + }, + { + "epoch": 0.004844472255303688, + "grad_norm": 5.771847248077393, + "learning_rate": 0.0001, + "loss": 247.4977, + "step": 30 + }, + { + "epoch": 0.006459296340404917, + "grad_norm": 2.630659580230713, + "learning_rate": 0.0001, + "loss": 229.9593, + "step": 40 + }, + { + "epoch": 0.008074120425506147, + "grad_norm": 2.487499475479126, + "learning_rate": 0.0001, + "loss": 228.65, + "step": 50 + }, + { + "epoch": 0.009688944510607376, + "grad_norm": 2.078263521194458, + "learning_rate": 0.0001, + "loss": 227.648, + "step": 60 + }, + { + "epoch": 0.011303768595708604, + "grad_norm": 3.105417490005493, + "learning_rate": 0.0001, + "loss": 227.9345, + "step": 70 + }, + { + "epoch": 0.012918592680809833, + "grad_norm": 2.2145848274230957, + "learning_rate": 0.0001, + "loss": 227.3662, + "step": 80 + }, + { + "epoch": 0.014533416765911064, + "grad_norm": 1.9644615650177002, + "learning_rate": 0.0001, + "loss": 226.9145, + "step": 90 + }, + { + "epoch": 0.016148240851012293, + "grad_norm": 1.867382287979126, + "learning_rate": 0.0001, + "loss": 227.5703, + "step": 100 + }, + { + "epoch": 0.017763064936113522, + "grad_norm": 1.4613710641860962, + "learning_rate": 0.0001, + "loss": 227.1107, + "step": 110 + }, + { + "epoch": 0.01937788902121475, + "grad_norm": 2.8953869342803955, + "learning_rate": 0.0001, + "loss": 226.8185, + "step": 120 + }, + { + "epoch": 0.02099271310631598, + "grad_norm": 1.8446482419967651, + "learning_rate": 0.0001, + "loss": 226.9624, + "step": 130 + }, + { + "epoch": 0.02260753719141721, + "grad_norm": 1.3048362731933594, + "learning_rate": 0.0001, + "loss": 226.8404, + "step": 140 + }, + { + "epoch": 0.024222361276518438, + "grad_norm": 1.7100074291229248, + "learning_rate": 0.0001, + "loss": 226.1071, + "step": 150 + }, + { + "epoch": 0.025837185361619667, + "grad_norm": 1.7875330448150635, + "learning_rate": 0.0001, + "loss": 226.0764, + "step": 160 + }, + { + "epoch": 0.0274520094467209, + "grad_norm": 1.7503769397735596, + "learning_rate": 0.0001, + "loss": 226.0772, + "step": 170 + }, + { + "epoch": 0.029066833531822128, + "grad_norm": 2.7083983421325684, + "learning_rate": 0.0001, + "loss": 224.7667, + "step": 180 + }, + { + "epoch": 0.030681657616923357, + "grad_norm": 1.8192371129989624, + "learning_rate": 0.0001, + "loss": 223.6494, + "step": 190 + }, + { + "epoch": 0.032296481702024586, + "grad_norm": 3.210358142852783, + "learning_rate": 0.0001, + "loss": 223.3545, + "step": 200 + }, + { + "epoch": 0.03391130578712581, + "grad_norm": 5.819669246673584, + "learning_rate": 0.0001, + "loss": 222.1732, + "step": 210 + }, + { + "epoch": 0.035526129872227044, + "grad_norm": 4.185793399810791, + "learning_rate": 0.0001, + "loss": 221.6866, + "step": 220 + }, + { + "epoch": 0.03714095395732828, + "grad_norm": 2.7514216899871826, + "learning_rate": 0.0001, + "loss": 221.0769, + "step": 230 + }, + { + "epoch": 0.0387557780424295, + "grad_norm": 1.5478988885879517, + "learning_rate": 0.0001, + "loss": 219.7232, + "step": 240 + }, + { + "epoch": 0.040370602127530734, + "grad_norm": 1.4958223104476929, + "learning_rate": 0.0001, + "loss": 218.5454, + "step": 250 + }, + { + "epoch": 0.04198542621263196, + "grad_norm": 6.36635684967041, + "learning_rate": 0.0001, + "loss": 218.5061, + "step": 260 + }, + { + "epoch": 0.04360025029773319, + "grad_norm": 3.029601812362671, + "learning_rate": 0.0001, + "loss": 216.3466, + "step": 270 + }, + { + "epoch": 0.04521507438283442, + "grad_norm": 2.6436100006103516, + "learning_rate": 0.0001, + "loss": 214.0003, + "step": 280 + }, + { + "epoch": 0.04682989846793565, + "grad_norm": 5.329655647277832, + "learning_rate": 0.0001, + "loss": 212.9169, + "step": 290 + }, + { + "epoch": 0.048444722553036876, + "grad_norm": 4.930875301361084, + "learning_rate": 0.0001, + "loss": 209.7737, + "step": 300 + }, + { + "epoch": 0.05005954663813811, + "grad_norm": 18.902585983276367, + "learning_rate": 0.0001, + "loss": 209.4267, + "step": 310 + }, + { + "epoch": 0.051674370723239334, + "grad_norm": 5.364975929260254, + "learning_rate": 0.0001, + "loss": 206.49, + "step": 320 + }, + { + "epoch": 0.053289194808340566, + "grad_norm": 3.1421146392822266, + "learning_rate": 0.0001, + "loss": 204.8132, + "step": 330 + }, + { + "epoch": 0.0549040188934418, + "grad_norm": 3.399320602416992, + "learning_rate": 0.0001, + "loss": 202.4785, + "step": 340 + }, + { + "epoch": 0.056518842978543024, + "grad_norm": 5.137377738952637, + "learning_rate": 0.0001, + "loss": 199.5143, + "step": 350 + }, + { + "epoch": 0.058133667063644257, + "grad_norm": 5.4967522621154785, + "learning_rate": 0.0001, + "loss": 196.2809, + "step": 360 + }, + { + "epoch": 0.05974849114874548, + "grad_norm": 6.435070037841797, + "learning_rate": 0.0001, + "loss": 196.8831, + "step": 370 + }, + { + "epoch": 0.061363315233846714, + "grad_norm": 5.7334370613098145, + "learning_rate": 0.0001, + "loss": 193.141, + "step": 380 + }, + { + "epoch": 0.06297813931894794, + "grad_norm": 3.6522979736328125, + "learning_rate": 0.0001, + "loss": 191.5819, + "step": 390 + }, + { + "epoch": 0.06459296340404917, + "grad_norm": 2.614051103591919, + "learning_rate": 0.0001, + "loss": 190.046, + "step": 400 + }, + { + "epoch": 0.0662077874891504, + "grad_norm": 4.845924377441406, + "learning_rate": 0.0001, + "loss": 188.7121, + "step": 410 + }, + { + "epoch": 0.06782261157425162, + "grad_norm": 4.319681644439697, + "learning_rate": 0.0001, + "loss": 185.3697, + "step": 420 + }, + { + "epoch": 0.06943743565935286, + "grad_norm": 2.7018792629241943, + "learning_rate": 0.0001, + "loss": 183.5171, + "step": 430 + }, + { + "epoch": 0.07105225974445409, + "grad_norm": 2.6407113075256348, + "learning_rate": 0.0001, + "loss": 181.245, + "step": 440 + }, + { + "epoch": 0.07266708382955532, + "grad_norm": 3.314251184463501, + "learning_rate": 0.0001, + "loss": 179.3874, + "step": 450 + }, + { + "epoch": 0.07428190791465655, + "grad_norm": 2.635136365890503, + "learning_rate": 0.0001, + "loss": 177.8257, + "step": 460 + }, + { + "epoch": 0.07589673199975777, + "grad_norm": 5.227738857269287, + "learning_rate": 0.0001, + "loss": 176.4699, + "step": 470 + }, + { + "epoch": 0.077511556084859, + "grad_norm": 4.998415470123291, + "learning_rate": 0.0001, + "loss": 175.7601, + "step": 480 + }, + { + "epoch": 0.07912638016996024, + "grad_norm": 2.946892261505127, + "learning_rate": 0.0001, + "loss": 173.936, + "step": 490 + }, + { + "epoch": 0.08074120425506147, + "grad_norm": 4.20358943939209, + "learning_rate": 0.0001, + "loss": 172.065, + "step": 500 + }, + { + "epoch": 0.08235602834016269, + "grad_norm": 3.125765562057495, + "learning_rate": 0.0001, + "loss": 171.3727, + "step": 510 + }, + { + "epoch": 0.08397085242526392, + "grad_norm": 2.4870612621307373, + "learning_rate": 0.0001, + "loss": 169.9, + "step": 520 + }, + { + "epoch": 0.08558567651036515, + "grad_norm": 3.917384147644043, + "learning_rate": 0.0001, + "loss": 168.5049, + "step": 530 + }, + { + "epoch": 0.08720050059546638, + "grad_norm": 2.302142858505249, + "learning_rate": 0.0001, + "loss": 167.2027, + "step": 540 + }, + { + "epoch": 0.08881532468056762, + "grad_norm": 4.794841289520264, + "learning_rate": 0.0001, + "loss": 166.0887, + "step": 550 + }, + { + "epoch": 0.09043014876566884, + "grad_norm": 2.200291395187378, + "learning_rate": 0.0001, + "loss": 164.5882, + "step": 560 + }, + { + "epoch": 0.09204497285077007, + "grad_norm": 1.9572436809539795, + "learning_rate": 0.0001, + "loss": 163.3007, + "step": 570 + }, + { + "epoch": 0.0936597969358713, + "grad_norm": 3.443633556365967, + "learning_rate": 0.0001, + "loss": 160.6805, + "step": 580 + }, + { + "epoch": 0.09527462102097253, + "grad_norm": 2.6079020500183105, + "learning_rate": 0.0001, + "loss": 160.3372, + "step": 590 + }, + { + "epoch": 0.09688944510607375, + "grad_norm": 2.4401192665100098, + "learning_rate": 0.0001, + "loss": 160.0956, + "step": 600 + }, + { + "epoch": 0.09850426919117498, + "grad_norm": 2.686101198196411, + "learning_rate": 0.0001, + "loss": 157.6138, + "step": 610 + }, + { + "epoch": 0.10011909327627622, + "grad_norm": 2.594151496887207, + "learning_rate": 0.0001, + "loss": 156.0726, + "step": 620 + }, + { + "epoch": 0.10173391736137745, + "grad_norm": 2.509141683578491, + "learning_rate": 0.0001, + "loss": 155.9923, + "step": 630 + }, + { + "epoch": 0.10334874144647867, + "grad_norm": 2.4077377319335938, + "learning_rate": 0.0001, + "loss": 155.214, + "step": 640 + }, + { + "epoch": 0.1049635655315799, + "grad_norm": 4.083552837371826, + "learning_rate": 0.0001, + "loss": 152.8257, + "step": 650 + }, + { + "epoch": 0.10657838961668113, + "grad_norm": 2.7188713550567627, + "learning_rate": 0.0001, + "loss": 151.875, + "step": 660 + }, + { + "epoch": 0.10819321370178236, + "grad_norm": 2.41287899017334, + "learning_rate": 0.0001, + "loss": 151.935, + "step": 670 + }, + { + "epoch": 0.1098080377868836, + "grad_norm": 3.995708465576172, + "learning_rate": 0.0001, + "loss": 151.0069, + "step": 680 + }, + { + "epoch": 0.11142286187198482, + "grad_norm": 3.217705488204956, + "learning_rate": 0.0001, + "loss": 149.5922, + "step": 690 + }, + { + "epoch": 0.11303768595708605, + "grad_norm": 3.4349842071533203, + "learning_rate": 0.0001, + "loss": 148.0044, + "step": 700 + }, + { + "epoch": 0.11465251004218728, + "grad_norm": 2.106865167617798, + "learning_rate": 0.0001, + "loss": 147.8404, + "step": 710 + }, + { + "epoch": 0.11626733412728851, + "grad_norm": 1.8729510307312012, + "learning_rate": 0.0001, + "loss": 145.1299, + "step": 720 + }, + { + "epoch": 0.11788215821238973, + "grad_norm": 4.14620304107666, + "learning_rate": 0.0001, + "loss": 144.3928, + "step": 730 + }, + { + "epoch": 0.11949698229749096, + "grad_norm": 2.202505350112915, + "learning_rate": 0.0001, + "loss": 144.8357, + "step": 740 + }, + { + "epoch": 0.1211118063825922, + "grad_norm": 2.201470375061035, + "learning_rate": 0.0001, + "loss": 143.2202, + "step": 750 + }, + { + "epoch": 0.12272663046769343, + "grad_norm": 1.5977853536605835, + "learning_rate": 0.0001, + "loss": 143.3537, + "step": 760 + }, + { + "epoch": 0.12434145455279466, + "grad_norm": 2.4403927326202393, + "learning_rate": 0.0001, + "loss": 141.2914, + "step": 770 + }, + { + "epoch": 0.12595627863789588, + "grad_norm": 1.6895846128463745, + "learning_rate": 0.0001, + "loss": 139.9639, + "step": 780 + }, + { + "epoch": 0.12757110272299713, + "grad_norm": 3.4370665550231934, + "learning_rate": 0.0001, + "loss": 140.715, + "step": 790 + }, + { + "epoch": 0.12918592680809834, + "grad_norm": 2.1449077129364014, + "learning_rate": 0.0001, + "loss": 139.2094, + "step": 800 + }, + { + "epoch": 0.13080075089319956, + "grad_norm": 2.4578912258148193, + "learning_rate": 0.0001, + "loss": 137.6724, + "step": 810 + }, + { + "epoch": 0.1324155749783008, + "grad_norm": 1.8994814157485962, + "learning_rate": 0.0001, + "loss": 136.9551, + "step": 820 + }, + { + "epoch": 0.13403039906340203, + "grad_norm": 2.6531031131744385, + "learning_rate": 0.0001, + "loss": 136.5812, + "step": 830 + }, + { + "epoch": 0.13564522314850325, + "grad_norm": 3.8680083751678467, + "learning_rate": 0.0001, + "loss": 134.3655, + "step": 840 + }, + { + "epoch": 0.1372600472336045, + "grad_norm": 1.9594208002090454, + "learning_rate": 0.0001, + "loss": 135.4958, + "step": 850 + }, + { + "epoch": 0.1388748713187057, + "grad_norm": 1.5475000143051147, + "learning_rate": 0.0001, + "loss": 135.2142, + "step": 860 + }, + { + "epoch": 0.14048969540380696, + "grad_norm": 1.6569288969039917, + "learning_rate": 0.0001, + "loss": 133.4162, + "step": 870 + }, + { + "epoch": 0.14210451948890818, + "grad_norm": 1.6724224090576172, + "learning_rate": 0.0001, + "loss": 133.7519, + "step": 880 + }, + { + "epoch": 0.1437193435740094, + "grad_norm": 1.7299500703811646, + "learning_rate": 0.0001, + "loss": 131.471, + "step": 890 + }, + { + "epoch": 0.14533416765911064, + "grad_norm": 1.9444470405578613, + "learning_rate": 0.0001, + "loss": 128.5966, + "step": 900 + }, + { + "epoch": 0.14694899174421186, + "grad_norm": 2.404723644256592, + "learning_rate": 0.0001, + "loss": 129.9824, + "step": 910 + }, + { + "epoch": 0.1485638158293131, + "grad_norm": 2.1493897438049316, + "learning_rate": 0.0001, + "loss": 128.0219, + "step": 920 + }, + { + "epoch": 0.15017863991441432, + "grad_norm": 2.8861117362976074, + "learning_rate": 0.0001, + "loss": 128.5297, + "step": 930 + }, + { + "epoch": 0.15179346399951554, + "grad_norm": 2.546415090560913, + "learning_rate": 0.0001, + "loss": 126.6817, + "step": 940 + }, + { + "epoch": 0.1534082880846168, + "grad_norm": 1.727663278579712, + "learning_rate": 0.0001, + "loss": 125.0413, + "step": 950 + }, + { + "epoch": 0.155023112169718, + "grad_norm": 3.7156920433044434, + "learning_rate": 0.0001, + "loss": 124.8085, + "step": 960 + }, + { + "epoch": 0.15663793625481925, + "grad_norm": 3.0576517581939697, + "learning_rate": 0.0001, + "loss": 125.9107, + "step": 970 + }, + { + "epoch": 0.15825276033992047, + "grad_norm": 1.9951574802398682, + "learning_rate": 0.0001, + "loss": 123.5359, + "step": 980 + }, + { + "epoch": 0.1598675844250217, + "grad_norm": 1.4620885848999023, + "learning_rate": 0.0001, + "loss": 121.9363, + "step": 990 + }, + { + "epoch": 0.16148240851012294, + "grad_norm": 1.7640169858932495, + "learning_rate": 0.0001, + "loss": 121.1764, + "step": 1000 + }, + { + "epoch": 0.16309723259522416, + "grad_norm": 2.921750545501709, + "learning_rate": 0.0001, + "loss": 119.6931, + "step": 1010 + }, + { + "epoch": 0.16471205668032537, + "grad_norm": 1.5623588562011719, + "learning_rate": 0.0001, + "loss": 119.097, + "step": 1020 + }, + { + "epoch": 0.16632688076542662, + "grad_norm": 2.2386627197265625, + "learning_rate": 0.0001, + "loss": 120.9224, + "step": 1030 + }, + { + "epoch": 0.16794170485052784, + "grad_norm": 1.3213791847229004, + "learning_rate": 0.0001, + "loss": 119.5649, + "step": 1040 + }, + { + "epoch": 0.16955652893562909, + "grad_norm": 1.5255440473556519, + "learning_rate": 0.0001, + "loss": 118.1524, + "step": 1050 + }, + { + "epoch": 0.1711713530207303, + "grad_norm": 1.6038514375686646, + "learning_rate": 0.0001, + "loss": 118.6307, + "step": 1060 + }, + { + "epoch": 0.17278617710583152, + "grad_norm": 2.356739044189453, + "learning_rate": 0.0001, + "loss": 116.8472, + "step": 1070 + }, + { + "epoch": 0.17440100119093277, + "grad_norm": 2.5326743125915527, + "learning_rate": 0.0001, + "loss": 115.6625, + "step": 1080 + }, + { + "epoch": 0.176015825276034, + "grad_norm": 1.2823396921157837, + "learning_rate": 0.0001, + "loss": 115.3312, + "step": 1090 + }, + { + "epoch": 0.17763064936113523, + "grad_norm": 1.981788158416748, + "learning_rate": 0.0001, + "loss": 116.2257, + "step": 1100 + }, + { + "epoch": 0.17924547344623645, + "grad_norm": 1.540440559387207, + "learning_rate": 0.0001, + "loss": 114.5031, + "step": 1110 + }, + { + "epoch": 0.18086029753133767, + "grad_norm": 1.7667800188064575, + "learning_rate": 0.0001, + "loss": 113.7663, + "step": 1120 + }, + { + "epoch": 0.18247512161643892, + "grad_norm": 2.2793102264404297, + "learning_rate": 0.0001, + "loss": 115.219, + "step": 1130 + }, + { + "epoch": 0.18408994570154014, + "grad_norm": 2.132046699523926, + "learning_rate": 0.0001, + "loss": 112.8482, + "step": 1140 + }, + { + "epoch": 0.18570476978664135, + "grad_norm": 1.3153988122940063, + "learning_rate": 0.0001, + "loss": 111.4546, + "step": 1150 + }, + { + "epoch": 0.1873195938717426, + "grad_norm": 1.5246292352676392, + "learning_rate": 0.0001, + "loss": 112.4348, + "step": 1160 + }, + { + "epoch": 0.18893441795684382, + "grad_norm": 1.365096092224121, + "learning_rate": 0.0001, + "loss": 109.755, + "step": 1170 + }, + { + "epoch": 0.19054924204194507, + "grad_norm": 1.2777953147888184, + "learning_rate": 0.0001, + "loss": 111.0318, + "step": 1180 + }, + { + "epoch": 0.19216406612704628, + "grad_norm": 1.8672829866409302, + "learning_rate": 0.0001, + "loss": 109.1648, + "step": 1190 + }, + { + "epoch": 0.1937788902121475, + "grad_norm": 1.709987759590149, + "learning_rate": 0.0001, + "loss": 108.1616, + "step": 1200 + }, + { + "epoch": 0.19539371429724875, + "grad_norm": 1.176277995109558, + "learning_rate": 0.0001, + "loss": 109.3591, + "step": 1210 + }, + { + "epoch": 0.19700853838234997, + "grad_norm": 1.4181180000305176, + "learning_rate": 0.0001, + "loss": 110.0216, + "step": 1220 + }, + { + "epoch": 0.19862336246745121, + "grad_norm": 1.3246116638183594, + "learning_rate": 0.0001, + "loss": 107.9769, + "step": 1230 + }, + { + "epoch": 0.20023818655255243, + "grad_norm": 1.2501498460769653, + "learning_rate": 0.0001, + "loss": 108.2601, + "step": 1240 + }, + { + "epoch": 0.20185301063765365, + "grad_norm": 1.2277404069900513, + "learning_rate": 0.0001, + "loss": 107.1258, + "step": 1250 + }, + { + "epoch": 0.2034678347227549, + "grad_norm": 1.8064894676208496, + "learning_rate": 0.0001, + "loss": 105.6278, + "step": 1260 + }, + { + "epoch": 0.20508265880785612, + "grad_norm": 1.708897352218628, + "learning_rate": 0.0001, + "loss": 106.9269, + "step": 1270 + }, + { + "epoch": 0.20669748289295733, + "grad_norm": 1.4507478475570679, + "learning_rate": 0.0001, + "loss": 106.9236, + "step": 1280 + }, + { + "epoch": 0.20831230697805858, + "grad_norm": 1.4904310703277588, + "learning_rate": 0.0001, + "loss": 106.0653, + "step": 1290 + }, + { + "epoch": 0.2099271310631598, + "grad_norm": 1.2637165784835815, + "learning_rate": 0.0001, + "loss": 104.8665, + "step": 1300 + }, + { + "epoch": 0.21154195514826105, + "grad_norm": 1.4085133075714111, + "learning_rate": 0.0001, + "loss": 104.8891, + "step": 1310 + }, + { + "epoch": 0.21315677923336226, + "grad_norm": 1.5710629224777222, + "learning_rate": 0.0001, + "loss": 104.3009, + "step": 1320 + }, + { + "epoch": 0.21477160331846348, + "grad_norm": 1.3634663820266724, + "learning_rate": 0.0001, + "loss": 104.4545, + "step": 1330 + }, + { + "epoch": 0.21638642740356473, + "grad_norm": 1.5565859079360962, + "learning_rate": 0.0001, + "loss": 103.6205, + "step": 1340 + }, + { + "epoch": 0.21800125148866595, + "grad_norm": 1.4491888284683228, + "learning_rate": 0.0001, + "loss": 104.0892, + "step": 1350 + }, + { + "epoch": 0.2196160755737672, + "grad_norm": 1.7609063386917114, + "learning_rate": 0.0001, + "loss": 104.5177, + "step": 1360 + }, + { + "epoch": 0.2212308996588684, + "grad_norm": 1.6723753213882446, + "learning_rate": 0.0001, + "loss": 104.5552, + "step": 1370 + }, + { + "epoch": 0.22284572374396963, + "grad_norm": 1.3781665563583374, + "learning_rate": 0.0001, + "loss": 102.1022, + "step": 1380 + }, + { + "epoch": 0.22446054782907088, + "grad_norm": 1.5397366285324097, + "learning_rate": 0.0001, + "loss": 102.5882, + "step": 1390 + }, + { + "epoch": 0.2260753719141721, + "grad_norm": 1.4567879438400269, + "learning_rate": 0.0001, + "loss": 101.3781, + "step": 1400 + }, + { + "epoch": 0.22769019599927334, + "grad_norm": 1.2882301807403564, + "learning_rate": 0.0001, + "loss": 100.9378, + "step": 1410 + }, + { + "epoch": 0.22930502008437456, + "grad_norm": 1.4510424137115479, + "learning_rate": 0.0001, + "loss": 101.5383, + "step": 1420 + }, + { + "epoch": 0.23091984416947578, + "grad_norm": 2.8331449031829834, + "learning_rate": 0.0001, + "loss": 100.2773, + "step": 1430 + }, + { + "epoch": 0.23253466825457703, + "grad_norm": 1.363616943359375, + "learning_rate": 0.0001, + "loss": 101.646, + "step": 1440 + }, + { + "epoch": 0.23414949233967824, + "grad_norm": 1.2184548377990723, + "learning_rate": 0.0001, + "loss": 101.1133, + "step": 1450 + }, + { + "epoch": 0.23576431642477946, + "grad_norm": 2.235656976699829, + "learning_rate": 0.0001, + "loss": 100.2044, + "step": 1460 + }, + { + "epoch": 0.2373791405098807, + "grad_norm": 1.2334418296813965, + "learning_rate": 0.0001, + "loss": 99.4031, + "step": 1470 + }, + { + "epoch": 0.23899396459498193, + "grad_norm": 1.2394684553146362, + "learning_rate": 0.0001, + "loss": 101.1247, + "step": 1480 + }, + { + "epoch": 0.24060878868008317, + "grad_norm": 1.5218186378479004, + "learning_rate": 0.0001, + "loss": 100.4413, + "step": 1490 + }, + { + "epoch": 0.2422236127651844, + "grad_norm": 1.2112364768981934, + "learning_rate": 0.0001, + "loss": 99.8722, + "step": 1500 + }, + { + "epoch": 0.2438384368502856, + "grad_norm": 1.2277610301971436, + "learning_rate": 0.0001, + "loss": 99.0276, + "step": 1510 + }, + { + "epoch": 0.24545326093538686, + "grad_norm": 1.1005053520202637, + "learning_rate": 0.0001, + "loss": 98.6764, + "step": 1520 + }, + { + "epoch": 0.24706808502048808, + "grad_norm": 1.0754456520080566, + "learning_rate": 0.0001, + "loss": 97.8968, + "step": 1530 + }, + { + "epoch": 0.24868290910558932, + "grad_norm": 1.0695950984954834, + "learning_rate": 0.0001, + "loss": 98.4032, + "step": 1540 + }, + { + "epoch": 0.25029773319069054, + "grad_norm": 1.5098627805709839, + "learning_rate": 0.0001, + "loss": 97.9391, + "step": 1550 + }, + { + "epoch": 0.25191255727579176, + "grad_norm": 1.006681203842163, + "learning_rate": 0.0001, + "loss": 96.5525, + "step": 1560 + }, + { + "epoch": 0.253527381360893, + "grad_norm": 1.3987332582473755, + "learning_rate": 0.0001, + "loss": 98.383, + "step": 1570 + }, + { + "epoch": 0.25514220544599425, + "grad_norm": 1.290887713432312, + "learning_rate": 0.0001, + "loss": 99.0901, + "step": 1580 + }, + { + "epoch": 0.25675702953109547, + "grad_norm": 0.991177499294281, + "learning_rate": 0.0001, + "loss": 97.7093, + "step": 1590 + }, + { + "epoch": 0.2583718536161967, + "grad_norm": 1.147647738456726, + "learning_rate": 0.0001, + "loss": 96.9181, + "step": 1600 + }, + { + "epoch": 0.2599866777012979, + "grad_norm": 1.1756514310836792, + "learning_rate": 0.0001, + "loss": 98.3935, + "step": 1610 + }, + { + "epoch": 0.2616015017863991, + "grad_norm": 1.1708602905273438, + "learning_rate": 0.0001, + "loss": 96.715, + "step": 1620 + }, + { + "epoch": 0.2632163258715004, + "grad_norm": 1.4577715396881104, + "learning_rate": 0.0001, + "loss": 97.2971, + "step": 1630 + }, + { + "epoch": 0.2648311499566016, + "grad_norm": 1.1172800064086914, + "learning_rate": 0.0001, + "loss": 97.6494, + "step": 1640 + }, + { + "epoch": 0.26644597404170284, + "grad_norm": 1.1491258144378662, + "learning_rate": 0.0001, + "loss": 97.5503, + "step": 1650 + }, + { + "epoch": 0.26806079812680406, + "grad_norm": 1.2188555002212524, + "learning_rate": 0.0001, + "loss": 95.3423, + "step": 1660 + }, + { + "epoch": 0.2696756222119053, + "grad_norm": 1.8595614433288574, + "learning_rate": 0.0001, + "loss": 96.0112, + "step": 1670 + }, + { + "epoch": 0.2712904462970065, + "grad_norm": 1.1887950897216797, + "learning_rate": 0.0001, + "loss": 95.4939, + "step": 1680 + }, + { + "epoch": 0.27290527038210777, + "grad_norm": 1.1777839660644531, + "learning_rate": 0.0001, + "loss": 95.9619, + "step": 1690 + }, + { + "epoch": 0.274520094467209, + "grad_norm": 1.1241233348846436, + "learning_rate": 0.0001, + "loss": 96.2834, + "step": 1700 + }, + { + "epoch": 0.2761349185523102, + "grad_norm": 1.0900121927261353, + "learning_rate": 0.0001, + "loss": 94.7518, + "step": 1710 + }, + { + "epoch": 0.2777497426374114, + "grad_norm": 1.0615205764770508, + "learning_rate": 0.0001, + "loss": 95.8032, + "step": 1720 + }, + { + "epoch": 0.27936456672251264, + "grad_norm": 1.2626168727874756, + "learning_rate": 0.0001, + "loss": 93.9454, + "step": 1730 + }, + { + "epoch": 0.2809793908076139, + "grad_norm": 0.8869941830635071, + "learning_rate": 0.0001, + "loss": 95.4085, + "step": 1740 + }, + { + "epoch": 0.28259421489271513, + "grad_norm": 1.0350773334503174, + "learning_rate": 0.0001, + "loss": 95.7954, + "step": 1750 + }, + { + "epoch": 0.28420903897781635, + "grad_norm": 1.364768147468567, + "learning_rate": 0.0001, + "loss": 93.6447, + "step": 1760 + }, + { + "epoch": 0.28582386306291757, + "grad_norm": 1.0115883350372314, + "learning_rate": 0.0001, + "loss": 93.7905, + "step": 1770 + }, + { + "epoch": 0.2874386871480188, + "grad_norm": 0.9393435120582581, + "learning_rate": 0.0001, + "loss": 94.3108, + "step": 1780 + }, + { + "epoch": 0.28905351123312006, + "grad_norm": 1.0769627094268799, + "learning_rate": 0.0001, + "loss": 92.566, + "step": 1790 + }, + { + "epoch": 0.2906683353182213, + "grad_norm": 2.526201009750366, + "learning_rate": 0.0001, + "loss": 92.8161, + "step": 1800 + }, + { + "epoch": 0.2922831594033225, + "grad_norm": 1.3066468238830566, + "learning_rate": 0.0001, + "loss": 92.0459, + "step": 1810 + }, + { + "epoch": 0.2938979834884237, + "grad_norm": 1.121713638305664, + "learning_rate": 0.0001, + "loss": 93.5032, + "step": 1820 + }, + { + "epoch": 0.29551280757352494, + "grad_norm": 1.3744316101074219, + "learning_rate": 0.0001, + "loss": 94.2623, + "step": 1830 + }, + { + "epoch": 0.2971276316586262, + "grad_norm": 1.1561540365219116, + "learning_rate": 0.0001, + "loss": 94.1611, + "step": 1840 + }, + { + "epoch": 0.29874245574372743, + "grad_norm": 1.3947464227676392, + "learning_rate": 0.0001, + "loss": 93.5962, + "step": 1850 + }, + { + "epoch": 0.30035727982882865, + "grad_norm": 1.0011566877365112, + "learning_rate": 0.0001, + "loss": 92.0199, + "step": 1860 + }, + { + "epoch": 0.30197210391392987, + "grad_norm": 0.9978747963905334, + "learning_rate": 0.0001, + "loss": 91.0945, + "step": 1870 + }, + { + "epoch": 0.3035869279990311, + "grad_norm": 0.92522132396698, + "learning_rate": 0.0001, + "loss": 93.274, + "step": 1880 + }, + { + "epoch": 0.30520175208413236, + "grad_norm": 1.0089120864868164, + "learning_rate": 0.0001, + "loss": 91.9308, + "step": 1890 + }, + { + "epoch": 0.3068165761692336, + "grad_norm": 1.0535134077072144, + "learning_rate": 0.0001, + "loss": 92.563, + "step": 1900 + }, + { + "epoch": 0.3084314002543348, + "grad_norm": 0.9782275557518005, + "learning_rate": 0.0001, + "loss": 90.3486, + "step": 1910 + }, + { + "epoch": 0.310046224339436, + "grad_norm": 1.1433162689208984, + "learning_rate": 0.0001, + "loss": 90.4977, + "step": 1920 + }, + { + "epoch": 0.31166104842453723, + "grad_norm": 0.98392653465271, + "learning_rate": 0.0001, + "loss": 92.6529, + "step": 1930 + }, + { + "epoch": 0.3132758725096385, + "grad_norm": 0.8538148403167725, + "learning_rate": 0.0001, + "loss": 90.6859, + "step": 1940 + }, + { + "epoch": 0.3148906965947397, + "grad_norm": 1.019405484199524, + "learning_rate": 0.0001, + "loss": 91.0028, + "step": 1950 + }, + { + "epoch": 0.31650552067984095, + "grad_norm": 0.9828067421913147, + "learning_rate": 0.0001, + "loss": 89.9826, + "step": 1960 + }, + { + "epoch": 0.31812034476494216, + "grad_norm": 0.9725450277328491, + "learning_rate": 0.0001, + "loss": 91.7342, + "step": 1970 + }, + { + "epoch": 0.3197351688500434, + "grad_norm": 1.0139408111572266, + "learning_rate": 0.0001, + "loss": 90.5362, + "step": 1980 + }, + { + "epoch": 0.3213499929351446, + "grad_norm": 1.2493853569030762, + "learning_rate": 0.0001, + "loss": 92.7222, + "step": 1990 + }, + { + "epoch": 0.3229648170202459, + "grad_norm": 1.0152915716171265, + "learning_rate": 0.0001, + "loss": 92.2271, + "step": 2000 + }, + { + "epoch": 0.3245796411053471, + "grad_norm": 0.9745819568634033, + "learning_rate": 0.0001, + "loss": 88.7034, + "step": 2010 + }, + { + "epoch": 0.3261944651904483, + "grad_norm": 1.009624719619751, + "learning_rate": 0.0001, + "loss": 90.7428, + "step": 2020 + }, + { + "epoch": 0.32780928927554953, + "grad_norm": 0.910858690738678, + "learning_rate": 0.0001, + "loss": 89.8199, + "step": 2030 + }, + { + "epoch": 0.32942411336065075, + "grad_norm": 1.1745878458023071, + "learning_rate": 0.0001, + "loss": 90.8267, + "step": 2040 + }, + { + "epoch": 0.331038937445752, + "grad_norm": 0.9409273266792297, + "learning_rate": 0.0001, + "loss": 90.2937, + "step": 2050 + }, + { + "epoch": 0.33265376153085324, + "grad_norm": 1.4152942895889282, + "learning_rate": 0.0001, + "loss": 90.3965, + "step": 2060 + }, + { + "epoch": 0.33426858561595446, + "grad_norm": 0.9529693722724915, + "learning_rate": 0.0001, + "loss": 90.4739, + "step": 2070 + }, + { + "epoch": 0.3358834097010557, + "grad_norm": 0.8531096577644348, + "learning_rate": 0.0001, + "loss": 90.9301, + "step": 2080 + }, + { + "epoch": 0.3374982337861569, + "grad_norm": 0.9314167499542236, + "learning_rate": 0.0001, + "loss": 90.4235, + "step": 2090 + }, + { + "epoch": 0.33911305787125817, + "grad_norm": 0.9253250360488892, + "learning_rate": 0.0001, + "loss": 88.1893, + "step": 2100 + }, + { + "epoch": 0.3407278819563594, + "grad_norm": 1.773053526878357, + "learning_rate": 0.0001, + "loss": 88.3417, + "step": 2110 + }, + { + "epoch": 0.3423427060414606, + "grad_norm": 0.9432951211929321, + "learning_rate": 0.0001, + "loss": 89.7141, + "step": 2120 + }, + { + "epoch": 0.34395753012656183, + "grad_norm": 3.8847527503967285, + "learning_rate": 0.0001, + "loss": 89.1009, + "step": 2130 + }, + { + "epoch": 0.34557235421166305, + "grad_norm": 1.0309821367263794, + "learning_rate": 0.0001, + "loss": 89.2965, + "step": 2140 + }, + { + "epoch": 0.3471871782967643, + "grad_norm": 0.9590495824813843, + "learning_rate": 0.0001, + "loss": 89.1734, + "step": 2150 + }, + { + "epoch": 0.34880200238186554, + "grad_norm": 0.9943560361862183, + "learning_rate": 0.0001, + "loss": 85.425, + "step": 2160 + }, + { + "epoch": 0.35041682646696676, + "grad_norm": 1.3322339057922363, + "learning_rate": 0.0001, + "loss": 89.7791, + "step": 2170 + }, + { + "epoch": 0.352031650552068, + "grad_norm": 1.0212253332138062, + "learning_rate": 0.0001, + "loss": 89.3771, + "step": 2180 + }, + { + "epoch": 0.3536464746371692, + "grad_norm": 1.0446089506149292, + "learning_rate": 0.0001, + "loss": 88.6894, + "step": 2190 + }, + { + "epoch": 0.35526129872227047, + "grad_norm": 1.05767822265625, + "learning_rate": 0.0001, + "loss": 89.5341, + "step": 2200 + }, + { + "epoch": 0.3568761228073717, + "grad_norm": 1.0373681783676147, + "learning_rate": 0.0001, + "loss": 89.7173, + "step": 2210 + }, + { + "epoch": 0.3584909468924729, + "grad_norm": 0.9098039269447327, + "learning_rate": 0.0001, + "loss": 89.2116, + "step": 2220 + }, + { + "epoch": 0.3601057709775741, + "grad_norm": 1.028130292892456, + "learning_rate": 0.0001, + "loss": 88.6627, + "step": 2230 + }, + { + "epoch": 0.36172059506267534, + "grad_norm": 1.0295171737670898, + "learning_rate": 0.0001, + "loss": 89.608, + "step": 2240 + }, + { + "epoch": 0.36333541914777656, + "grad_norm": 0.9720722436904907, + "learning_rate": 0.0001, + "loss": 88.3624, + "step": 2250 + }, + { + "epoch": 0.36495024323287784, + "grad_norm": 0.839654803276062, + "learning_rate": 0.0001, + "loss": 87.2983, + "step": 2260 + }, + { + "epoch": 0.36656506731797905, + "grad_norm": 1.1600219011306763, + "learning_rate": 0.0001, + "loss": 86.6004, + "step": 2270 + }, + { + "epoch": 0.3681798914030803, + "grad_norm": 1.0513924360275269, + "learning_rate": 0.0001, + "loss": 86.6062, + "step": 2280 + }, + { + "epoch": 0.3697947154881815, + "grad_norm": 0.9728474617004395, + "learning_rate": 0.0001, + "loss": 86.1856, + "step": 2290 + }, + { + "epoch": 0.3714095395732827, + "grad_norm": 1.0196386575698853, + "learning_rate": 0.0001, + "loss": 87.6088, + "step": 2300 + }, + { + "epoch": 0.373024363658384, + "grad_norm": 1.0439444780349731, + "learning_rate": 0.0001, + "loss": 85.2374, + "step": 2310 + }, + { + "epoch": 0.3746391877434852, + "grad_norm": 1.0020275115966797, + "learning_rate": 0.0001, + "loss": 86.0596, + "step": 2320 + }, + { + "epoch": 0.3762540118285864, + "grad_norm": 1.0107691287994385, + "learning_rate": 0.0001, + "loss": 86.9513, + "step": 2330 + }, + { + "epoch": 0.37786883591368764, + "grad_norm": 1.680018663406372, + "learning_rate": 0.0001, + "loss": 87.8885, + "step": 2340 + }, + { + "epoch": 0.37948365999878886, + "grad_norm": 0.9181709885597229, + "learning_rate": 0.0001, + "loss": 86.7662, + "step": 2350 + }, + { + "epoch": 0.38109848408389013, + "grad_norm": 0.9598278403282166, + "learning_rate": 0.0001, + "loss": 88.3775, + "step": 2360 + }, + { + "epoch": 0.38271330816899135, + "grad_norm": 1.0707905292510986, + "learning_rate": 0.0001, + "loss": 86.0997, + "step": 2370 + }, + { + "epoch": 0.38432813225409257, + "grad_norm": 1.125797152519226, + "learning_rate": 0.0001, + "loss": 85.8117, + "step": 2380 + }, + { + "epoch": 0.3859429563391938, + "grad_norm": 1.0251439809799194, + "learning_rate": 0.0001, + "loss": 86.0806, + "step": 2390 + }, + { + "epoch": 0.387557780424295, + "grad_norm": 1.0193125009536743, + "learning_rate": 0.0001, + "loss": 87.264, + "step": 2400 + }, + { + "epoch": 0.3891726045093963, + "grad_norm": 0.9309751391410828, + "learning_rate": 0.0001, + "loss": 85.9187, + "step": 2410 + }, + { + "epoch": 0.3907874285944975, + "grad_norm": 1.1025906801223755, + "learning_rate": 0.0001, + "loss": 86.4365, + "step": 2420 + }, + { + "epoch": 0.3924022526795987, + "grad_norm": 0.9880627393722534, + "learning_rate": 0.0001, + "loss": 86.2303, + "step": 2430 + }, + { + "epoch": 0.39401707676469994, + "grad_norm": 0.9520611763000488, + "learning_rate": 0.0001, + "loss": 84.9035, + "step": 2440 + }, + { + "epoch": 0.39563190084980115, + "grad_norm": 0.8641896843910217, + "learning_rate": 0.0001, + "loss": 85.4754, + "step": 2450 + }, + { + "epoch": 0.39724672493490243, + "grad_norm": 1.1162904500961304, + "learning_rate": 0.0001, + "loss": 85.6767, + "step": 2460 + }, + { + "epoch": 0.39886154902000365, + "grad_norm": 0.9073636531829834, + "learning_rate": 0.0001, + "loss": 85.4198, + "step": 2470 + }, + { + "epoch": 0.40047637310510487, + "grad_norm": 1.1135166883468628, + "learning_rate": 0.0001, + "loss": 86.0884, + "step": 2480 + }, + { + "epoch": 0.4020911971902061, + "grad_norm": 0.9273980259895325, + "learning_rate": 0.0001, + "loss": 85.5473, + "step": 2490 + }, + { + "epoch": 0.4037060212753073, + "grad_norm": 0.8653831481933594, + "learning_rate": 0.0001, + "loss": 87.279, + "step": 2500 + }, + { + "epoch": 0.4053208453604086, + "grad_norm": 0.8535159826278687, + "learning_rate": 0.0001, + "loss": 86.7072, + "step": 2510 + }, + { + "epoch": 0.4069356694455098, + "grad_norm": 0.8864684700965881, + "learning_rate": 0.0001, + "loss": 86.441, + "step": 2520 + }, + { + "epoch": 0.408550493530611, + "grad_norm": 0.8042080998420715, + "learning_rate": 0.0001, + "loss": 83.1706, + "step": 2530 + }, + { + "epoch": 0.41016531761571223, + "grad_norm": 1.0486373901367188, + "learning_rate": 0.0001, + "loss": 83.54, + "step": 2540 + }, + { + "epoch": 0.41178014170081345, + "grad_norm": 0.8087959289550781, + "learning_rate": 0.0001, + "loss": 85.7615, + "step": 2550 + }, + { + "epoch": 0.41339496578591467, + "grad_norm": 1.5032018423080444, + "learning_rate": 0.0001, + "loss": 85.7368, + "step": 2560 + }, + { + "epoch": 0.41500978987101594, + "grad_norm": 1.0786182880401611, + "learning_rate": 0.0001, + "loss": 86.3484, + "step": 2570 + }, + { + "epoch": 0.41662461395611716, + "grad_norm": 0.8744294047355652, + "learning_rate": 0.0001, + "loss": 83.5945, + "step": 2580 + }, + { + "epoch": 0.4182394380412184, + "grad_norm": 0.7840721607208252, + "learning_rate": 0.0001, + "loss": 86.3831, + "step": 2590 + }, + { + "epoch": 0.4198542621263196, + "grad_norm": 0.7977836728096008, + "learning_rate": 0.0001, + "loss": 83.6314, + "step": 2600 + }, + { + "epoch": 0.4214690862114208, + "grad_norm": 0.892484188079834, + "learning_rate": 0.0001, + "loss": 86.4223, + "step": 2610 + }, + { + "epoch": 0.4230839102965221, + "grad_norm": 0.912995457649231, + "learning_rate": 0.0001, + "loss": 83.8971, + "step": 2620 + }, + { + "epoch": 0.4246987343816233, + "grad_norm": 0.9566861987113953, + "learning_rate": 0.0001, + "loss": 85.291, + "step": 2630 + }, + { + "epoch": 0.42631355846672453, + "grad_norm": 0.8324154615402222, + "learning_rate": 0.0001, + "loss": 84.6233, + "step": 2640 + }, + { + "epoch": 0.42792838255182575, + "grad_norm": 0.9490360617637634, + "learning_rate": 0.0001, + "loss": 81.799, + "step": 2650 + }, + { + "epoch": 0.42954320663692697, + "grad_norm": 0.9744566679000854, + "learning_rate": 0.0001, + "loss": 85.4837, + "step": 2660 + }, + { + "epoch": 0.43115803072202824, + "grad_norm": 0.941614031791687, + "learning_rate": 0.0001, + "loss": 82.838, + "step": 2670 + }, + { + "epoch": 0.43277285480712946, + "grad_norm": 1.338364601135254, + "learning_rate": 0.0001, + "loss": 84.4224, + "step": 2680 + }, + { + "epoch": 0.4343876788922307, + "grad_norm": 0.8643816113471985, + "learning_rate": 0.0001, + "loss": 84.3959, + "step": 2690 + }, + { + "epoch": 0.4360025029773319, + "grad_norm": 1.0231473445892334, + "learning_rate": 0.0001, + "loss": 82.3144, + "step": 2700 + }, + { + "epoch": 0.4376173270624331, + "grad_norm": 0.9089650511741638, + "learning_rate": 0.0001, + "loss": 83.0326, + "step": 2710 + }, + { + "epoch": 0.4392321511475344, + "grad_norm": 0.893628716468811, + "learning_rate": 0.0001, + "loss": 83.0981, + "step": 2720 + }, + { + "epoch": 0.4408469752326356, + "grad_norm": 0.869076132774353, + "learning_rate": 0.0001, + "loss": 84.2571, + "step": 2730 + }, + { + "epoch": 0.4424617993177368, + "grad_norm": 0.7964235544204712, + "learning_rate": 0.0001, + "loss": 83.3329, + "step": 2740 + }, + { + "epoch": 0.44407662340283804, + "grad_norm": 3.1645138263702393, + "learning_rate": 0.0001, + "loss": 83.7232, + "step": 2750 + }, + { + "epoch": 0.44569144748793926, + "grad_norm": 0.934644877910614, + "learning_rate": 0.0001, + "loss": 85.0688, + "step": 2760 + }, + { + "epoch": 0.44730627157304054, + "grad_norm": 0.8120711445808411, + "learning_rate": 0.0001, + "loss": 82.7706, + "step": 2770 + }, + { + "epoch": 0.44892109565814176, + "grad_norm": 0.8161435127258301, + "learning_rate": 0.0001, + "loss": 84.4826, + "step": 2780 + }, + { + "epoch": 0.450535919743243, + "grad_norm": 0.8659493327140808, + "learning_rate": 0.0001, + "loss": 82.6222, + "step": 2790 + }, + { + "epoch": 0.4521507438283442, + "grad_norm": 0.9306702613830566, + "learning_rate": 0.0001, + "loss": 83.5897, + "step": 2800 + }, + { + "epoch": 0.4537655679134454, + "grad_norm": 2.2544431686401367, + "learning_rate": 0.0001, + "loss": 82.9603, + "step": 2810 + }, + { + "epoch": 0.4553803919985467, + "grad_norm": 0.8868395686149597, + "learning_rate": 0.0001, + "loss": 82.6674, + "step": 2820 + }, + { + "epoch": 0.4569952160836479, + "grad_norm": 0.9336455464363098, + "learning_rate": 0.0001, + "loss": 84.6974, + "step": 2830 + }, + { + "epoch": 0.4586100401687491, + "grad_norm": 0.8814165592193604, + "learning_rate": 0.0001, + "loss": 83.9329, + "step": 2840 + }, + { + "epoch": 0.46022486425385034, + "grad_norm": 0.8125660419464111, + "learning_rate": 0.0001, + "loss": 84.6168, + "step": 2850 + }, + { + "epoch": 0.46183968833895156, + "grad_norm": 0.9447190165519714, + "learning_rate": 0.0001, + "loss": 82.4098, + "step": 2860 + }, + { + "epoch": 0.4634545124240528, + "grad_norm": 0.8080969452857971, + "learning_rate": 0.0001, + "loss": 83.4076, + "step": 2870 + }, + { + "epoch": 0.46506933650915405, + "grad_norm": 1.0586694478988647, + "learning_rate": 0.0001, + "loss": 83.3295, + "step": 2880 + }, + { + "epoch": 0.46668416059425527, + "grad_norm": 0.8207664489746094, + "learning_rate": 0.0001, + "loss": 84.5056, + "step": 2890 + }, + { + "epoch": 0.4682989846793565, + "grad_norm": 1.0647571086883545, + "learning_rate": 0.0001, + "loss": 83.0766, + "step": 2900 + }, + { + "epoch": 0.4699138087644577, + "grad_norm": 0.9416915774345398, + "learning_rate": 0.0001, + "loss": 83.8166, + "step": 2910 + }, + { + "epoch": 0.4715286328495589, + "grad_norm": 0.7479362487792969, + "learning_rate": 0.0001, + "loss": 82.7507, + "step": 2920 + }, + { + "epoch": 0.4731434569346602, + "grad_norm": 1.0775514841079712, + "learning_rate": 0.0001, + "loss": 84.4641, + "step": 2930 + }, + { + "epoch": 0.4747582810197614, + "grad_norm": 0.8736841082572937, + "learning_rate": 0.0001, + "loss": 82.0451, + "step": 2940 + }, + { + "epoch": 0.47637310510486264, + "grad_norm": 0.9500495791435242, + "learning_rate": 0.0001, + "loss": 82.6425, + "step": 2950 + }, + { + "epoch": 0.47798792918996386, + "grad_norm": 0.8551210165023804, + "learning_rate": 0.0001, + "loss": 81.1971, + "step": 2960 + }, + { + "epoch": 0.4796027532750651, + "grad_norm": 0.8149814605712891, + "learning_rate": 0.0001, + "loss": 82.7919, + "step": 2970 + }, + { + "epoch": 0.48121757736016635, + "grad_norm": 0.9012249112129211, + "learning_rate": 0.0001, + "loss": 81.975, + "step": 2980 + }, + { + "epoch": 0.48283240144526757, + "grad_norm": 1.3784620761871338, + "learning_rate": 0.0001, + "loss": 81.541, + "step": 2990 + }, + { + "epoch": 0.4844472255303688, + "grad_norm": 0.7921801209449768, + "learning_rate": 0.0001, + "loss": 83.6215, + "step": 3000 + }, + { + "epoch": 0.48606204961547, + "grad_norm": 0.9051580429077148, + "learning_rate": 0.0001, + "loss": 82.4048, + "step": 3010 + }, + { + "epoch": 0.4876768737005712, + "grad_norm": 1.0717192888259888, + "learning_rate": 0.0001, + "loss": 82.4079, + "step": 3020 + }, + { + "epoch": 0.4892916977856725, + "grad_norm": 0.8971781730651855, + "learning_rate": 0.0001, + "loss": 83.3668, + "step": 3030 + }, + { + "epoch": 0.4909065218707737, + "grad_norm": 0.9635320901870728, + "learning_rate": 0.0001, + "loss": 82.8244, + "step": 3040 + }, + { + "epoch": 0.49252134595587493, + "grad_norm": 1.1009949445724487, + "learning_rate": 0.0001, + "loss": 82.2174, + "step": 3050 + }, + { + "epoch": 0.49413617004097615, + "grad_norm": 0.9805473685264587, + "learning_rate": 0.0001, + "loss": 83.7897, + "step": 3060 + }, + { + "epoch": 0.49575099412607737, + "grad_norm": 1.2107361555099487, + "learning_rate": 0.0001, + "loss": 82.5108, + "step": 3070 + }, + { + "epoch": 0.49736581821117865, + "grad_norm": 0.9171829223632812, + "learning_rate": 0.0001, + "loss": 80.9126, + "step": 3080 + }, + { + "epoch": 0.49898064229627986, + "grad_norm": 0.8093295097351074, + "learning_rate": 0.0001, + "loss": 83.8147, + "step": 3090 + }, + { + "epoch": 0.5005954663813811, + "grad_norm": 1.2166653871536255, + "learning_rate": 0.0001, + "loss": 82.0288, + "step": 3100 + }, + { + "epoch": 0.5022102904664824, + "grad_norm": 1.0471502542495728, + "learning_rate": 0.0001, + "loss": 82.0706, + "step": 3110 + }, + { + "epoch": 0.5038251145515835, + "grad_norm": 1.007645845413208, + "learning_rate": 0.0001, + "loss": 81.7387, + "step": 3120 + }, + { + "epoch": 0.5054399386366848, + "grad_norm": 0.7252638339996338, + "learning_rate": 0.0001, + "loss": 82.9107, + "step": 3130 + }, + { + "epoch": 0.507054762721786, + "grad_norm": 0.8565683364868164, + "learning_rate": 0.0001, + "loss": 80.2935, + "step": 3140 + }, + { + "epoch": 0.5086695868068872, + "grad_norm": 0.9905404448509216, + "learning_rate": 0.0001, + "loss": 82.5201, + "step": 3150 + }, + { + "epoch": 0.5102844108919885, + "grad_norm": 0.887428343296051, + "learning_rate": 0.0001, + "loss": 80.8528, + "step": 3160 + }, + { + "epoch": 0.5118992349770897, + "grad_norm": 1.062942624092102, + "learning_rate": 0.0001, + "loss": 80.7035, + "step": 3170 + }, + { + "epoch": 0.5135140590621909, + "grad_norm": 1.203599214553833, + "learning_rate": 0.0001, + "loss": 81.2521, + "step": 3180 + }, + { + "epoch": 0.5151288831472921, + "grad_norm": 0.8452383875846863, + "learning_rate": 0.0001, + "loss": 81.6079, + "step": 3190 + }, + { + "epoch": 0.5167437072323934, + "grad_norm": 1.2530587911605835, + "learning_rate": 0.0001, + "loss": 82.2148, + "step": 3200 + }, + { + "epoch": 0.5183585313174947, + "grad_norm": 0.9351627826690674, + "learning_rate": 0.0001, + "loss": 82.7857, + "step": 3210 + }, + { + "epoch": 0.5199733554025958, + "grad_norm": 1.0128077268600464, + "learning_rate": 0.0001, + "loss": 81.0532, + "step": 3220 + }, + { + "epoch": 0.5215881794876971, + "grad_norm": 0.9316917061805725, + "learning_rate": 0.0001, + "loss": 80.7443, + "step": 3230 + }, + { + "epoch": 0.5232030035727983, + "grad_norm": 1.502705693244934, + "learning_rate": 0.0001, + "loss": 80.0608, + "step": 3240 + }, + { + "epoch": 0.5248178276578995, + "grad_norm": 0.8166837692260742, + "learning_rate": 0.0001, + "loss": 81.2013, + "step": 3250 + }, + { + "epoch": 0.5264326517430008, + "grad_norm": 0.8210042119026184, + "learning_rate": 0.0001, + "loss": 80.2491, + "step": 3260 + }, + { + "epoch": 0.528047475828102, + "grad_norm": 0.7854771018028259, + "learning_rate": 0.0001, + "loss": 81.1017, + "step": 3270 + }, + { + "epoch": 0.5296622999132032, + "grad_norm": 0.8530565500259399, + "learning_rate": 0.0001, + "loss": 80.8123, + "step": 3280 + }, + { + "epoch": 0.5312771239983044, + "grad_norm": 0.9446498155593872, + "learning_rate": 0.0001, + "loss": 80.0029, + "step": 3290 + }, + { + "epoch": 0.5328919480834057, + "grad_norm": 0.9646537899971008, + "learning_rate": 0.0001, + "loss": 81.5413, + "step": 3300 + }, + { + "epoch": 0.5345067721685068, + "grad_norm": 0.9262019395828247, + "learning_rate": 0.0001, + "loss": 80.9399, + "step": 3310 + }, + { + "epoch": 0.5361215962536081, + "grad_norm": 0.8515486121177673, + "learning_rate": 0.0001, + "loss": 80.9421, + "step": 3320 + }, + { + "epoch": 0.5377364203387094, + "grad_norm": 1.679321050643921, + "learning_rate": 0.0001, + "loss": 82.4851, + "step": 3330 + }, + { + "epoch": 0.5393512444238105, + "grad_norm": 0.8289812207221985, + "learning_rate": 0.0001, + "loss": 80.5664, + "step": 3340 + }, + { + "epoch": 0.5409660685089118, + "grad_norm": 0.7463789582252502, + "learning_rate": 0.0001, + "loss": 81.5193, + "step": 3350 + }, + { + "epoch": 0.542580892594013, + "grad_norm": 0.8332446217536926, + "learning_rate": 0.0001, + "loss": 80.8091, + "step": 3360 + }, + { + "epoch": 0.5441957166791143, + "grad_norm": 0.7612847089767456, + "learning_rate": 0.0001, + "loss": 79.8893, + "step": 3370 + }, + { + "epoch": 0.5458105407642155, + "grad_norm": 1.0557948350906372, + "learning_rate": 0.0001, + "loss": 81.4305, + "step": 3380 + }, + { + "epoch": 0.5474253648493167, + "grad_norm": 0.7975788116455078, + "learning_rate": 0.0001, + "loss": 80.0105, + "step": 3390 + }, + { + "epoch": 0.549040188934418, + "grad_norm": 0.9049227237701416, + "learning_rate": 0.0001, + "loss": 81.2689, + "step": 3400 + }, + { + "epoch": 0.5506550130195191, + "grad_norm": 1.1195895671844482, + "learning_rate": 0.0001, + "loss": 80.7163, + "step": 3410 + }, + { + "epoch": 0.5522698371046204, + "grad_norm": 0.7506540417671204, + "learning_rate": 0.0001, + "loss": 79.0855, + "step": 3420 + }, + { + "epoch": 0.5538846611897217, + "grad_norm": 0.73989337682724, + "learning_rate": 0.0001, + "loss": 81.1865, + "step": 3430 + }, + { + "epoch": 0.5554994852748228, + "grad_norm": 0.8418401479721069, + "learning_rate": 0.0001, + "loss": 79.5714, + "step": 3440 + }, + { + "epoch": 0.5571143093599241, + "grad_norm": 0.9807226061820984, + "learning_rate": 0.0001, + "loss": 78.5699, + "step": 3450 + }, + { + "epoch": 0.5587291334450253, + "grad_norm": 1.2890545129776, + "learning_rate": 0.0001, + "loss": 81.0128, + "step": 3460 + }, + { + "epoch": 0.5603439575301266, + "grad_norm": 0.8593361973762512, + "learning_rate": 0.0001, + "loss": 80.1638, + "step": 3470 + }, + { + "epoch": 0.5619587816152278, + "grad_norm": 0.7766354084014893, + "learning_rate": 0.0001, + "loss": 81.5597, + "step": 3480 + }, + { + "epoch": 0.563573605700329, + "grad_norm": 0.7785626649856567, + "learning_rate": 0.0001, + "loss": 79.3862, + "step": 3490 + }, + { + "epoch": 0.5651884297854303, + "grad_norm": 0.8429279327392578, + "learning_rate": 0.0001, + "loss": 76.6294, + "step": 3500 + }, + { + "epoch": 0.5668032538705314, + "grad_norm": 0.8141765594482422, + "learning_rate": 0.0001, + "loss": 80.2584, + "step": 3510 + }, + { + "epoch": 0.5684180779556327, + "grad_norm": 0.7065490484237671, + "learning_rate": 0.0001, + "loss": 79.3626, + "step": 3520 + }, + { + "epoch": 0.570032902040734, + "grad_norm": 0.7002031803131104, + "learning_rate": 0.0001, + "loss": 79.283, + "step": 3530 + }, + { + "epoch": 0.5716477261258351, + "grad_norm": 0.8715317845344543, + "learning_rate": 0.0001, + "loss": 80.3145, + "step": 3540 + }, + { + "epoch": 0.5732625502109364, + "grad_norm": 0.9584882855415344, + "learning_rate": 0.0001, + "loss": 78.7443, + "step": 3550 + }, + { + "epoch": 0.5748773742960376, + "grad_norm": 0.7908946871757507, + "learning_rate": 0.0001, + "loss": 79.1906, + "step": 3560 + }, + { + "epoch": 0.5764921983811389, + "grad_norm": 0.8416746258735657, + "learning_rate": 0.0001, + "loss": 77.8172, + "step": 3570 + }, + { + "epoch": 0.5781070224662401, + "grad_norm": 0.823667049407959, + "learning_rate": 0.0001, + "loss": 81.864, + "step": 3580 + }, + { + "epoch": 0.5797218465513413, + "grad_norm": 1.4282617568969727, + "learning_rate": 0.0001, + "loss": 79.9052, + "step": 3590 + }, + { + "epoch": 0.5813366706364426, + "grad_norm": 0.8747813701629639, + "learning_rate": 0.0001, + "loss": 78.8234, + "step": 3600 + }, + { + "epoch": 0.5829514947215437, + "grad_norm": 0.8415511846542358, + "learning_rate": 0.0001, + "loss": 78.8171, + "step": 3610 + }, + { + "epoch": 0.584566318806645, + "grad_norm": 0.9014492034912109, + "learning_rate": 0.0001, + "loss": 80.5011, + "step": 3620 + }, + { + "epoch": 0.5861811428917463, + "grad_norm": 0.7535204887390137, + "learning_rate": 0.0001, + "loss": 79.4895, + "step": 3630 + }, + { + "epoch": 0.5877959669768474, + "grad_norm": 0.8196508884429932, + "learning_rate": 0.0001, + "loss": 79.7101, + "step": 3640 + }, + { + "epoch": 0.5894107910619487, + "grad_norm": 1.5975492000579834, + "learning_rate": 0.0001, + "loss": 79.0687, + "step": 3650 + }, + { + "epoch": 0.5910256151470499, + "grad_norm": 0.9251692295074463, + "learning_rate": 0.0001, + "loss": 80.2377, + "step": 3660 + }, + { + "epoch": 0.5926404392321512, + "grad_norm": 0.7143557071685791, + "learning_rate": 0.0001, + "loss": 78.0671, + "step": 3670 + }, + { + "epoch": 0.5942552633172524, + "grad_norm": 0.7876290678977966, + "learning_rate": 0.0001, + "loss": 79.1455, + "step": 3680 + }, + { + "epoch": 0.5958700874023536, + "grad_norm": 1.5361895561218262, + "learning_rate": 0.0001, + "loss": 78.1253, + "step": 3690 + }, + { + "epoch": 0.5974849114874549, + "grad_norm": 0.7637040019035339, + "learning_rate": 0.0001, + "loss": 77.4658, + "step": 3700 + }, + { + "epoch": 0.599099735572556, + "grad_norm": 0.8166250586509705, + "learning_rate": 0.0001, + "loss": 79.5806, + "step": 3710 + }, + { + "epoch": 0.6007145596576573, + "grad_norm": 1.2767380475997925, + "learning_rate": 0.0001, + "loss": 79.1894, + "step": 3720 + }, + { + "epoch": 0.6023293837427586, + "grad_norm": 1.3048640489578247, + "learning_rate": 0.0001, + "loss": 78.0593, + "step": 3730 + }, + { + "epoch": 0.6039442078278597, + "grad_norm": 0.7945445775985718, + "learning_rate": 0.0001, + "loss": 79.7623, + "step": 3740 + }, + { + "epoch": 0.605559031912961, + "grad_norm": 0.7366836667060852, + "learning_rate": 0.0001, + "loss": 77.9571, + "step": 3750 + }, + { + "epoch": 0.6071738559980622, + "grad_norm": 0.9126115441322327, + "learning_rate": 0.0001, + "loss": 79.9502, + "step": 3760 + }, + { + "epoch": 0.6087886800831634, + "grad_norm": 0.800355851650238, + "learning_rate": 0.0001, + "loss": 78.7244, + "step": 3770 + }, + { + "epoch": 0.6104035041682647, + "grad_norm": 0.8290980458259583, + "learning_rate": 0.0001, + "loss": 78.7724, + "step": 3780 + }, + { + "epoch": 0.6120183282533659, + "grad_norm": 0.8512316942214966, + "learning_rate": 0.0001, + "loss": 78.6277, + "step": 3790 + }, + { + "epoch": 0.6136331523384672, + "grad_norm": 0.8909353613853455, + "learning_rate": 0.0001, + "loss": 78.5956, + "step": 3800 + }, + { + "epoch": 0.6152479764235683, + "grad_norm": 0.8328800797462463, + "learning_rate": 0.0001, + "loss": 78.1323, + "step": 3810 + }, + { + "epoch": 0.6168628005086696, + "grad_norm": 0.7950423359870911, + "learning_rate": 0.0001, + "loss": 78.7529, + "step": 3820 + }, + { + "epoch": 0.6184776245937709, + "grad_norm": 0.9988545179367065, + "learning_rate": 0.0001, + "loss": 80.7727, + "step": 3830 + }, + { + "epoch": 0.620092448678872, + "grad_norm": 0.7243085503578186, + "learning_rate": 0.0001, + "loss": 76.7616, + "step": 3840 + }, + { + "epoch": 0.6217072727639733, + "grad_norm": 0.7674855589866638, + "learning_rate": 0.0001, + "loss": 79.1938, + "step": 3850 + }, + { + "epoch": 0.6233220968490745, + "grad_norm": 0.8979483842849731, + "learning_rate": 0.0001, + "loss": 79.1522, + "step": 3860 + }, + { + "epoch": 0.6249369209341757, + "grad_norm": 0.7759745717048645, + "learning_rate": 0.0001, + "loss": 77.7001, + "step": 3870 + }, + { + "epoch": 0.626551745019277, + "grad_norm": 0.9940371513366699, + "learning_rate": 0.0001, + "loss": 79.8712, + "step": 3880 + }, + { + "epoch": 0.6281665691043782, + "grad_norm": 0.8103354573249817, + "learning_rate": 0.0001, + "loss": 77.5229, + "step": 3890 + }, + { + "epoch": 0.6297813931894795, + "grad_norm": 0.9188411831855774, + "learning_rate": 0.0001, + "loss": 78.2933, + "step": 3900 + }, + { + "epoch": 0.6313962172745806, + "grad_norm": 0.823017954826355, + "learning_rate": 0.0001, + "loss": 77.8692, + "step": 3910 + }, + { + "epoch": 0.6330110413596819, + "grad_norm": 0.8833358287811279, + "learning_rate": 0.0001, + "loss": 78.4465, + "step": 3920 + }, + { + "epoch": 0.634625865444783, + "grad_norm": 0.6880685687065125, + "learning_rate": 0.0001, + "loss": 78.8881, + "step": 3930 + }, + { + "epoch": 0.6362406895298843, + "grad_norm": 0.778455913066864, + "learning_rate": 0.0001, + "loss": 76.0277, + "step": 3940 + }, + { + "epoch": 0.6378555136149856, + "grad_norm": 0.8132772445678711, + "learning_rate": 0.0001, + "loss": 79.1284, + "step": 3950 + }, + { + "epoch": 0.6394703377000868, + "grad_norm": 0.7703788876533508, + "learning_rate": 0.0001, + "loss": 76.9013, + "step": 3960 + }, + { + "epoch": 0.641085161785188, + "grad_norm": 0.7876151204109192, + "learning_rate": 0.0001, + "loss": 77.1354, + "step": 3970 + }, + { + "epoch": 0.6426999858702892, + "grad_norm": 0.8027064800262451, + "learning_rate": 0.0001, + "loss": 77.4936, + "step": 3980 + }, + { + "epoch": 0.6443148099553905, + "grad_norm": 0.8261818885803223, + "learning_rate": 0.0001, + "loss": 76.7634, + "step": 3990 + }, + { + "epoch": 0.6459296340404918, + "grad_norm": 0.7024852633476257, + "learning_rate": 0.0001, + "loss": 79.951, + "step": 4000 + }, + { + "epoch": 0.6475444581255929, + "grad_norm": 1.015307068824768, + "learning_rate": 0.0001, + "loss": 76.0354, + "step": 4010 + }, + { + "epoch": 0.6491592822106942, + "grad_norm": 0.715346097946167, + "learning_rate": 0.0001, + "loss": 78.9021, + "step": 4020 + }, + { + "epoch": 0.6507741062957954, + "grad_norm": 0.8994929790496826, + "learning_rate": 0.0001, + "loss": 77.0587, + "step": 4030 + }, + { + "epoch": 0.6523889303808966, + "grad_norm": 0.7683436274528503, + "learning_rate": 0.0001, + "loss": 78.6943, + "step": 4040 + }, + { + "epoch": 0.6540037544659979, + "grad_norm": 0.7273952960968018, + "learning_rate": 0.0001, + "loss": 77.8396, + "step": 4050 + }, + { + "epoch": 0.6556185785510991, + "grad_norm": 1.0676878690719604, + "learning_rate": 0.0001, + "loss": 79.3449, + "step": 4060 + }, + { + "epoch": 0.6572334026362003, + "grad_norm": 0.6916168928146362, + "learning_rate": 0.0001, + "loss": 76.5462, + "step": 4070 + }, + { + "epoch": 0.6588482267213015, + "grad_norm": 0.6466719508171082, + "learning_rate": 0.0001, + "loss": 77.9354, + "step": 4080 + }, + { + "epoch": 0.6604630508064028, + "grad_norm": 0.7406665086746216, + "learning_rate": 0.0001, + "loss": 76.0135, + "step": 4090 + }, + { + "epoch": 0.662077874891504, + "grad_norm": 0.7104353904724121, + "learning_rate": 0.0001, + "loss": 78.3727, + "step": 4100 + }, + { + "epoch": 0.6636926989766052, + "grad_norm": 0.8328452110290527, + "learning_rate": 0.0001, + "loss": 77.5354, + "step": 4110 + }, + { + "epoch": 0.6653075230617065, + "grad_norm": 0.7012260556221008, + "learning_rate": 0.0001, + "loss": 78.7406, + "step": 4120 + }, + { + "epoch": 0.6669223471468076, + "grad_norm": 1.0436632633209229, + "learning_rate": 0.0001, + "loss": 78.0063, + "step": 4130 + }, + { + "epoch": 0.6685371712319089, + "grad_norm": 0.8471246361732483, + "learning_rate": 0.0001, + "loss": 77.8871, + "step": 4140 + }, + { + "epoch": 0.6701519953170102, + "grad_norm": 0.7378994822502136, + "learning_rate": 0.0001, + "loss": 76.9109, + "step": 4150 + }, + { + "epoch": 0.6717668194021114, + "grad_norm": 0.7778745889663696, + "learning_rate": 0.0001, + "loss": 78.3374, + "step": 4160 + }, + { + "epoch": 0.6733816434872126, + "grad_norm": 0.7841216325759888, + "learning_rate": 0.0001, + "loss": 78.1657, + "step": 4170 + }, + { + "epoch": 0.6749964675723138, + "grad_norm": 1.0027995109558105, + "learning_rate": 0.0001, + "loss": 76.6232, + "step": 4180 + }, + { + "epoch": 0.6766112916574151, + "grad_norm": 1.0034840106964111, + "learning_rate": 0.0001, + "loss": 79.5785, + "step": 4190 + }, + { + "epoch": 0.6782261157425163, + "grad_norm": 0.8399350643157959, + "learning_rate": 0.0001, + "loss": 76.8101, + "step": 4200 + }, + { + "epoch": 0.6798409398276175, + "grad_norm": 0.8864424824714661, + "learning_rate": 0.0001, + "loss": 77.8671, + "step": 4210 + }, + { + "epoch": 0.6814557639127188, + "grad_norm": 0.8322188258171082, + "learning_rate": 0.0001, + "loss": 77.6597, + "step": 4220 + }, + { + "epoch": 0.68307058799782, + "grad_norm": 0.8962367177009583, + "learning_rate": 0.0001, + "loss": 77.7324, + "step": 4230 + }, + { + "epoch": 0.6846854120829212, + "grad_norm": 0.9727205038070679, + "learning_rate": 0.0001, + "loss": 77.8531, + "step": 4240 + }, + { + "epoch": 0.6863002361680225, + "grad_norm": 0.956584095954895, + "learning_rate": 0.0001, + "loss": 78.709, + "step": 4250 + }, + { + "epoch": 0.6879150602531237, + "grad_norm": 0.8531600832939148, + "learning_rate": 0.0001, + "loss": 78.7755, + "step": 4260 + }, + { + "epoch": 0.6895298843382249, + "grad_norm": 0.8059844374656677, + "learning_rate": 0.0001, + "loss": 77.1636, + "step": 4270 + }, + { + "epoch": 0.6911447084233261, + "grad_norm": 0.8373456597328186, + "learning_rate": 0.0001, + "loss": 78.6561, + "step": 4280 + }, + { + "epoch": 0.6927595325084274, + "grad_norm": 0.7819933891296387, + "learning_rate": 0.0001, + "loss": 77.6061, + "step": 4290 + }, + { + "epoch": 0.6943743565935286, + "grad_norm": 0.8829817771911621, + "learning_rate": 0.0001, + "loss": 77.3387, + "step": 4300 + }, + { + "epoch": 0.6959891806786298, + "grad_norm": 0.791772723197937, + "learning_rate": 0.0001, + "loss": 77.4368, + "step": 4310 + }, + { + "epoch": 0.6976040047637311, + "grad_norm": 0.9949231147766113, + "learning_rate": 0.0001, + "loss": 73.4359, + "step": 4320 + }, + { + "epoch": 0.6992188288488322, + "grad_norm": 0.8676188588142395, + "learning_rate": 0.0001, + "loss": 78.4836, + "step": 4330 + }, + { + "epoch": 0.7008336529339335, + "grad_norm": 1.1704007387161255, + "learning_rate": 0.0001, + "loss": 76.5091, + "step": 4340 + }, + { + "epoch": 0.7024484770190348, + "grad_norm": 0.8877903819084167, + "learning_rate": 0.0001, + "loss": 78.021, + "step": 4350 + }, + { + "epoch": 0.704063301104136, + "grad_norm": 1.381914496421814, + "learning_rate": 0.0001, + "loss": 77.8544, + "step": 4360 + }, + { + "epoch": 0.7056781251892372, + "grad_norm": 0.835023045539856, + "learning_rate": 0.0001, + "loss": 76.1552, + "step": 4370 + }, + { + "epoch": 0.7072929492743384, + "grad_norm": 0.9814149141311646, + "learning_rate": 0.0001, + "loss": 76.7646, + "step": 4380 + }, + { + "epoch": 0.7089077733594397, + "grad_norm": 0.8012816309928894, + "learning_rate": 0.0001, + "loss": 75.794, + "step": 4390 + }, + { + "epoch": 0.7105225974445409, + "grad_norm": 0.8340234160423279, + "learning_rate": 0.0001, + "loss": 77.4417, + "step": 4400 + }, + { + "epoch": 0.7121374215296421, + "grad_norm": 0.7257950305938721, + "learning_rate": 0.0001, + "loss": 77.1807, + "step": 4410 + }, + { + "epoch": 0.7137522456147434, + "grad_norm": 0.7342708706855774, + "learning_rate": 0.0001, + "loss": 75.5091, + "step": 4420 + }, + { + "epoch": 0.7153670696998445, + "grad_norm": 0.8298904895782471, + "learning_rate": 0.0001, + "loss": 76.6651, + "step": 4430 + }, + { + "epoch": 0.7169818937849458, + "grad_norm": 0.8564396500587463, + "learning_rate": 0.0001, + "loss": 76.0055, + "step": 4440 + }, + { + "epoch": 0.7185967178700471, + "grad_norm": 0.9145296812057495, + "learning_rate": 0.0001, + "loss": 76.0369, + "step": 4450 + }, + { + "epoch": 0.7202115419551482, + "grad_norm": 0.8074191212654114, + "learning_rate": 0.0001, + "loss": 77.4436, + "step": 4460 + }, + { + "epoch": 0.7218263660402495, + "grad_norm": 0.8212682008743286, + "learning_rate": 0.0001, + "loss": 76.3129, + "step": 4470 + }, + { + "epoch": 0.7234411901253507, + "grad_norm": 0.751323938369751, + "learning_rate": 0.0001, + "loss": 77.603, + "step": 4480 + }, + { + "epoch": 0.725056014210452, + "grad_norm": 0.9664170145988464, + "learning_rate": 0.0001, + "loss": 77.481, + "step": 4490 + }, + { + "epoch": 0.7266708382955531, + "grad_norm": 0.7369455695152283, + "learning_rate": 0.0001, + "loss": 76.3238, + "step": 4500 + }, + { + "epoch": 0.7282856623806544, + "grad_norm": 0.8448513746261597, + "learning_rate": 0.0001, + "loss": 76.5679, + "step": 4510 + }, + { + "epoch": 0.7299004864657557, + "grad_norm": 0.8605676293373108, + "learning_rate": 0.0001, + "loss": 75.7667, + "step": 4520 + }, + { + "epoch": 0.7315153105508568, + "grad_norm": 0.8034664392471313, + "learning_rate": 0.0001, + "loss": 76.2163, + "step": 4530 + }, + { + "epoch": 0.7331301346359581, + "grad_norm": 0.7846826910972595, + "learning_rate": 0.0001, + "loss": 73.5979, + "step": 4540 + }, + { + "epoch": 0.7347449587210593, + "grad_norm": 0.7422606348991394, + "learning_rate": 0.0001, + "loss": 76.2334, + "step": 4550 + }, + { + "epoch": 0.7363597828061605, + "grad_norm": 1.4478483200073242, + "learning_rate": 0.0001, + "loss": 77.6168, + "step": 4560 + }, + { + "epoch": 0.7379746068912618, + "grad_norm": 2.0957767963409424, + "learning_rate": 0.0001, + "loss": 76.3199, + "step": 4570 + }, + { + "epoch": 0.739589430976363, + "grad_norm": 0.6757177114486694, + "learning_rate": 0.0001, + "loss": 77.2923, + "step": 4580 + }, + { + "epoch": 0.7412042550614643, + "grad_norm": 0.9281920790672302, + "learning_rate": 0.0001, + "loss": 76.9963, + "step": 4590 + }, + { + "epoch": 0.7428190791465654, + "grad_norm": 0.8702669143676758, + "learning_rate": 0.0001, + "loss": 76.2553, + "step": 4600 + }, + { + "epoch": 0.7444339032316667, + "grad_norm": 0.7976512312889099, + "learning_rate": 0.0001, + "loss": 74.7622, + "step": 4610 + }, + { + "epoch": 0.746048727316768, + "grad_norm": 0.9116189479827881, + "learning_rate": 0.0001, + "loss": 76.3198, + "step": 4620 + }, + { + "epoch": 0.7476635514018691, + "grad_norm": 0.6239303946495056, + "learning_rate": 0.0001, + "loss": 75.7678, + "step": 4630 + }, + { + "epoch": 0.7492783754869704, + "grad_norm": 0.8380976915359497, + "learning_rate": 0.0001, + "loss": 76.8305, + "step": 4640 + }, + { + "epoch": 0.7508931995720716, + "grad_norm": 0.7394141554832458, + "learning_rate": 0.0001, + "loss": 76.5033, + "step": 4650 + }, + { + "epoch": 0.7525080236571728, + "grad_norm": 1.0694036483764648, + "learning_rate": 0.0001, + "loss": 75.0084, + "step": 4660 + }, + { + "epoch": 0.7541228477422741, + "grad_norm": 0.7530115246772766, + "learning_rate": 0.0001, + "loss": 74.4977, + "step": 4670 + }, + { + "epoch": 0.7557376718273753, + "grad_norm": 1.9561909437179565, + "learning_rate": 0.0001, + "loss": 74.7077, + "step": 4680 + }, + { + "epoch": 0.7573524959124766, + "grad_norm": 0.625341534614563, + "learning_rate": 0.0001, + "loss": 76.1238, + "step": 4690 + }, + { + "epoch": 0.7589673199975777, + "grad_norm": 0.9790500998497009, + "learning_rate": 0.0001, + "loss": 76.3997, + "step": 4700 + }, + { + "epoch": 0.760582144082679, + "grad_norm": 0.7087915539741516, + "learning_rate": 0.0001, + "loss": 75.6893, + "step": 4710 + }, + { + "epoch": 0.7621969681677803, + "grad_norm": 1.0995875597000122, + "learning_rate": 0.0001, + "loss": 74.8588, + "step": 4720 + }, + { + "epoch": 0.7638117922528814, + "grad_norm": 1.0309056043624878, + "learning_rate": 0.0001, + "loss": 73.5934, + "step": 4730 + }, + { + "epoch": 0.7654266163379827, + "grad_norm": 1.0132921934127808, + "learning_rate": 0.0001, + "loss": 73.9868, + "step": 4740 + }, + { + "epoch": 0.7670414404230839, + "grad_norm": 0.7138039469718933, + "learning_rate": 0.0001, + "loss": 76.4459, + "step": 4750 + }, + { + "epoch": 0.7686562645081851, + "grad_norm": 0.7055657505989075, + "learning_rate": 0.0001, + "loss": 75.1068, + "step": 4760 + }, + { + "epoch": 0.7702710885932864, + "grad_norm": 0.7398490905761719, + "learning_rate": 0.0001, + "loss": 75.3697, + "step": 4770 + }, + { + "epoch": 0.7718859126783876, + "grad_norm": 0.6836295127868652, + "learning_rate": 0.0001, + "loss": 77.4988, + "step": 4780 + }, + { + "epoch": 0.7735007367634888, + "grad_norm": 0.7552574276924133, + "learning_rate": 0.0001, + "loss": 75.149, + "step": 4790 + }, + { + "epoch": 0.77511556084859, + "grad_norm": 0.7930851578712463, + "learning_rate": 0.0001, + "loss": 73.8661, + "step": 4800 + }, + { + "epoch": 0.7767303849336913, + "grad_norm": 0.7549886107444763, + "learning_rate": 0.0001, + "loss": 75.042, + "step": 4810 + }, + { + "epoch": 0.7783452090187926, + "grad_norm": 0.9932214617729187, + "learning_rate": 0.0001, + "loss": 75.6619, + "step": 4820 + }, + { + "epoch": 0.7799600331038937, + "grad_norm": 0.7685471773147583, + "learning_rate": 0.0001, + "loss": 75.8404, + "step": 4830 + }, + { + "epoch": 0.781574857188995, + "grad_norm": 0.828737199306488, + "learning_rate": 0.0001, + "loss": 75.5839, + "step": 4840 + }, + { + "epoch": 0.7831896812740962, + "grad_norm": 0.8575420379638672, + "learning_rate": 0.0001, + "loss": 75.7889, + "step": 4850 + }, + { + "epoch": 0.7848045053591974, + "grad_norm": 0.7053834199905396, + "learning_rate": 0.0001, + "loss": 75.9372, + "step": 4860 + }, + { + "epoch": 0.7864193294442987, + "grad_norm": 0.7861505150794983, + "learning_rate": 0.0001, + "loss": 76.1532, + "step": 4870 + }, + { + "epoch": 0.7880341535293999, + "grad_norm": 0.8357386589050293, + "learning_rate": 0.0001, + "loss": 75.8536, + "step": 4880 + }, + { + "epoch": 0.7896489776145011, + "grad_norm": 3.448834180831909, + "learning_rate": 0.0001, + "loss": 74.1794, + "step": 4890 + }, + { + "epoch": 0.7912638016996023, + "grad_norm": 0.813522458076477, + "learning_rate": 0.0001, + "loss": 74.5202, + "step": 4900 + }, + { + "epoch": 0.7928786257847036, + "grad_norm": 0.840230405330658, + "learning_rate": 0.0001, + "loss": 74.6634, + "step": 4910 + }, + { + "epoch": 0.7944934498698049, + "grad_norm": 0.7855745553970337, + "learning_rate": 0.0001, + "loss": 73.5369, + "step": 4920 + }, + { + "epoch": 0.796108273954906, + "grad_norm": 0.7708212733268738, + "learning_rate": 0.0001, + "loss": 74.6225, + "step": 4930 + }, + { + "epoch": 0.7977230980400073, + "grad_norm": 0.7140830159187317, + "learning_rate": 0.0001, + "loss": 75.0585, + "step": 4940 + }, + { + "epoch": 0.7993379221251085, + "grad_norm": 0.7960936427116394, + "learning_rate": 0.0001, + "loss": 74.0574, + "step": 4950 + }, + { + "epoch": 0.8009527462102097, + "grad_norm": 0.7269165515899658, + "learning_rate": 0.0001, + "loss": 74.0379, + "step": 4960 + }, + { + "epoch": 0.802567570295311, + "grad_norm": 0.8505013585090637, + "learning_rate": 0.0001, + "loss": 74.7448, + "step": 4970 + }, + { + "epoch": 0.8041823943804122, + "grad_norm": 0.8935654759407043, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 4980 + }, + { + "epoch": 0.8057972184655134, + "grad_norm": 0.9447595477104187, + "learning_rate": 0.0001, + "loss": 75.9674, + "step": 4990 + }, + { + "epoch": 0.8074120425506146, + "grad_norm": 1.6739715337753296, + "learning_rate": 0.0001, + "loss": 73.9035, + "step": 5000 + }, + { + "epoch": 0.8090268666357159, + "grad_norm": 0.861902117729187, + "learning_rate": 0.0001, + "loss": 74.906, + "step": 5010 + }, + { + "epoch": 0.8106416907208172, + "grad_norm": 0.8328655362129211, + "learning_rate": 0.0001, + "loss": 74.1493, + "step": 5020 + }, + { + "epoch": 0.8122565148059183, + "grad_norm": 0.8193135261535645, + "learning_rate": 0.0001, + "loss": 74.6614, + "step": 5030 + }, + { + "epoch": 0.8138713388910196, + "grad_norm": 0.6954565048217773, + "learning_rate": 0.0001, + "loss": 73.7366, + "step": 5040 + }, + { + "epoch": 0.8154861629761208, + "grad_norm": 0.7381289005279541, + "learning_rate": 0.0001, + "loss": 73.5086, + "step": 5050 + }, + { + "epoch": 0.817100987061222, + "grad_norm": 1.090443730354309, + "learning_rate": 0.0001, + "loss": 75.0695, + "step": 5060 + }, + { + "epoch": 0.8187158111463233, + "grad_norm": 1.2544814348220825, + "learning_rate": 0.0001, + "loss": 72.89, + "step": 5070 + }, + { + "epoch": 0.8203306352314245, + "grad_norm": 0.632146418094635, + "learning_rate": 0.0001, + "loss": 74.8604, + "step": 5080 + }, + { + "epoch": 0.8219454593165257, + "grad_norm": 1.1038259267807007, + "learning_rate": 0.0001, + "loss": 74.7354, + "step": 5090 + }, + { + "epoch": 0.8235602834016269, + "grad_norm": 0.6886508464813232, + "learning_rate": 0.0001, + "loss": 75.4243, + "step": 5100 + }, + { + "epoch": 0.8251751074867282, + "grad_norm": 0.8367568850517273, + "learning_rate": 0.0001, + "loss": 75.2732, + "step": 5110 + }, + { + "epoch": 0.8267899315718293, + "grad_norm": 0.9302715063095093, + "learning_rate": 0.0001, + "loss": 75.7924, + "step": 5120 + }, + { + "epoch": 0.8284047556569306, + "grad_norm": 0.6871219873428345, + "learning_rate": 0.0001, + "loss": 74.2905, + "step": 5130 + }, + { + "epoch": 0.8300195797420319, + "grad_norm": 0.7319445013999939, + "learning_rate": 0.0001, + "loss": 73.7774, + "step": 5140 + }, + { + "epoch": 0.831634403827133, + "grad_norm": 0.7184522151947021, + "learning_rate": 0.0001, + "loss": 73.0554, + "step": 5150 + }, + { + "epoch": 0.8332492279122343, + "grad_norm": 0.8462343811988831, + "learning_rate": 0.0001, + "loss": 73.7615, + "step": 5160 + }, + { + "epoch": 0.8348640519973355, + "grad_norm": 0.9436241984367371, + "learning_rate": 0.0001, + "loss": 73.9464, + "step": 5170 + }, + { + "epoch": 0.8364788760824368, + "grad_norm": 0.9079630374908447, + "learning_rate": 0.0001, + "loss": 73.2727, + "step": 5180 + }, + { + "epoch": 0.838093700167538, + "grad_norm": 0.8280741572380066, + "learning_rate": 0.0001, + "loss": 73.9751, + "step": 5190 + }, + { + "epoch": 0.8397085242526392, + "grad_norm": 0.6475157141685486, + "learning_rate": 0.0001, + "loss": 74.7105, + "step": 5200 + }, + { + "epoch": 0.8413233483377405, + "grad_norm": 0.7090312838554382, + "learning_rate": 0.0001, + "loss": 73.6489, + "step": 5210 + }, + { + "epoch": 0.8429381724228416, + "grad_norm": 1.20967698097229, + "learning_rate": 0.0001, + "loss": 75.1335, + "step": 5220 + }, + { + "epoch": 0.8445529965079429, + "grad_norm": 2.053532600402832, + "learning_rate": 0.0001, + "loss": 75.3111, + "step": 5230 + }, + { + "epoch": 0.8461678205930442, + "grad_norm": 0.7563302516937256, + "learning_rate": 0.0001, + "loss": 72.9935, + "step": 5240 + }, + { + "epoch": 0.8477826446781453, + "grad_norm": 0.7801489233970642, + "learning_rate": 0.0001, + "loss": 74.7856, + "step": 5250 + }, + { + "epoch": 0.8493974687632466, + "grad_norm": 0.737870454788208, + "learning_rate": 0.0001, + "loss": 75.6281, + "step": 5260 + }, + { + "epoch": 0.8510122928483478, + "grad_norm": 0.704352617263794, + "learning_rate": 0.0001, + "loss": 75.4598, + "step": 5270 + }, + { + "epoch": 0.8526271169334491, + "grad_norm": 0.6452057361602783, + "learning_rate": 0.0001, + "loss": 74.0341, + "step": 5280 + }, + { + "epoch": 0.8542419410185503, + "grad_norm": 0.6863471865653992, + "learning_rate": 0.0001, + "loss": 74.3727, + "step": 5290 + }, + { + "epoch": 0.8558567651036515, + "grad_norm": 9.756170272827148, + "learning_rate": 0.0001, + "loss": 75.5231, + "step": 5300 + }, + { + "epoch": 0.8574715891887528, + "grad_norm": 0.8624544739723206, + "learning_rate": 0.0001, + "loss": 74.7821, + "step": 5310 + }, + { + "epoch": 0.8590864132738539, + "grad_norm": 0.6637155413627625, + "learning_rate": 0.0001, + "loss": 74.1153, + "step": 5320 + }, + { + "epoch": 0.8607012373589552, + "grad_norm": 1.0764445066452026, + "learning_rate": 0.0001, + "loss": 73.7208, + "step": 5330 + }, + { + "epoch": 0.8623160614440565, + "grad_norm": 0.8567569255828857, + "learning_rate": 0.0001, + "loss": 73.1399, + "step": 5340 + }, + { + "epoch": 0.8639308855291576, + "grad_norm": 0.7681576013565063, + "learning_rate": 0.0001, + "loss": 73.7577, + "step": 5350 + }, + { + "epoch": 0.8655457096142589, + "grad_norm": 0.9336384534835815, + "learning_rate": 0.0001, + "loss": 74.1022, + "step": 5360 + }, + { + "epoch": 0.8671605336993601, + "grad_norm": 0.8603724837303162, + "learning_rate": 0.0001, + "loss": 75.9579, + "step": 5370 + }, + { + "epoch": 0.8687753577844614, + "grad_norm": 0.77194744348526, + "learning_rate": 0.0001, + "loss": 72.4799, + "step": 5380 + }, + { + "epoch": 0.8703901818695626, + "grad_norm": 0.7480376958847046, + "learning_rate": 0.0001, + "loss": 76.1257, + "step": 5390 + }, + { + "epoch": 0.8720050059546638, + "grad_norm": 0.7841023206710815, + "learning_rate": 0.0001, + "loss": 73.1512, + "step": 5400 + }, + { + "epoch": 0.8736198300397651, + "grad_norm": 0.8741798996925354, + "learning_rate": 0.0001, + "loss": 74.2151, + "step": 5410 + }, + { + "epoch": 0.8752346541248662, + "grad_norm": 0.9762082099914551, + "learning_rate": 0.0001, + "loss": 76.7138, + "step": 5420 + }, + { + "epoch": 0.8768494782099675, + "grad_norm": 0.7670348286628723, + "learning_rate": 0.0001, + "loss": 72.9567, + "step": 5430 + }, + { + "epoch": 0.8784643022950688, + "grad_norm": 0.7449469566345215, + "learning_rate": 0.0001, + "loss": 74.8841, + "step": 5440 + }, + { + "epoch": 0.8800791263801699, + "grad_norm": 0.6984641551971436, + "learning_rate": 0.0001, + "loss": 73.8906, + "step": 5450 + }, + { + "epoch": 0.8816939504652712, + "grad_norm": 0.7854361534118652, + "learning_rate": 0.0001, + "loss": 74.626, + "step": 5460 + }, + { + "epoch": 0.8833087745503724, + "grad_norm": 0.9584240317344666, + "learning_rate": 0.0001, + "loss": 74.7048, + "step": 5470 + }, + { + "epoch": 0.8849235986354737, + "grad_norm": 0.6040284037590027, + "learning_rate": 0.0001, + "loss": 74.7194, + "step": 5480 + }, + { + "epoch": 0.8865384227205749, + "grad_norm": 0.7756215929985046, + "learning_rate": 0.0001, + "loss": 73.8178, + "step": 5490 + }, + { + "epoch": 0.8881532468056761, + "grad_norm": 0.9070586562156677, + "learning_rate": 0.0001, + "loss": 72.8309, + "step": 5500 + }, + { + "epoch": 0.8897680708907774, + "grad_norm": 0.8464986681938171, + "learning_rate": 0.0001, + "loss": 73.8847, + "step": 5510 + }, + { + "epoch": 0.8913828949758785, + "grad_norm": 0.7968605160713196, + "learning_rate": 0.0001, + "loss": 73.1691, + "step": 5520 + }, + { + "epoch": 0.8929977190609798, + "grad_norm": 0.9934583306312561, + "learning_rate": 0.0001, + "loss": 76.2662, + "step": 5530 + }, + { + "epoch": 0.8946125431460811, + "grad_norm": 0.7272152900695801, + "learning_rate": 0.0001, + "loss": 75.5295, + "step": 5540 + }, + { + "epoch": 0.8962273672311822, + "grad_norm": 0.857345461845398, + "learning_rate": 0.0001, + "loss": 75.3703, + "step": 5550 + }, + { + "epoch": 0.8978421913162835, + "grad_norm": 0.713918924331665, + "learning_rate": 0.0001, + "loss": 73.4563, + "step": 5560 + }, + { + "epoch": 0.8994570154013847, + "grad_norm": 0.759233832359314, + "learning_rate": 0.0001, + "loss": 73.2744, + "step": 5570 + }, + { + "epoch": 0.901071839486486, + "grad_norm": 0.7085754871368408, + "learning_rate": 0.0001, + "loss": 74.7792, + "step": 5580 + }, + { + "epoch": 0.9026866635715872, + "grad_norm": 0.7370456457138062, + "learning_rate": 0.0001, + "loss": 74.4216, + "step": 5590 + }, + { + "epoch": 0.9043014876566884, + "grad_norm": 0.779746949672699, + "learning_rate": 0.0001, + "loss": 75.9822, + "step": 5600 + }, + { + "epoch": 0.9059163117417897, + "grad_norm": 0.7915763854980469, + "learning_rate": 0.0001, + "loss": 74.3706, + "step": 5610 + }, + { + "epoch": 0.9075311358268908, + "grad_norm": 0.7886125445365906, + "learning_rate": 0.0001, + "loss": 72.9453, + "step": 5620 + }, + { + "epoch": 0.9091459599119921, + "grad_norm": 0.7804751992225647, + "learning_rate": 0.0001, + "loss": 73.514, + "step": 5630 + }, + { + "epoch": 0.9107607839970934, + "grad_norm": 0.7168806195259094, + "learning_rate": 0.0001, + "loss": 71.4159, + "step": 5640 + }, + { + "epoch": 0.9123756080821945, + "grad_norm": 0.7553626894950867, + "learning_rate": 0.0001, + "loss": 74.4353, + "step": 5650 + }, + { + "epoch": 0.9139904321672958, + "grad_norm": 0.6688916683197021, + "learning_rate": 0.0001, + "loss": 74.3786, + "step": 5660 + }, + { + "epoch": 0.915605256252397, + "grad_norm": 0.7425532341003418, + "learning_rate": 0.0001, + "loss": 73.3355, + "step": 5670 + }, + { + "epoch": 0.9172200803374982, + "grad_norm": 0.6473882794380188, + "learning_rate": 0.0001, + "loss": 74.2315, + "step": 5680 + }, + { + "epoch": 0.9188349044225994, + "grad_norm": 0.7222439050674438, + "learning_rate": 0.0001, + "loss": 74.2885, + "step": 5690 + }, + { + "epoch": 0.9204497285077007, + "grad_norm": 1.5716948509216309, + "learning_rate": 0.0001, + "loss": 74.6739, + "step": 5700 + }, + { + "epoch": 0.922064552592802, + "grad_norm": 0.8077759742736816, + "learning_rate": 0.0001, + "loss": 72.6617, + "step": 5710 + }, + { + "epoch": 0.9236793766779031, + "grad_norm": 0.6948631405830383, + "learning_rate": 0.0001, + "loss": 73.6247, + "step": 5720 + }, + { + "epoch": 0.9252942007630044, + "grad_norm": 0.8823757767677307, + "learning_rate": 0.0001, + "loss": 73.2536, + "step": 5730 + }, + { + "epoch": 0.9269090248481056, + "grad_norm": 0.7066764235496521, + "learning_rate": 0.0001, + "loss": 74.1748, + "step": 5740 + }, + { + "epoch": 0.9285238489332068, + "grad_norm": 0.6979793906211853, + "learning_rate": 0.0001, + "loss": 72.3189, + "step": 5750 + }, + { + "epoch": 0.9301386730183081, + "grad_norm": 0.6961462497711182, + "learning_rate": 0.0001, + "loss": 74.6196, + "step": 5760 + }, + { + "epoch": 0.9317534971034093, + "grad_norm": 0.7797656655311584, + "learning_rate": 0.0001, + "loss": 74.5472, + "step": 5770 + }, + { + "epoch": 0.9333683211885105, + "grad_norm": 1.0671838521957397, + "learning_rate": 0.0001, + "loss": 73.241, + "step": 5780 + }, + { + "epoch": 0.9349831452736117, + "grad_norm": 0.7566089034080505, + "learning_rate": 0.0001, + "loss": 73.6784, + "step": 5790 + }, + { + "epoch": 0.936597969358713, + "grad_norm": 0.8563273549079895, + "learning_rate": 0.0001, + "loss": 73.6871, + "step": 5800 + }, + { + "epoch": 0.9382127934438143, + "grad_norm": 0.7885987162590027, + "learning_rate": 0.0001, + "loss": 73.4447, + "step": 5810 + }, + { + "epoch": 0.9398276175289154, + "grad_norm": 0.8751112818717957, + "learning_rate": 0.0001, + "loss": 74.0531, + "step": 5820 + }, + { + "epoch": 0.9414424416140167, + "grad_norm": 0.6663336157798767, + "learning_rate": 0.0001, + "loss": 73.0557, + "step": 5830 + }, + { + "epoch": 0.9430572656991179, + "grad_norm": 0.7993099689483643, + "learning_rate": 0.0001, + "loss": 74.4467, + "step": 5840 + }, + { + "epoch": 0.9446720897842191, + "grad_norm": 0.6987188458442688, + "learning_rate": 0.0001, + "loss": 74.0327, + "step": 5850 + }, + { + "epoch": 0.9462869138693204, + "grad_norm": 0.8458583354949951, + "learning_rate": 0.0001, + "loss": 73.1757, + "step": 5860 + }, + { + "epoch": 0.9479017379544216, + "grad_norm": 0.6372436881065369, + "learning_rate": 0.0001, + "loss": 73.1067, + "step": 5870 + }, + { + "epoch": 0.9495165620395228, + "grad_norm": 0.7447753548622131, + "learning_rate": 0.0001, + "loss": 75.5461, + "step": 5880 + }, + { + "epoch": 0.951131386124624, + "grad_norm": 0.6633971929550171, + "learning_rate": 0.0001, + "loss": 72.9966, + "step": 5890 + }, + { + "epoch": 0.9527462102097253, + "grad_norm": 0.8576784729957581, + "learning_rate": 0.0001, + "loss": 71.5712, + "step": 5900 + }, + { + "epoch": 0.9543610342948265, + "grad_norm": 1.4696853160858154, + "learning_rate": 0.0001, + "loss": 74.7893, + "step": 5910 + }, + { + "epoch": 0.9559758583799277, + "grad_norm": 0.611210286617279, + "learning_rate": 0.0001, + "loss": 74.3426, + "step": 5920 + }, + { + "epoch": 0.957590682465029, + "grad_norm": 0.5947410464286804, + "learning_rate": 0.0001, + "loss": 71.2563, + "step": 5930 + }, + { + "epoch": 0.9592055065501301, + "grad_norm": 0.7169992923736572, + "learning_rate": 0.0001, + "loss": 71.6049, + "step": 5940 + }, + { + "epoch": 0.9608203306352314, + "grad_norm": 0.6773818731307983, + "learning_rate": 0.0001, + "loss": 75.0328, + "step": 5950 + }, + { + "epoch": 0.9624351547203327, + "grad_norm": 0.6714807748794556, + "learning_rate": 0.0001, + "loss": 71.5105, + "step": 5960 + }, + { + "epoch": 0.9640499788054339, + "grad_norm": 1.0064337253570557, + "learning_rate": 0.0001, + "loss": 75.8887, + "step": 5970 + }, + { + "epoch": 0.9656648028905351, + "grad_norm": 0.9802227020263672, + "learning_rate": 0.0001, + "loss": 73.5655, + "step": 5980 + }, + { + "epoch": 0.9672796269756363, + "grad_norm": 0.7097306251525879, + "learning_rate": 0.0001, + "loss": 73.298, + "step": 5990 + }, + { + "epoch": 0.9688944510607376, + "grad_norm": 0.6680361032485962, + "learning_rate": 0.0001, + "loss": 73.8873, + "step": 6000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.347007134262886e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6a151f546555c78ebc502dd7b703b665f0672d6f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35235df8408202e91954a585851d27187bafaf4e1d75c4e63c51fbf721136ff5 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-6000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/latest b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/latest new file mode 100644 index 0000000000000000000000000000000000000000..6c558c76ac9e7515267f79d24d9fde4a7f8688f1 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/latest @@ -0,0 +1 @@ +global_step8000 \ No newline at end of file diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ca222a8735c11956c97ce9e3b22182ad9d3ec947 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0cbd4c6f1ce22ead6e1a5f2a6e65959d230586a21d91eceab0f59fc418d4986 +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_0.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..41f72ced1b1d9d8070d189bbb1715bc174bffd22 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8be61455d2fb3408341c212c20ca488f3b3b97796c69f49c66b9b2413abe5d +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_1.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..1ca04355c641016dd4e8b1cbb9649f1de350a9c9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94975cc4029e019c1ca6d934fdac8325422210671d7b91070ad7632d81d237ad +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_2.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..adbe31783300a517d11cb24ad3d0ee5028685cec --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a534566748a5438c03b1a13a26115cd889c6e9b2713a964362a603f12b787271 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_3.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..852b4f2f57e61e283696f03b786ee0cddd0913b2 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a15217daffb2847f4e89eaaac0acb64b04fcc8ead6af7fef7ea61ea61616a841 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_4.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..9372367385358c9e1d84fb85a111302a72d1667b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b314487f51a4c15ae7dfa133230ee748274f47ae889fa91c91afbc41e31b517 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_5.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_5.pth new file mode 100644 index 0000000000000000000000000000000000000000..1514c2e861d9fb1d3a530368e1f59eefdb0aafba --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b8678c585b4e1606fe76297c40edc592c1274384ef2ef0e0ac149b638307fb8 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_6.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_6.pth new file mode 100644 index 0000000000000000000000000000000000000000..6d1c3da08000daedcd0ff83febf4d57026de8e3a --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5357abc7774681d691ab8c78df7147bc663f70e2d5ca1e8be981c71a8c2a3c80 +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_7.pth b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_7.pth new file mode 100644 index 0000000000000000000000000000000000000000..d25bbac3491d02efab1093128a3034c9a4c1cf7e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5a66ecf45b994af3be2f9a8f9d576af22e1840a74555c13753f60c2e131b44b +size 15984 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/scheduler.pt b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8fb4ef8eda5769247453e7f4c1cbe7f89242b5c --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f130956eca2bbfbb04376b671db10d86b648fabcae3263ea3116e90a644e7ab1 +size 1064 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/trainer_state.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..18b8c755781128a91c502c2928e7d149ef893074 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/trainer_state.json @@ -0,0 +1,5634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.291798712177792, + "eval_steps": 2000, + "global_step": 8000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0016148240851012292, + "grad_norm": 43.90101623535156, + "learning_rate": 0.0001, + "loss": 307.2084, + "step": 10 + }, + { + "epoch": 0.0032296481702024584, + "grad_norm": 19.106313705444336, + "learning_rate": 0.0001, + "loss": 320.2794, + "step": 20 + }, + { + "epoch": 0.004844472255303688, + "grad_norm": 5.771847248077393, + "learning_rate": 0.0001, + "loss": 247.4977, + "step": 30 + }, + { + "epoch": 0.006459296340404917, + "grad_norm": 2.630659580230713, + "learning_rate": 0.0001, + "loss": 229.9593, + "step": 40 + }, + { + "epoch": 0.008074120425506147, + "grad_norm": 2.487499475479126, + "learning_rate": 0.0001, + "loss": 228.65, + "step": 50 + }, + { + "epoch": 0.009688944510607376, + "grad_norm": 2.078263521194458, + "learning_rate": 0.0001, + "loss": 227.648, + "step": 60 + }, + { + "epoch": 0.011303768595708604, + "grad_norm": 3.105417490005493, + "learning_rate": 0.0001, + "loss": 227.9345, + "step": 70 + }, + { + "epoch": 0.012918592680809833, + "grad_norm": 2.2145848274230957, + "learning_rate": 0.0001, + "loss": 227.3662, + "step": 80 + }, + { + "epoch": 0.014533416765911064, + "grad_norm": 1.9644615650177002, + "learning_rate": 0.0001, + "loss": 226.9145, + "step": 90 + }, + { + "epoch": 0.016148240851012293, + "grad_norm": 1.867382287979126, + "learning_rate": 0.0001, + "loss": 227.5703, + "step": 100 + }, + { + "epoch": 0.017763064936113522, + "grad_norm": 1.4613710641860962, + "learning_rate": 0.0001, + "loss": 227.1107, + "step": 110 + }, + { + "epoch": 0.01937788902121475, + "grad_norm": 2.8953869342803955, + "learning_rate": 0.0001, + "loss": 226.8185, + "step": 120 + }, + { + "epoch": 0.02099271310631598, + "grad_norm": 1.8446482419967651, + "learning_rate": 0.0001, + "loss": 226.9624, + "step": 130 + }, + { + "epoch": 0.02260753719141721, + "grad_norm": 1.3048362731933594, + "learning_rate": 0.0001, + "loss": 226.8404, + "step": 140 + }, + { + "epoch": 0.024222361276518438, + "grad_norm": 1.7100074291229248, + "learning_rate": 0.0001, + "loss": 226.1071, + "step": 150 + }, + { + "epoch": 0.025837185361619667, + "grad_norm": 1.7875330448150635, + "learning_rate": 0.0001, + "loss": 226.0764, + "step": 160 + }, + { + "epoch": 0.0274520094467209, + "grad_norm": 1.7503769397735596, + "learning_rate": 0.0001, + "loss": 226.0772, + "step": 170 + }, + { + "epoch": 0.029066833531822128, + "grad_norm": 2.7083983421325684, + "learning_rate": 0.0001, + "loss": 224.7667, + "step": 180 + }, + { + "epoch": 0.030681657616923357, + "grad_norm": 1.8192371129989624, + "learning_rate": 0.0001, + "loss": 223.6494, + "step": 190 + }, + { + "epoch": 0.032296481702024586, + "grad_norm": 3.210358142852783, + "learning_rate": 0.0001, + "loss": 223.3545, + "step": 200 + }, + { + "epoch": 0.03391130578712581, + "grad_norm": 5.819669246673584, + "learning_rate": 0.0001, + "loss": 222.1732, + "step": 210 + }, + { + "epoch": 0.035526129872227044, + "grad_norm": 4.185793399810791, + "learning_rate": 0.0001, + "loss": 221.6866, + "step": 220 + }, + { + "epoch": 0.03714095395732828, + "grad_norm": 2.7514216899871826, + "learning_rate": 0.0001, + "loss": 221.0769, + "step": 230 + }, + { + "epoch": 0.0387557780424295, + "grad_norm": 1.5478988885879517, + "learning_rate": 0.0001, + "loss": 219.7232, + "step": 240 + }, + { + "epoch": 0.040370602127530734, + "grad_norm": 1.4958223104476929, + "learning_rate": 0.0001, + "loss": 218.5454, + "step": 250 + }, + { + "epoch": 0.04198542621263196, + "grad_norm": 6.36635684967041, + "learning_rate": 0.0001, + "loss": 218.5061, + "step": 260 + }, + { + "epoch": 0.04360025029773319, + "grad_norm": 3.029601812362671, + "learning_rate": 0.0001, + "loss": 216.3466, + "step": 270 + }, + { + "epoch": 0.04521507438283442, + "grad_norm": 2.6436100006103516, + "learning_rate": 0.0001, + "loss": 214.0003, + "step": 280 + }, + { + "epoch": 0.04682989846793565, + "grad_norm": 5.329655647277832, + "learning_rate": 0.0001, + "loss": 212.9169, + "step": 290 + }, + { + "epoch": 0.048444722553036876, + "grad_norm": 4.930875301361084, + "learning_rate": 0.0001, + "loss": 209.7737, + "step": 300 + }, + { + "epoch": 0.05005954663813811, + "grad_norm": 18.902585983276367, + "learning_rate": 0.0001, + "loss": 209.4267, + "step": 310 + }, + { + "epoch": 0.051674370723239334, + "grad_norm": 5.364975929260254, + "learning_rate": 0.0001, + "loss": 206.49, + "step": 320 + }, + { + "epoch": 0.053289194808340566, + "grad_norm": 3.1421146392822266, + "learning_rate": 0.0001, + "loss": 204.8132, + "step": 330 + }, + { + "epoch": 0.0549040188934418, + "grad_norm": 3.399320602416992, + "learning_rate": 0.0001, + "loss": 202.4785, + "step": 340 + }, + { + "epoch": 0.056518842978543024, + "grad_norm": 5.137377738952637, + "learning_rate": 0.0001, + "loss": 199.5143, + "step": 350 + }, + { + "epoch": 0.058133667063644257, + "grad_norm": 5.4967522621154785, + "learning_rate": 0.0001, + "loss": 196.2809, + "step": 360 + }, + { + "epoch": 0.05974849114874548, + "grad_norm": 6.435070037841797, + "learning_rate": 0.0001, + "loss": 196.8831, + "step": 370 + }, + { + "epoch": 0.061363315233846714, + "grad_norm": 5.7334370613098145, + "learning_rate": 0.0001, + "loss": 193.141, + "step": 380 + }, + { + "epoch": 0.06297813931894794, + "grad_norm": 3.6522979736328125, + "learning_rate": 0.0001, + "loss": 191.5819, + "step": 390 + }, + { + "epoch": 0.06459296340404917, + "grad_norm": 2.614051103591919, + "learning_rate": 0.0001, + "loss": 190.046, + "step": 400 + }, + { + "epoch": 0.0662077874891504, + "grad_norm": 4.845924377441406, + "learning_rate": 0.0001, + "loss": 188.7121, + "step": 410 + }, + { + "epoch": 0.06782261157425162, + "grad_norm": 4.319681644439697, + "learning_rate": 0.0001, + "loss": 185.3697, + "step": 420 + }, + { + "epoch": 0.06943743565935286, + "grad_norm": 2.7018792629241943, + "learning_rate": 0.0001, + "loss": 183.5171, + "step": 430 + }, + { + "epoch": 0.07105225974445409, + "grad_norm": 2.6407113075256348, + "learning_rate": 0.0001, + "loss": 181.245, + "step": 440 + }, + { + "epoch": 0.07266708382955532, + "grad_norm": 3.314251184463501, + "learning_rate": 0.0001, + "loss": 179.3874, + "step": 450 + }, + { + "epoch": 0.07428190791465655, + "grad_norm": 2.635136365890503, + "learning_rate": 0.0001, + "loss": 177.8257, + "step": 460 + }, + { + "epoch": 0.07589673199975777, + "grad_norm": 5.227738857269287, + "learning_rate": 0.0001, + "loss": 176.4699, + "step": 470 + }, + { + "epoch": 0.077511556084859, + "grad_norm": 4.998415470123291, + "learning_rate": 0.0001, + "loss": 175.7601, + "step": 480 + }, + { + "epoch": 0.07912638016996024, + "grad_norm": 2.946892261505127, + "learning_rate": 0.0001, + "loss": 173.936, + "step": 490 + }, + { + "epoch": 0.08074120425506147, + "grad_norm": 4.20358943939209, + "learning_rate": 0.0001, + "loss": 172.065, + "step": 500 + }, + { + "epoch": 0.08235602834016269, + "grad_norm": 3.125765562057495, + "learning_rate": 0.0001, + "loss": 171.3727, + "step": 510 + }, + { + "epoch": 0.08397085242526392, + "grad_norm": 2.4870612621307373, + "learning_rate": 0.0001, + "loss": 169.9, + "step": 520 + }, + { + "epoch": 0.08558567651036515, + "grad_norm": 3.917384147644043, + "learning_rate": 0.0001, + "loss": 168.5049, + "step": 530 + }, + { + "epoch": 0.08720050059546638, + "grad_norm": 2.302142858505249, + "learning_rate": 0.0001, + "loss": 167.2027, + "step": 540 + }, + { + "epoch": 0.08881532468056762, + "grad_norm": 4.794841289520264, + "learning_rate": 0.0001, + "loss": 166.0887, + "step": 550 + }, + { + "epoch": 0.09043014876566884, + "grad_norm": 2.200291395187378, + "learning_rate": 0.0001, + "loss": 164.5882, + "step": 560 + }, + { + "epoch": 0.09204497285077007, + "grad_norm": 1.9572436809539795, + "learning_rate": 0.0001, + "loss": 163.3007, + "step": 570 + }, + { + "epoch": 0.0936597969358713, + "grad_norm": 3.443633556365967, + "learning_rate": 0.0001, + "loss": 160.6805, + "step": 580 + }, + { + "epoch": 0.09527462102097253, + "grad_norm": 2.6079020500183105, + "learning_rate": 0.0001, + "loss": 160.3372, + "step": 590 + }, + { + "epoch": 0.09688944510607375, + "grad_norm": 2.4401192665100098, + "learning_rate": 0.0001, + "loss": 160.0956, + "step": 600 + }, + { + "epoch": 0.09850426919117498, + "grad_norm": 2.686101198196411, + "learning_rate": 0.0001, + "loss": 157.6138, + "step": 610 + }, + { + "epoch": 0.10011909327627622, + "grad_norm": 2.594151496887207, + "learning_rate": 0.0001, + "loss": 156.0726, + "step": 620 + }, + { + "epoch": 0.10173391736137745, + "grad_norm": 2.509141683578491, + "learning_rate": 0.0001, + "loss": 155.9923, + "step": 630 + }, + { + "epoch": 0.10334874144647867, + "grad_norm": 2.4077377319335938, + "learning_rate": 0.0001, + "loss": 155.214, + "step": 640 + }, + { + "epoch": 0.1049635655315799, + "grad_norm": 4.083552837371826, + "learning_rate": 0.0001, + "loss": 152.8257, + "step": 650 + }, + { + "epoch": 0.10657838961668113, + "grad_norm": 2.7188713550567627, + "learning_rate": 0.0001, + "loss": 151.875, + "step": 660 + }, + { + "epoch": 0.10819321370178236, + "grad_norm": 2.41287899017334, + "learning_rate": 0.0001, + "loss": 151.935, + "step": 670 + }, + { + "epoch": 0.1098080377868836, + "grad_norm": 3.995708465576172, + "learning_rate": 0.0001, + "loss": 151.0069, + "step": 680 + }, + { + "epoch": 0.11142286187198482, + "grad_norm": 3.217705488204956, + "learning_rate": 0.0001, + "loss": 149.5922, + "step": 690 + }, + { + "epoch": 0.11303768595708605, + "grad_norm": 3.4349842071533203, + "learning_rate": 0.0001, + "loss": 148.0044, + "step": 700 + }, + { + "epoch": 0.11465251004218728, + "grad_norm": 2.106865167617798, + "learning_rate": 0.0001, + "loss": 147.8404, + "step": 710 + }, + { + "epoch": 0.11626733412728851, + "grad_norm": 1.8729510307312012, + "learning_rate": 0.0001, + "loss": 145.1299, + "step": 720 + }, + { + "epoch": 0.11788215821238973, + "grad_norm": 4.14620304107666, + "learning_rate": 0.0001, + "loss": 144.3928, + "step": 730 + }, + { + "epoch": 0.11949698229749096, + "grad_norm": 2.202505350112915, + "learning_rate": 0.0001, + "loss": 144.8357, + "step": 740 + }, + { + "epoch": 0.1211118063825922, + "grad_norm": 2.201470375061035, + "learning_rate": 0.0001, + "loss": 143.2202, + "step": 750 + }, + { + "epoch": 0.12272663046769343, + "grad_norm": 1.5977853536605835, + "learning_rate": 0.0001, + "loss": 143.3537, + "step": 760 + }, + { + "epoch": 0.12434145455279466, + "grad_norm": 2.4403927326202393, + "learning_rate": 0.0001, + "loss": 141.2914, + "step": 770 + }, + { + "epoch": 0.12595627863789588, + "grad_norm": 1.6895846128463745, + "learning_rate": 0.0001, + "loss": 139.9639, + "step": 780 + }, + { + "epoch": 0.12757110272299713, + "grad_norm": 3.4370665550231934, + "learning_rate": 0.0001, + "loss": 140.715, + "step": 790 + }, + { + "epoch": 0.12918592680809834, + "grad_norm": 2.1449077129364014, + "learning_rate": 0.0001, + "loss": 139.2094, + "step": 800 + }, + { + "epoch": 0.13080075089319956, + "grad_norm": 2.4578912258148193, + "learning_rate": 0.0001, + "loss": 137.6724, + "step": 810 + }, + { + "epoch": 0.1324155749783008, + "grad_norm": 1.8994814157485962, + "learning_rate": 0.0001, + "loss": 136.9551, + "step": 820 + }, + { + "epoch": 0.13403039906340203, + "grad_norm": 2.6531031131744385, + "learning_rate": 0.0001, + "loss": 136.5812, + "step": 830 + }, + { + "epoch": 0.13564522314850325, + "grad_norm": 3.8680083751678467, + "learning_rate": 0.0001, + "loss": 134.3655, + "step": 840 + }, + { + "epoch": 0.1372600472336045, + "grad_norm": 1.9594208002090454, + "learning_rate": 0.0001, + "loss": 135.4958, + "step": 850 + }, + { + "epoch": 0.1388748713187057, + "grad_norm": 1.5475000143051147, + "learning_rate": 0.0001, + "loss": 135.2142, + "step": 860 + }, + { + "epoch": 0.14048969540380696, + "grad_norm": 1.6569288969039917, + "learning_rate": 0.0001, + "loss": 133.4162, + "step": 870 + }, + { + "epoch": 0.14210451948890818, + "grad_norm": 1.6724224090576172, + "learning_rate": 0.0001, + "loss": 133.7519, + "step": 880 + }, + { + "epoch": 0.1437193435740094, + "grad_norm": 1.7299500703811646, + "learning_rate": 0.0001, + "loss": 131.471, + "step": 890 + }, + { + "epoch": 0.14533416765911064, + "grad_norm": 1.9444470405578613, + "learning_rate": 0.0001, + "loss": 128.5966, + "step": 900 + }, + { + "epoch": 0.14694899174421186, + "grad_norm": 2.404723644256592, + "learning_rate": 0.0001, + "loss": 129.9824, + "step": 910 + }, + { + "epoch": 0.1485638158293131, + "grad_norm": 2.1493897438049316, + "learning_rate": 0.0001, + "loss": 128.0219, + "step": 920 + }, + { + "epoch": 0.15017863991441432, + "grad_norm": 2.8861117362976074, + "learning_rate": 0.0001, + "loss": 128.5297, + "step": 930 + }, + { + "epoch": 0.15179346399951554, + "grad_norm": 2.546415090560913, + "learning_rate": 0.0001, + "loss": 126.6817, + "step": 940 + }, + { + "epoch": 0.1534082880846168, + "grad_norm": 1.727663278579712, + "learning_rate": 0.0001, + "loss": 125.0413, + "step": 950 + }, + { + "epoch": 0.155023112169718, + "grad_norm": 3.7156920433044434, + "learning_rate": 0.0001, + "loss": 124.8085, + "step": 960 + }, + { + "epoch": 0.15663793625481925, + "grad_norm": 3.0576517581939697, + "learning_rate": 0.0001, + "loss": 125.9107, + "step": 970 + }, + { + "epoch": 0.15825276033992047, + "grad_norm": 1.9951574802398682, + "learning_rate": 0.0001, + "loss": 123.5359, + "step": 980 + }, + { + "epoch": 0.1598675844250217, + "grad_norm": 1.4620885848999023, + "learning_rate": 0.0001, + "loss": 121.9363, + "step": 990 + }, + { + "epoch": 0.16148240851012294, + "grad_norm": 1.7640169858932495, + "learning_rate": 0.0001, + "loss": 121.1764, + "step": 1000 + }, + { + "epoch": 0.16309723259522416, + "grad_norm": 2.921750545501709, + "learning_rate": 0.0001, + "loss": 119.6931, + "step": 1010 + }, + { + "epoch": 0.16471205668032537, + "grad_norm": 1.5623588562011719, + "learning_rate": 0.0001, + "loss": 119.097, + "step": 1020 + }, + { + "epoch": 0.16632688076542662, + "grad_norm": 2.2386627197265625, + "learning_rate": 0.0001, + "loss": 120.9224, + "step": 1030 + }, + { + "epoch": 0.16794170485052784, + "grad_norm": 1.3213791847229004, + "learning_rate": 0.0001, + "loss": 119.5649, + "step": 1040 + }, + { + "epoch": 0.16955652893562909, + "grad_norm": 1.5255440473556519, + "learning_rate": 0.0001, + "loss": 118.1524, + "step": 1050 + }, + { + "epoch": 0.1711713530207303, + "grad_norm": 1.6038514375686646, + "learning_rate": 0.0001, + "loss": 118.6307, + "step": 1060 + }, + { + "epoch": 0.17278617710583152, + "grad_norm": 2.356739044189453, + "learning_rate": 0.0001, + "loss": 116.8472, + "step": 1070 + }, + { + "epoch": 0.17440100119093277, + "grad_norm": 2.5326743125915527, + "learning_rate": 0.0001, + "loss": 115.6625, + "step": 1080 + }, + { + "epoch": 0.176015825276034, + "grad_norm": 1.2823396921157837, + "learning_rate": 0.0001, + "loss": 115.3312, + "step": 1090 + }, + { + "epoch": 0.17763064936113523, + "grad_norm": 1.981788158416748, + "learning_rate": 0.0001, + "loss": 116.2257, + "step": 1100 + }, + { + "epoch": 0.17924547344623645, + "grad_norm": 1.540440559387207, + "learning_rate": 0.0001, + "loss": 114.5031, + "step": 1110 + }, + { + "epoch": 0.18086029753133767, + "grad_norm": 1.7667800188064575, + "learning_rate": 0.0001, + "loss": 113.7663, + "step": 1120 + }, + { + "epoch": 0.18247512161643892, + "grad_norm": 2.2793102264404297, + "learning_rate": 0.0001, + "loss": 115.219, + "step": 1130 + }, + { + "epoch": 0.18408994570154014, + "grad_norm": 2.132046699523926, + "learning_rate": 0.0001, + "loss": 112.8482, + "step": 1140 + }, + { + "epoch": 0.18570476978664135, + "grad_norm": 1.3153988122940063, + "learning_rate": 0.0001, + "loss": 111.4546, + "step": 1150 + }, + { + "epoch": 0.1873195938717426, + "grad_norm": 1.5246292352676392, + "learning_rate": 0.0001, + "loss": 112.4348, + "step": 1160 + }, + { + "epoch": 0.18893441795684382, + "grad_norm": 1.365096092224121, + "learning_rate": 0.0001, + "loss": 109.755, + "step": 1170 + }, + { + "epoch": 0.19054924204194507, + "grad_norm": 1.2777953147888184, + "learning_rate": 0.0001, + "loss": 111.0318, + "step": 1180 + }, + { + "epoch": 0.19216406612704628, + "grad_norm": 1.8672829866409302, + "learning_rate": 0.0001, + "loss": 109.1648, + "step": 1190 + }, + { + "epoch": 0.1937788902121475, + "grad_norm": 1.709987759590149, + "learning_rate": 0.0001, + "loss": 108.1616, + "step": 1200 + }, + { + "epoch": 0.19539371429724875, + "grad_norm": 1.176277995109558, + "learning_rate": 0.0001, + "loss": 109.3591, + "step": 1210 + }, + { + "epoch": 0.19700853838234997, + "grad_norm": 1.4181180000305176, + "learning_rate": 0.0001, + "loss": 110.0216, + "step": 1220 + }, + { + "epoch": 0.19862336246745121, + "grad_norm": 1.3246116638183594, + "learning_rate": 0.0001, + "loss": 107.9769, + "step": 1230 + }, + { + "epoch": 0.20023818655255243, + "grad_norm": 1.2501498460769653, + "learning_rate": 0.0001, + "loss": 108.2601, + "step": 1240 + }, + { + "epoch": 0.20185301063765365, + "grad_norm": 1.2277404069900513, + "learning_rate": 0.0001, + "loss": 107.1258, + "step": 1250 + }, + { + "epoch": 0.2034678347227549, + "grad_norm": 1.8064894676208496, + "learning_rate": 0.0001, + "loss": 105.6278, + "step": 1260 + }, + { + "epoch": 0.20508265880785612, + "grad_norm": 1.708897352218628, + "learning_rate": 0.0001, + "loss": 106.9269, + "step": 1270 + }, + { + "epoch": 0.20669748289295733, + "grad_norm": 1.4507478475570679, + "learning_rate": 0.0001, + "loss": 106.9236, + "step": 1280 + }, + { + "epoch": 0.20831230697805858, + "grad_norm": 1.4904310703277588, + "learning_rate": 0.0001, + "loss": 106.0653, + "step": 1290 + }, + { + "epoch": 0.2099271310631598, + "grad_norm": 1.2637165784835815, + "learning_rate": 0.0001, + "loss": 104.8665, + "step": 1300 + }, + { + "epoch": 0.21154195514826105, + "grad_norm": 1.4085133075714111, + "learning_rate": 0.0001, + "loss": 104.8891, + "step": 1310 + }, + { + "epoch": 0.21315677923336226, + "grad_norm": 1.5710629224777222, + "learning_rate": 0.0001, + "loss": 104.3009, + "step": 1320 + }, + { + "epoch": 0.21477160331846348, + "grad_norm": 1.3634663820266724, + "learning_rate": 0.0001, + "loss": 104.4545, + "step": 1330 + }, + { + "epoch": 0.21638642740356473, + "grad_norm": 1.5565859079360962, + "learning_rate": 0.0001, + "loss": 103.6205, + "step": 1340 + }, + { + "epoch": 0.21800125148866595, + "grad_norm": 1.4491888284683228, + "learning_rate": 0.0001, + "loss": 104.0892, + "step": 1350 + }, + { + "epoch": 0.2196160755737672, + "grad_norm": 1.7609063386917114, + "learning_rate": 0.0001, + "loss": 104.5177, + "step": 1360 + }, + { + "epoch": 0.2212308996588684, + "grad_norm": 1.6723753213882446, + "learning_rate": 0.0001, + "loss": 104.5552, + "step": 1370 + }, + { + "epoch": 0.22284572374396963, + "grad_norm": 1.3781665563583374, + "learning_rate": 0.0001, + "loss": 102.1022, + "step": 1380 + }, + { + "epoch": 0.22446054782907088, + "grad_norm": 1.5397366285324097, + "learning_rate": 0.0001, + "loss": 102.5882, + "step": 1390 + }, + { + "epoch": 0.2260753719141721, + "grad_norm": 1.4567879438400269, + "learning_rate": 0.0001, + "loss": 101.3781, + "step": 1400 + }, + { + "epoch": 0.22769019599927334, + "grad_norm": 1.2882301807403564, + "learning_rate": 0.0001, + "loss": 100.9378, + "step": 1410 + }, + { + "epoch": 0.22930502008437456, + "grad_norm": 1.4510424137115479, + "learning_rate": 0.0001, + "loss": 101.5383, + "step": 1420 + }, + { + "epoch": 0.23091984416947578, + "grad_norm": 2.8331449031829834, + "learning_rate": 0.0001, + "loss": 100.2773, + "step": 1430 + }, + { + "epoch": 0.23253466825457703, + "grad_norm": 1.363616943359375, + "learning_rate": 0.0001, + "loss": 101.646, + "step": 1440 + }, + { + "epoch": 0.23414949233967824, + "grad_norm": 1.2184548377990723, + "learning_rate": 0.0001, + "loss": 101.1133, + "step": 1450 + }, + { + "epoch": 0.23576431642477946, + "grad_norm": 2.235656976699829, + "learning_rate": 0.0001, + "loss": 100.2044, + "step": 1460 + }, + { + "epoch": 0.2373791405098807, + "grad_norm": 1.2334418296813965, + "learning_rate": 0.0001, + "loss": 99.4031, + "step": 1470 + }, + { + "epoch": 0.23899396459498193, + "grad_norm": 1.2394684553146362, + "learning_rate": 0.0001, + "loss": 101.1247, + "step": 1480 + }, + { + "epoch": 0.24060878868008317, + "grad_norm": 1.5218186378479004, + "learning_rate": 0.0001, + "loss": 100.4413, + "step": 1490 + }, + { + "epoch": 0.2422236127651844, + "grad_norm": 1.2112364768981934, + "learning_rate": 0.0001, + "loss": 99.8722, + "step": 1500 + }, + { + "epoch": 0.2438384368502856, + "grad_norm": 1.2277610301971436, + "learning_rate": 0.0001, + "loss": 99.0276, + "step": 1510 + }, + { + "epoch": 0.24545326093538686, + "grad_norm": 1.1005053520202637, + "learning_rate": 0.0001, + "loss": 98.6764, + "step": 1520 + }, + { + "epoch": 0.24706808502048808, + "grad_norm": 1.0754456520080566, + "learning_rate": 0.0001, + "loss": 97.8968, + "step": 1530 + }, + { + "epoch": 0.24868290910558932, + "grad_norm": 1.0695950984954834, + "learning_rate": 0.0001, + "loss": 98.4032, + "step": 1540 + }, + { + "epoch": 0.25029773319069054, + "grad_norm": 1.5098627805709839, + "learning_rate": 0.0001, + "loss": 97.9391, + "step": 1550 + }, + { + "epoch": 0.25191255727579176, + "grad_norm": 1.006681203842163, + "learning_rate": 0.0001, + "loss": 96.5525, + "step": 1560 + }, + { + "epoch": 0.253527381360893, + "grad_norm": 1.3987332582473755, + "learning_rate": 0.0001, + "loss": 98.383, + "step": 1570 + }, + { + "epoch": 0.25514220544599425, + "grad_norm": 1.290887713432312, + "learning_rate": 0.0001, + "loss": 99.0901, + "step": 1580 + }, + { + "epoch": 0.25675702953109547, + "grad_norm": 0.991177499294281, + "learning_rate": 0.0001, + "loss": 97.7093, + "step": 1590 + }, + { + "epoch": 0.2583718536161967, + "grad_norm": 1.147647738456726, + "learning_rate": 0.0001, + "loss": 96.9181, + "step": 1600 + }, + { + "epoch": 0.2599866777012979, + "grad_norm": 1.1756514310836792, + "learning_rate": 0.0001, + "loss": 98.3935, + "step": 1610 + }, + { + "epoch": 0.2616015017863991, + "grad_norm": 1.1708602905273438, + "learning_rate": 0.0001, + "loss": 96.715, + "step": 1620 + }, + { + "epoch": 0.2632163258715004, + "grad_norm": 1.4577715396881104, + "learning_rate": 0.0001, + "loss": 97.2971, + "step": 1630 + }, + { + "epoch": 0.2648311499566016, + "grad_norm": 1.1172800064086914, + "learning_rate": 0.0001, + "loss": 97.6494, + "step": 1640 + }, + { + "epoch": 0.26644597404170284, + "grad_norm": 1.1491258144378662, + "learning_rate": 0.0001, + "loss": 97.5503, + "step": 1650 + }, + { + "epoch": 0.26806079812680406, + "grad_norm": 1.2188555002212524, + "learning_rate": 0.0001, + "loss": 95.3423, + "step": 1660 + }, + { + "epoch": 0.2696756222119053, + "grad_norm": 1.8595614433288574, + "learning_rate": 0.0001, + "loss": 96.0112, + "step": 1670 + }, + { + "epoch": 0.2712904462970065, + "grad_norm": 1.1887950897216797, + "learning_rate": 0.0001, + "loss": 95.4939, + "step": 1680 + }, + { + "epoch": 0.27290527038210777, + "grad_norm": 1.1777839660644531, + "learning_rate": 0.0001, + "loss": 95.9619, + "step": 1690 + }, + { + "epoch": 0.274520094467209, + "grad_norm": 1.1241233348846436, + "learning_rate": 0.0001, + "loss": 96.2834, + "step": 1700 + }, + { + "epoch": 0.2761349185523102, + "grad_norm": 1.0900121927261353, + "learning_rate": 0.0001, + "loss": 94.7518, + "step": 1710 + }, + { + "epoch": 0.2777497426374114, + "grad_norm": 1.0615205764770508, + "learning_rate": 0.0001, + "loss": 95.8032, + "step": 1720 + }, + { + "epoch": 0.27936456672251264, + "grad_norm": 1.2626168727874756, + "learning_rate": 0.0001, + "loss": 93.9454, + "step": 1730 + }, + { + "epoch": 0.2809793908076139, + "grad_norm": 0.8869941830635071, + "learning_rate": 0.0001, + "loss": 95.4085, + "step": 1740 + }, + { + "epoch": 0.28259421489271513, + "grad_norm": 1.0350773334503174, + "learning_rate": 0.0001, + "loss": 95.7954, + "step": 1750 + }, + { + "epoch": 0.28420903897781635, + "grad_norm": 1.364768147468567, + "learning_rate": 0.0001, + "loss": 93.6447, + "step": 1760 + }, + { + "epoch": 0.28582386306291757, + "grad_norm": 1.0115883350372314, + "learning_rate": 0.0001, + "loss": 93.7905, + "step": 1770 + }, + { + "epoch": 0.2874386871480188, + "grad_norm": 0.9393435120582581, + "learning_rate": 0.0001, + "loss": 94.3108, + "step": 1780 + }, + { + "epoch": 0.28905351123312006, + "grad_norm": 1.0769627094268799, + "learning_rate": 0.0001, + "loss": 92.566, + "step": 1790 + }, + { + "epoch": 0.2906683353182213, + "grad_norm": 2.526201009750366, + "learning_rate": 0.0001, + "loss": 92.8161, + "step": 1800 + }, + { + "epoch": 0.2922831594033225, + "grad_norm": 1.3066468238830566, + "learning_rate": 0.0001, + "loss": 92.0459, + "step": 1810 + }, + { + "epoch": 0.2938979834884237, + "grad_norm": 1.121713638305664, + "learning_rate": 0.0001, + "loss": 93.5032, + "step": 1820 + }, + { + "epoch": 0.29551280757352494, + "grad_norm": 1.3744316101074219, + "learning_rate": 0.0001, + "loss": 94.2623, + "step": 1830 + }, + { + "epoch": 0.2971276316586262, + "grad_norm": 1.1561540365219116, + "learning_rate": 0.0001, + "loss": 94.1611, + "step": 1840 + }, + { + "epoch": 0.29874245574372743, + "grad_norm": 1.3947464227676392, + "learning_rate": 0.0001, + "loss": 93.5962, + "step": 1850 + }, + { + "epoch": 0.30035727982882865, + "grad_norm": 1.0011566877365112, + "learning_rate": 0.0001, + "loss": 92.0199, + "step": 1860 + }, + { + "epoch": 0.30197210391392987, + "grad_norm": 0.9978747963905334, + "learning_rate": 0.0001, + "loss": 91.0945, + "step": 1870 + }, + { + "epoch": 0.3035869279990311, + "grad_norm": 0.92522132396698, + "learning_rate": 0.0001, + "loss": 93.274, + "step": 1880 + }, + { + "epoch": 0.30520175208413236, + "grad_norm": 1.0089120864868164, + "learning_rate": 0.0001, + "loss": 91.9308, + "step": 1890 + }, + { + "epoch": 0.3068165761692336, + "grad_norm": 1.0535134077072144, + "learning_rate": 0.0001, + "loss": 92.563, + "step": 1900 + }, + { + "epoch": 0.3084314002543348, + "grad_norm": 0.9782275557518005, + "learning_rate": 0.0001, + "loss": 90.3486, + "step": 1910 + }, + { + "epoch": 0.310046224339436, + "grad_norm": 1.1433162689208984, + "learning_rate": 0.0001, + "loss": 90.4977, + "step": 1920 + }, + { + "epoch": 0.31166104842453723, + "grad_norm": 0.98392653465271, + "learning_rate": 0.0001, + "loss": 92.6529, + "step": 1930 + }, + { + "epoch": 0.3132758725096385, + "grad_norm": 0.8538148403167725, + "learning_rate": 0.0001, + "loss": 90.6859, + "step": 1940 + }, + { + "epoch": 0.3148906965947397, + "grad_norm": 1.019405484199524, + "learning_rate": 0.0001, + "loss": 91.0028, + "step": 1950 + }, + { + "epoch": 0.31650552067984095, + "grad_norm": 0.9828067421913147, + "learning_rate": 0.0001, + "loss": 89.9826, + "step": 1960 + }, + { + "epoch": 0.31812034476494216, + "grad_norm": 0.9725450277328491, + "learning_rate": 0.0001, + "loss": 91.7342, + "step": 1970 + }, + { + "epoch": 0.3197351688500434, + "grad_norm": 1.0139408111572266, + "learning_rate": 0.0001, + "loss": 90.5362, + "step": 1980 + }, + { + "epoch": 0.3213499929351446, + "grad_norm": 1.2493853569030762, + "learning_rate": 0.0001, + "loss": 92.7222, + "step": 1990 + }, + { + "epoch": 0.3229648170202459, + "grad_norm": 1.0152915716171265, + "learning_rate": 0.0001, + "loss": 92.2271, + "step": 2000 + }, + { + "epoch": 0.3245796411053471, + "grad_norm": 0.9745819568634033, + "learning_rate": 0.0001, + "loss": 88.7034, + "step": 2010 + }, + { + "epoch": 0.3261944651904483, + "grad_norm": 1.009624719619751, + "learning_rate": 0.0001, + "loss": 90.7428, + "step": 2020 + }, + { + "epoch": 0.32780928927554953, + "grad_norm": 0.910858690738678, + "learning_rate": 0.0001, + "loss": 89.8199, + "step": 2030 + }, + { + "epoch": 0.32942411336065075, + "grad_norm": 1.1745878458023071, + "learning_rate": 0.0001, + "loss": 90.8267, + "step": 2040 + }, + { + "epoch": 0.331038937445752, + "grad_norm": 0.9409273266792297, + "learning_rate": 0.0001, + "loss": 90.2937, + "step": 2050 + }, + { + "epoch": 0.33265376153085324, + "grad_norm": 1.4152942895889282, + "learning_rate": 0.0001, + "loss": 90.3965, + "step": 2060 + }, + { + "epoch": 0.33426858561595446, + "grad_norm": 0.9529693722724915, + "learning_rate": 0.0001, + "loss": 90.4739, + "step": 2070 + }, + { + "epoch": 0.3358834097010557, + "grad_norm": 0.8531096577644348, + "learning_rate": 0.0001, + "loss": 90.9301, + "step": 2080 + }, + { + "epoch": 0.3374982337861569, + "grad_norm": 0.9314167499542236, + "learning_rate": 0.0001, + "loss": 90.4235, + "step": 2090 + }, + { + "epoch": 0.33911305787125817, + "grad_norm": 0.9253250360488892, + "learning_rate": 0.0001, + "loss": 88.1893, + "step": 2100 + }, + { + "epoch": 0.3407278819563594, + "grad_norm": 1.773053526878357, + "learning_rate": 0.0001, + "loss": 88.3417, + "step": 2110 + }, + { + "epoch": 0.3423427060414606, + "grad_norm": 0.9432951211929321, + "learning_rate": 0.0001, + "loss": 89.7141, + "step": 2120 + }, + { + "epoch": 0.34395753012656183, + "grad_norm": 3.8847527503967285, + "learning_rate": 0.0001, + "loss": 89.1009, + "step": 2130 + }, + { + "epoch": 0.34557235421166305, + "grad_norm": 1.0309821367263794, + "learning_rate": 0.0001, + "loss": 89.2965, + "step": 2140 + }, + { + "epoch": 0.3471871782967643, + "grad_norm": 0.9590495824813843, + "learning_rate": 0.0001, + "loss": 89.1734, + "step": 2150 + }, + { + "epoch": 0.34880200238186554, + "grad_norm": 0.9943560361862183, + "learning_rate": 0.0001, + "loss": 85.425, + "step": 2160 + }, + { + "epoch": 0.35041682646696676, + "grad_norm": 1.3322339057922363, + "learning_rate": 0.0001, + "loss": 89.7791, + "step": 2170 + }, + { + "epoch": 0.352031650552068, + "grad_norm": 1.0212253332138062, + "learning_rate": 0.0001, + "loss": 89.3771, + "step": 2180 + }, + { + "epoch": 0.3536464746371692, + "grad_norm": 1.0446089506149292, + "learning_rate": 0.0001, + "loss": 88.6894, + "step": 2190 + }, + { + "epoch": 0.35526129872227047, + "grad_norm": 1.05767822265625, + "learning_rate": 0.0001, + "loss": 89.5341, + "step": 2200 + }, + { + "epoch": 0.3568761228073717, + "grad_norm": 1.0373681783676147, + "learning_rate": 0.0001, + "loss": 89.7173, + "step": 2210 + }, + { + "epoch": 0.3584909468924729, + "grad_norm": 0.9098039269447327, + "learning_rate": 0.0001, + "loss": 89.2116, + "step": 2220 + }, + { + "epoch": 0.3601057709775741, + "grad_norm": 1.028130292892456, + "learning_rate": 0.0001, + "loss": 88.6627, + "step": 2230 + }, + { + "epoch": 0.36172059506267534, + "grad_norm": 1.0295171737670898, + "learning_rate": 0.0001, + "loss": 89.608, + "step": 2240 + }, + { + "epoch": 0.36333541914777656, + "grad_norm": 0.9720722436904907, + "learning_rate": 0.0001, + "loss": 88.3624, + "step": 2250 + }, + { + "epoch": 0.36495024323287784, + "grad_norm": 0.839654803276062, + "learning_rate": 0.0001, + "loss": 87.2983, + "step": 2260 + }, + { + "epoch": 0.36656506731797905, + "grad_norm": 1.1600219011306763, + "learning_rate": 0.0001, + "loss": 86.6004, + "step": 2270 + }, + { + "epoch": 0.3681798914030803, + "grad_norm": 1.0513924360275269, + "learning_rate": 0.0001, + "loss": 86.6062, + "step": 2280 + }, + { + "epoch": 0.3697947154881815, + "grad_norm": 0.9728474617004395, + "learning_rate": 0.0001, + "loss": 86.1856, + "step": 2290 + }, + { + "epoch": 0.3714095395732827, + "grad_norm": 1.0196386575698853, + "learning_rate": 0.0001, + "loss": 87.6088, + "step": 2300 + }, + { + "epoch": 0.373024363658384, + "grad_norm": 1.0439444780349731, + "learning_rate": 0.0001, + "loss": 85.2374, + "step": 2310 + }, + { + "epoch": 0.3746391877434852, + "grad_norm": 1.0020275115966797, + "learning_rate": 0.0001, + "loss": 86.0596, + "step": 2320 + }, + { + "epoch": 0.3762540118285864, + "grad_norm": 1.0107691287994385, + "learning_rate": 0.0001, + "loss": 86.9513, + "step": 2330 + }, + { + "epoch": 0.37786883591368764, + "grad_norm": 1.680018663406372, + "learning_rate": 0.0001, + "loss": 87.8885, + "step": 2340 + }, + { + "epoch": 0.37948365999878886, + "grad_norm": 0.9181709885597229, + "learning_rate": 0.0001, + "loss": 86.7662, + "step": 2350 + }, + { + "epoch": 0.38109848408389013, + "grad_norm": 0.9598278403282166, + "learning_rate": 0.0001, + "loss": 88.3775, + "step": 2360 + }, + { + "epoch": 0.38271330816899135, + "grad_norm": 1.0707905292510986, + "learning_rate": 0.0001, + "loss": 86.0997, + "step": 2370 + }, + { + "epoch": 0.38432813225409257, + "grad_norm": 1.125797152519226, + "learning_rate": 0.0001, + "loss": 85.8117, + "step": 2380 + }, + { + "epoch": 0.3859429563391938, + "grad_norm": 1.0251439809799194, + "learning_rate": 0.0001, + "loss": 86.0806, + "step": 2390 + }, + { + "epoch": 0.387557780424295, + "grad_norm": 1.0193125009536743, + "learning_rate": 0.0001, + "loss": 87.264, + "step": 2400 + }, + { + "epoch": 0.3891726045093963, + "grad_norm": 0.9309751391410828, + "learning_rate": 0.0001, + "loss": 85.9187, + "step": 2410 + }, + { + "epoch": 0.3907874285944975, + "grad_norm": 1.1025906801223755, + "learning_rate": 0.0001, + "loss": 86.4365, + "step": 2420 + }, + { + "epoch": 0.3924022526795987, + "grad_norm": 0.9880627393722534, + "learning_rate": 0.0001, + "loss": 86.2303, + "step": 2430 + }, + { + "epoch": 0.39401707676469994, + "grad_norm": 0.9520611763000488, + "learning_rate": 0.0001, + "loss": 84.9035, + "step": 2440 + }, + { + "epoch": 0.39563190084980115, + "grad_norm": 0.8641896843910217, + "learning_rate": 0.0001, + "loss": 85.4754, + "step": 2450 + }, + { + "epoch": 0.39724672493490243, + "grad_norm": 1.1162904500961304, + "learning_rate": 0.0001, + "loss": 85.6767, + "step": 2460 + }, + { + "epoch": 0.39886154902000365, + "grad_norm": 0.9073636531829834, + "learning_rate": 0.0001, + "loss": 85.4198, + "step": 2470 + }, + { + "epoch": 0.40047637310510487, + "grad_norm": 1.1135166883468628, + "learning_rate": 0.0001, + "loss": 86.0884, + "step": 2480 + }, + { + "epoch": 0.4020911971902061, + "grad_norm": 0.9273980259895325, + "learning_rate": 0.0001, + "loss": 85.5473, + "step": 2490 + }, + { + "epoch": 0.4037060212753073, + "grad_norm": 0.8653831481933594, + "learning_rate": 0.0001, + "loss": 87.279, + "step": 2500 + }, + { + "epoch": 0.4053208453604086, + "grad_norm": 0.8535159826278687, + "learning_rate": 0.0001, + "loss": 86.7072, + "step": 2510 + }, + { + "epoch": 0.4069356694455098, + "grad_norm": 0.8864684700965881, + "learning_rate": 0.0001, + "loss": 86.441, + "step": 2520 + }, + { + "epoch": 0.408550493530611, + "grad_norm": 0.8042080998420715, + "learning_rate": 0.0001, + "loss": 83.1706, + "step": 2530 + }, + { + "epoch": 0.41016531761571223, + "grad_norm": 1.0486373901367188, + "learning_rate": 0.0001, + "loss": 83.54, + "step": 2540 + }, + { + "epoch": 0.41178014170081345, + "grad_norm": 0.8087959289550781, + "learning_rate": 0.0001, + "loss": 85.7615, + "step": 2550 + }, + { + "epoch": 0.41339496578591467, + "grad_norm": 1.5032018423080444, + "learning_rate": 0.0001, + "loss": 85.7368, + "step": 2560 + }, + { + "epoch": 0.41500978987101594, + "grad_norm": 1.0786182880401611, + "learning_rate": 0.0001, + "loss": 86.3484, + "step": 2570 + }, + { + "epoch": 0.41662461395611716, + "grad_norm": 0.8744294047355652, + "learning_rate": 0.0001, + "loss": 83.5945, + "step": 2580 + }, + { + "epoch": 0.4182394380412184, + "grad_norm": 0.7840721607208252, + "learning_rate": 0.0001, + "loss": 86.3831, + "step": 2590 + }, + { + "epoch": 0.4198542621263196, + "grad_norm": 0.7977836728096008, + "learning_rate": 0.0001, + "loss": 83.6314, + "step": 2600 + }, + { + "epoch": 0.4214690862114208, + "grad_norm": 0.892484188079834, + "learning_rate": 0.0001, + "loss": 86.4223, + "step": 2610 + }, + { + "epoch": 0.4230839102965221, + "grad_norm": 0.912995457649231, + "learning_rate": 0.0001, + "loss": 83.8971, + "step": 2620 + }, + { + "epoch": 0.4246987343816233, + "grad_norm": 0.9566861987113953, + "learning_rate": 0.0001, + "loss": 85.291, + "step": 2630 + }, + { + "epoch": 0.42631355846672453, + "grad_norm": 0.8324154615402222, + "learning_rate": 0.0001, + "loss": 84.6233, + "step": 2640 + }, + { + "epoch": 0.42792838255182575, + "grad_norm": 0.9490360617637634, + "learning_rate": 0.0001, + "loss": 81.799, + "step": 2650 + }, + { + "epoch": 0.42954320663692697, + "grad_norm": 0.9744566679000854, + "learning_rate": 0.0001, + "loss": 85.4837, + "step": 2660 + }, + { + "epoch": 0.43115803072202824, + "grad_norm": 0.941614031791687, + "learning_rate": 0.0001, + "loss": 82.838, + "step": 2670 + }, + { + "epoch": 0.43277285480712946, + "grad_norm": 1.338364601135254, + "learning_rate": 0.0001, + "loss": 84.4224, + "step": 2680 + }, + { + "epoch": 0.4343876788922307, + "grad_norm": 0.8643816113471985, + "learning_rate": 0.0001, + "loss": 84.3959, + "step": 2690 + }, + { + "epoch": 0.4360025029773319, + "grad_norm": 1.0231473445892334, + "learning_rate": 0.0001, + "loss": 82.3144, + "step": 2700 + }, + { + "epoch": 0.4376173270624331, + "grad_norm": 0.9089650511741638, + "learning_rate": 0.0001, + "loss": 83.0326, + "step": 2710 + }, + { + "epoch": 0.4392321511475344, + "grad_norm": 0.893628716468811, + "learning_rate": 0.0001, + "loss": 83.0981, + "step": 2720 + }, + { + "epoch": 0.4408469752326356, + "grad_norm": 0.869076132774353, + "learning_rate": 0.0001, + "loss": 84.2571, + "step": 2730 + }, + { + "epoch": 0.4424617993177368, + "grad_norm": 0.7964235544204712, + "learning_rate": 0.0001, + "loss": 83.3329, + "step": 2740 + }, + { + "epoch": 0.44407662340283804, + "grad_norm": 3.1645138263702393, + "learning_rate": 0.0001, + "loss": 83.7232, + "step": 2750 + }, + { + "epoch": 0.44569144748793926, + "grad_norm": 0.934644877910614, + "learning_rate": 0.0001, + "loss": 85.0688, + "step": 2760 + }, + { + "epoch": 0.44730627157304054, + "grad_norm": 0.8120711445808411, + "learning_rate": 0.0001, + "loss": 82.7706, + "step": 2770 + }, + { + "epoch": 0.44892109565814176, + "grad_norm": 0.8161435127258301, + "learning_rate": 0.0001, + "loss": 84.4826, + "step": 2780 + }, + { + "epoch": 0.450535919743243, + "grad_norm": 0.8659493327140808, + "learning_rate": 0.0001, + "loss": 82.6222, + "step": 2790 + }, + { + "epoch": 0.4521507438283442, + "grad_norm": 0.9306702613830566, + "learning_rate": 0.0001, + "loss": 83.5897, + "step": 2800 + }, + { + "epoch": 0.4537655679134454, + "grad_norm": 2.2544431686401367, + "learning_rate": 0.0001, + "loss": 82.9603, + "step": 2810 + }, + { + "epoch": 0.4553803919985467, + "grad_norm": 0.8868395686149597, + "learning_rate": 0.0001, + "loss": 82.6674, + "step": 2820 + }, + { + "epoch": 0.4569952160836479, + "grad_norm": 0.9336455464363098, + "learning_rate": 0.0001, + "loss": 84.6974, + "step": 2830 + }, + { + "epoch": 0.4586100401687491, + "grad_norm": 0.8814165592193604, + "learning_rate": 0.0001, + "loss": 83.9329, + "step": 2840 + }, + { + "epoch": 0.46022486425385034, + "grad_norm": 0.8125660419464111, + "learning_rate": 0.0001, + "loss": 84.6168, + "step": 2850 + }, + { + "epoch": 0.46183968833895156, + "grad_norm": 0.9447190165519714, + "learning_rate": 0.0001, + "loss": 82.4098, + "step": 2860 + }, + { + "epoch": 0.4634545124240528, + "grad_norm": 0.8080969452857971, + "learning_rate": 0.0001, + "loss": 83.4076, + "step": 2870 + }, + { + "epoch": 0.46506933650915405, + "grad_norm": 1.0586694478988647, + "learning_rate": 0.0001, + "loss": 83.3295, + "step": 2880 + }, + { + "epoch": 0.46668416059425527, + "grad_norm": 0.8207664489746094, + "learning_rate": 0.0001, + "loss": 84.5056, + "step": 2890 + }, + { + "epoch": 0.4682989846793565, + "grad_norm": 1.0647571086883545, + "learning_rate": 0.0001, + "loss": 83.0766, + "step": 2900 + }, + { + "epoch": 0.4699138087644577, + "grad_norm": 0.9416915774345398, + "learning_rate": 0.0001, + "loss": 83.8166, + "step": 2910 + }, + { + "epoch": 0.4715286328495589, + "grad_norm": 0.7479362487792969, + "learning_rate": 0.0001, + "loss": 82.7507, + "step": 2920 + }, + { + "epoch": 0.4731434569346602, + "grad_norm": 1.0775514841079712, + "learning_rate": 0.0001, + "loss": 84.4641, + "step": 2930 + }, + { + "epoch": 0.4747582810197614, + "grad_norm": 0.8736841082572937, + "learning_rate": 0.0001, + "loss": 82.0451, + "step": 2940 + }, + { + "epoch": 0.47637310510486264, + "grad_norm": 0.9500495791435242, + "learning_rate": 0.0001, + "loss": 82.6425, + "step": 2950 + }, + { + "epoch": 0.47798792918996386, + "grad_norm": 0.8551210165023804, + "learning_rate": 0.0001, + "loss": 81.1971, + "step": 2960 + }, + { + "epoch": 0.4796027532750651, + "grad_norm": 0.8149814605712891, + "learning_rate": 0.0001, + "loss": 82.7919, + "step": 2970 + }, + { + "epoch": 0.48121757736016635, + "grad_norm": 0.9012249112129211, + "learning_rate": 0.0001, + "loss": 81.975, + "step": 2980 + }, + { + "epoch": 0.48283240144526757, + "grad_norm": 1.3784620761871338, + "learning_rate": 0.0001, + "loss": 81.541, + "step": 2990 + }, + { + "epoch": 0.4844472255303688, + "grad_norm": 0.7921801209449768, + "learning_rate": 0.0001, + "loss": 83.6215, + "step": 3000 + }, + { + "epoch": 0.48606204961547, + "grad_norm": 0.9051580429077148, + "learning_rate": 0.0001, + "loss": 82.4048, + "step": 3010 + }, + { + "epoch": 0.4876768737005712, + "grad_norm": 1.0717192888259888, + "learning_rate": 0.0001, + "loss": 82.4079, + "step": 3020 + }, + { + "epoch": 0.4892916977856725, + "grad_norm": 0.8971781730651855, + "learning_rate": 0.0001, + "loss": 83.3668, + "step": 3030 + }, + { + "epoch": 0.4909065218707737, + "grad_norm": 0.9635320901870728, + "learning_rate": 0.0001, + "loss": 82.8244, + "step": 3040 + }, + { + "epoch": 0.49252134595587493, + "grad_norm": 1.1009949445724487, + "learning_rate": 0.0001, + "loss": 82.2174, + "step": 3050 + }, + { + "epoch": 0.49413617004097615, + "grad_norm": 0.9805473685264587, + "learning_rate": 0.0001, + "loss": 83.7897, + "step": 3060 + }, + { + "epoch": 0.49575099412607737, + "grad_norm": 1.2107361555099487, + "learning_rate": 0.0001, + "loss": 82.5108, + "step": 3070 + }, + { + "epoch": 0.49736581821117865, + "grad_norm": 0.9171829223632812, + "learning_rate": 0.0001, + "loss": 80.9126, + "step": 3080 + }, + { + "epoch": 0.49898064229627986, + "grad_norm": 0.8093295097351074, + "learning_rate": 0.0001, + "loss": 83.8147, + "step": 3090 + }, + { + "epoch": 0.5005954663813811, + "grad_norm": 1.2166653871536255, + "learning_rate": 0.0001, + "loss": 82.0288, + "step": 3100 + }, + { + "epoch": 0.5022102904664824, + "grad_norm": 1.0471502542495728, + "learning_rate": 0.0001, + "loss": 82.0706, + "step": 3110 + }, + { + "epoch": 0.5038251145515835, + "grad_norm": 1.007645845413208, + "learning_rate": 0.0001, + "loss": 81.7387, + "step": 3120 + }, + { + "epoch": 0.5054399386366848, + "grad_norm": 0.7252638339996338, + "learning_rate": 0.0001, + "loss": 82.9107, + "step": 3130 + }, + { + "epoch": 0.507054762721786, + "grad_norm": 0.8565683364868164, + "learning_rate": 0.0001, + "loss": 80.2935, + "step": 3140 + }, + { + "epoch": 0.5086695868068872, + "grad_norm": 0.9905404448509216, + "learning_rate": 0.0001, + "loss": 82.5201, + "step": 3150 + }, + { + "epoch": 0.5102844108919885, + "grad_norm": 0.887428343296051, + "learning_rate": 0.0001, + "loss": 80.8528, + "step": 3160 + }, + { + "epoch": 0.5118992349770897, + "grad_norm": 1.062942624092102, + "learning_rate": 0.0001, + "loss": 80.7035, + "step": 3170 + }, + { + "epoch": 0.5135140590621909, + "grad_norm": 1.203599214553833, + "learning_rate": 0.0001, + "loss": 81.2521, + "step": 3180 + }, + { + "epoch": 0.5151288831472921, + "grad_norm": 0.8452383875846863, + "learning_rate": 0.0001, + "loss": 81.6079, + "step": 3190 + }, + { + "epoch": 0.5167437072323934, + "grad_norm": 1.2530587911605835, + "learning_rate": 0.0001, + "loss": 82.2148, + "step": 3200 + }, + { + "epoch": 0.5183585313174947, + "grad_norm": 0.9351627826690674, + "learning_rate": 0.0001, + "loss": 82.7857, + "step": 3210 + }, + { + "epoch": 0.5199733554025958, + "grad_norm": 1.0128077268600464, + "learning_rate": 0.0001, + "loss": 81.0532, + "step": 3220 + }, + { + "epoch": 0.5215881794876971, + "grad_norm": 0.9316917061805725, + "learning_rate": 0.0001, + "loss": 80.7443, + "step": 3230 + }, + { + "epoch": 0.5232030035727983, + "grad_norm": 1.502705693244934, + "learning_rate": 0.0001, + "loss": 80.0608, + "step": 3240 + }, + { + "epoch": 0.5248178276578995, + "grad_norm": 0.8166837692260742, + "learning_rate": 0.0001, + "loss": 81.2013, + "step": 3250 + }, + { + "epoch": 0.5264326517430008, + "grad_norm": 0.8210042119026184, + "learning_rate": 0.0001, + "loss": 80.2491, + "step": 3260 + }, + { + "epoch": 0.528047475828102, + "grad_norm": 0.7854771018028259, + "learning_rate": 0.0001, + "loss": 81.1017, + "step": 3270 + }, + { + "epoch": 0.5296622999132032, + "grad_norm": 0.8530565500259399, + "learning_rate": 0.0001, + "loss": 80.8123, + "step": 3280 + }, + { + "epoch": 0.5312771239983044, + "grad_norm": 0.9446498155593872, + "learning_rate": 0.0001, + "loss": 80.0029, + "step": 3290 + }, + { + "epoch": 0.5328919480834057, + "grad_norm": 0.9646537899971008, + "learning_rate": 0.0001, + "loss": 81.5413, + "step": 3300 + }, + { + "epoch": 0.5345067721685068, + "grad_norm": 0.9262019395828247, + "learning_rate": 0.0001, + "loss": 80.9399, + "step": 3310 + }, + { + "epoch": 0.5361215962536081, + "grad_norm": 0.8515486121177673, + "learning_rate": 0.0001, + "loss": 80.9421, + "step": 3320 + }, + { + "epoch": 0.5377364203387094, + "grad_norm": 1.679321050643921, + "learning_rate": 0.0001, + "loss": 82.4851, + "step": 3330 + }, + { + "epoch": 0.5393512444238105, + "grad_norm": 0.8289812207221985, + "learning_rate": 0.0001, + "loss": 80.5664, + "step": 3340 + }, + { + "epoch": 0.5409660685089118, + "grad_norm": 0.7463789582252502, + "learning_rate": 0.0001, + "loss": 81.5193, + "step": 3350 + }, + { + "epoch": 0.542580892594013, + "grad_norm": 0.8332446217536926, + "learning_rate": 0.0001, + "loss": 80.8091, + "step": 3360 + }, + { + "epoch": 0.5441957166791143, + "grad_norm": 0.7612847089767456, + "learning_rate": 0.0001, + "loss": 79.8893, + "step": 3370 + }, + { + "epoch": 0.5458105407642155, + "grad_norm": 1.0557948350906372, + "learning_rate": 0.0001, + "loss": 81.4305, + "step": 3380 + }, + { + "epoch": 0.5474253648493167, + "grad_norm": 0.7975788116455078, + "learning_rate": 0.0001, + "loss": 80.0105, + "step": 3390 + }, + { + "epoch": 0.549040188934418, + "grad_norm": 0.9049227237701416, + "learning_rate": 0.0001, + "loss": 81.2689, + "step": 3400 + }, + { + "epoch": 0.5506550130195191, + "grad_norm": 1.1195895671844482, + "learning_rate": 0.0001, + "loss": 80.7163, + "step": 3410 + }, + { + "epoch": 0.5522698371046204, + "grad_norm": 0.7506540417671204, + "learning_rate": 0.0001, + "loss": 79.0855, + "step": 3420 + }, + { + "epoch": 0.5538846611897217, + "grad_norm": 0.73989337682724, + "learning_rate": 0.0001, + "loss": 81.1865, + "step": 3430 + }, + { + "epoch": 0.5554994852748228, + "grad_norm": 0.8418401479721069, + "learning_rate": 0.0001, + "loss": 79.5714, + "step": 3440 + }, + { + "epoch": 0.5571143093599241, + "grad_norm": 0.9807226061820984, + "learning_rate": 0.0001, + "loss": 78.5699, + "step": 3450 + }, + { + "epoch": 0.5587291334450253, + "grad_norm": 1.2890545129776, + "learning_rate": 0.0001, + "loss": 81.0128, + "step": 3460 + }, + { + "epoch": 0.5603439575301266, + "grad_norm": 0.8593361973762512, + "learning_rate": 0.0001, + "loss": 80.1638, + "step": 3470 + }, + { + "epoch": 0.5619587816152278, + "grad_norm": 0.7766354084014893, + "learning_rate": 0.0001, + "loss": 81.5597, + "step": 3480 + }, + { + "epoch": 0.563573605700329, + "grad_norm": 0.7785626649856567, + "learning_rate": 0.0001, + "loss": 79.3862, + "step": 3490 + }, + { + "epoch": 0.5651884297854303, + "grad_norm": 0.8429279327392578, + "learning_rate": 0.0001, + "loss": 76.6294, + "step": 3500 + }, + { + "epoch": 0.5668032538705314, + "grad_norm": 0.8141765594482422, + "learning_rate": 0.0001, + "loss": 80.2584, + "step": 3510 + }, + { + "epoch": 0.5684180779556327, + "grad_norm": 0.7065490484237671, + "learning_rate": 0.0001, + "loss": 79.3626, + "step": 3520 + }, + { + "epoch": 0.570032902040734, + "grad_norm": 0.7002031803131104, + "learning_rate": 0.0001, + "loss": 79.283, + "step": 3530 + }, + { + "epoch": 0.5716477261258351, + "grad_norm": 0.8715317845344543, + "learning_rate": 0.0001, + "loss": 80.3145, + "step": 3540 + }, + { + "epoch": 0.5732625502109364, + "grad_norm": 0.9584882855415344, + "learning_rate": 0.0001, + "loss": 78.7443, + "step": 3550 + }, + { + "epoch": 0.5748773742960376, + "grad_norm": 0.7908946871757507, + "learning_rate": 0.0001, + "loss": 79.1906, + "step": 3560 + }, + { + "epoch": 0.5764921983811389, + "grad_norm": 0.8416746258735657, + "learning_rate": 0.0001, + "loss": 77.8172, + "step": 3570 + }, + { + "epoch": 0.5781070224662401, + "grad_norm": 0.823667049407959, + "learning_rate": 0.0001, + "loss": 81.864, + "step": 3580 + }, + { + "epoch": 0.5797218465513413, + "grad_norm": 1.4282617568969727, + "learning_rate": 0.0001, + "loss": 79.9052, + "step": 3590 + }, + { + "epoch": 0.5813366706364426, + "grad_norm": 0.8747813701629639, + "learning_rate": 0.0001, + "loss": 78.8234, + "step": 3600 + }, + { + "epoch": 0.5829514947215437, + "grad_norm": 0.8415511846542358, + "learning_rate": 0.0001, + "loss": 78.8171, + "step": 3610 + }, + { + "epoch": 0.584566318806645, + "grad_norm": 0.9014492034912109, + "learning_rate": 0.0001, + "loss": 80.5011, + "step": 3620 + }, + { + "epoch": 0.5861811428917463, + "grad_norm": 0.7535204887390137, + "learning_rate": 0.0001, + "loss": 79.4895, + "step": 3630 + }, + { + "epoch": 0.5877959669768474, + "grad_norm": 0.8196508884429932, + "learning_rate": 0.0001, + "loss": 79.7101, + "step": 3640 + }, + { + "epoch": 0.5894107910619487, + "grad_norm": 1.5975492000579834, + "learning_rate": 0.0001, + "loss": 79.0687, + "step": 3650 + }, + { + "epoch": 0.5910256151470499, + "grad_norm": 0.9251692295074463, + "learning_rate": 0.0001, + "loss": 80.2377, + "step": 3660 + }, + { + "epoch": 0.5926404392321512, + "grad_norm": 0.7143557071685791, + "learning_rate": 0.0001, + "loss": 78.0671, + "step": 3670 + }, + { + "epoch": 0.5942552633172524, + "grad_norm": 0.7876290678977966, + "learning_rate": 0.0001, + "loss": 79.1455, + "step": 3680 + }, + { + "epoch": 0.5958700874023536, + "grad_norm": 1.5361895561218262, + "learning_rate": 0.0001, + "loss": 78.1253, + "step": 3690 + }, + { + "epoch": 0.5974849114874549, + "grad_norm": 0.7637040019035339, + "learning_rate": 0.0001, + "loss": 77.4658, + "step": 3700 + }, + { + "epoch": 0.599099735572556, + "grad_norm": 0.8166250586509705, + "learning_rate": 0.0001, + "loss": 79.5806, + "step": 3710 + }, + { + "epoch": 0.6007145596576573, + "grad_norm": 1.2767380475997925, + "learning_rate": 0.0001, + "loss": 79.1894, + "step": 3720 + }, + { + "epoch": 0.6023293837427586, + "grad_norm": 1.3048640489578247, + "learning_rate": 0.0001, + "loss": 78.0593, + "step": 3730 + }, + { + "epoch": 0.6039442078278597, + "grad_norm": 0.7945445775985718, + "learning_rate": 0.0001, + "loss": 79.7623, + "step": 3740 + }, + { + "epoch": 0.605559031912961, + "grad_norm": 0.7366836667060852, + "learning_rate": 0.0001, + "loss": 77.9571, + "step": 3750 + }, + { + "epoch": 0.6071738559980622, + "grad_norm": 0.9126115441322327, + "learning_rate": 0.0001, + "loss": 79.9502, + "step": 3760 + }, + { + "epoch": 0.6087886800831634, + "grad_norm": 0.800355851650238, + "learning_rate": 0.0001, + "loss": 78.7244, + "step": 3770 + }, + { + "epoch": 0.6104035041682647, + "grad_norm": 0.8290980458259583, + "learning_rate": 0.0001, + "loss": 78.7724, + "step": 3780 + }, + { + "epoch": 0.6120183282533659, + "grad_norm": 0.8512316942214966, + "learning_rate": 0.0001, + "loss": 78.6277, + "step": 3790 + }, + { + "epoch": 0.6136331523384672, + "grad_norm": 0.8909353613853455, + "learning_rate": 0.0001, + "loss": 78.5956, + "step": 3800 + }, + { + "epoch": 0.6152479764235683, + "grad_norm": 0.8328800797462463, + "learning_rate": 0.0001, + "loss": 78.1323, + "step": 3810 + }, + { + "epoch": 0.6168628005086696, + "grad_norm": 0.7950423359870911, + "learning_rate": 0.0001, + "loss": 78.7529, + "step": 3820 + }, + { + "epoch": 0.6184776245937709, + "grad_norm": 0.9988545179367065, + "learning_rate": 0.0001, + "loss": 80.7727, + "step": 3830 + }, + { + "epoch": 0.620092448678872, + "grad_norm": 0.7243085503578186, + "learning_rate": 0.0001, + "loss": 76.7616, + "step": 3840 + }, + { + "epoch": 0.6217072727639733, + "grad_norm": 0.7674855589866638, + "learning_rate": 0.0001, + "loss": 79.1938, + "step": 3850 + }, + { + "epoch": 0.6233220968490745, + "grad_norm": 0.8979483842849731, + "learning_rate": 0.0001, + "loss": 79.1522, + "step": 3860 + }, + { + "epoch": 0.6249369209341757, + "grad_norm": 0.7759745717048645, + "learning_rate": 0.0001, + "loss": 77.7001, + "step": 3870 + }, + { + "epoch": 0.626551745019277, + "grad_norm": 0.9940371513366699, + "learning_rate": 0.0001, + "loss": 79.8712, + "step": 3880 + }, + { + "epoch": 0.6281665691043782, + "grad_norm": 0.8103354573249817, + "learning_rate": 0.0001, + "loss": 77.5229, + "step": 3890 + }, + { + "epoch": 0.6297813931894795, + "grad_norm": 0.9188411831855774, + "learning_rate": 0.0001, + "loss": 78.2933, + "step": 3900 + }, + { + "epoch": 0.6313962172745806, + "grad_norm": 0.823017954826355, + "learning_rate": 0.0001, + "loss": 77.8692, + "step": 3910 + }, + { + "epoch": 0.6330110413596819, + "grad_norm": 0.8833358287811279, + "learning_rate": 0.0001, + "loss": 78.4465, + "step": 3920 + }, + { + "epoch": 0.634625865444783, + "grad_norm": 0.6880685687065125, + "learning_rate": 0.0001, + "loss": 78.8881, + "step": 3930 + }, + { + "epoch": 0.6362406895298843, + "grad_norm": 0.778455913066864, + "learning_rate": 0.0001, + "loss": 76.0277, + "step": 3940 + }, + { + "epoch": 0.6378555136149856, + "grad_norm": 0.8132772445678711, + "learning_rate": 0.0001, + "loss": 79.1284, + "step": 3950 + }, + { + "epoch": 0.6394703377000868, + "grad_norm": 0.7703788876533508, + "learning_rate": 0.0001, + "loss": 76.9013, + "step": 3960 + }, + { + "epoch": 0.641085161785188, + "grad_norm": 0.7876151204109192, + "learning_rate": 0.0001, + "loss": 77.1354, + "step": 3970 + }, + { + "epoch": 0.6426999858702892, + "grad_norm": 0.8027064800262451, + "learning_rate": 0.0001, + "loss": 77.4936, + "step": 3980 + }, + { + "epoch": 0.6443148099553905, + "grad_norm": 0.8261818885803223, + "learning_rate": 0.0001, + "loss": 76.7634, + "step": 3990 + }, + { + "epoch": 0.6459296340404918, + "grad_norm": 0.7024852633476257, + "learning_rate": 0.0001, + "loss": 79.951, + "step": 4000 + }, + { + "epoch": 0.6475444581255929, + "grad_norm": 1.015307068824768, + "learning_rate": 0.0001, + "loss": 76.0354, + "step": 4010 + }, + { + "epoch": 0.6491592822106942, + "grad_norm": 0.715346097946167, + "learning_rate": 0.0001, + "loss": 78.9021, + "step": 4020 + }, + { + "epoch": 0.6507741062957954, + "grad_norm": 0.8994929790496826, + "learning_rate": 0.0001, + "loss": 77.0587, + "step": 4030 + }, + { + "epoch": 0.6523889303808966, + "grad_norm": 0.7683436274528503, + "learning_rate": 0.0001, + "loss": 78.6943, + "step": 4040 + }, + { + "epoch": 0.6540037544659979, + "grad_norm": 0.7273952960968018, + "learning_rate": 0.0001, + "loss": 77.8396, + "step": 4050 + }, + { + "epoch": 0.6556185785510991, + "grad_norm": 1.0676878690719604, + "learning_rate": 0.0001, + "loss": 79.3449, + "step": 4060 + }, + { + "epoch": 0.6572334026362003, + "grad_norm": 0.6916168928146362, + "learning_rate": 0.0001, + "loss": 76.5462, + "step": 4070 + }, + { + "epoch": 0.6588482267213015, + "grad_norm": 0.6466719508171082, + "learning_rate": 0.0001, + "loss": 77.9354, + "step": 4080 + }, + { + "epoch": 0.6604630508064028, + "grad_norm": 0.7406665086746216, + "learning_rate": 0.0001, + "loss": 76.0135, + "step": 4090 + }, + { + "epoch": 0.662077874891504, + "grad_norm": 0.7104353904724121, + "learning_rate": 0.0001, + "loss": 78.3727, + "step": 4100 + }, + { + "epoch": 0.6636926989766052, + "grad_norm": 0.8328452110290527, + "learning_rate": 0.0001, + "loss": 77.5354, + "step": 4110 + }, + { + "epoch": 0.6653075230617065, + "grad_norm": 0.7012260556221008, + "learning_rate": 0.0001, + "loss": 78.7406, + "step": 4120 + }, + { + "epoch": 0.6669223471468076, + "grad_norm": 1.0436632633209229, + "learning_rate": 0.0001, + "loss": 78.0063, + "step": 4130 + }, + { + "epoch": 0.6685371712319089, + "grad_norm": 0.8471246361732483, + "learning_rate": 0.0001, + "loss": 77.8871, + "step": 4140 + }, + { + "epoch": 0.6701519953170102, + "grad_norm": 0.7378994822502136, + "learning_rate": 0.0001, + "loss": 76.9109, + "step": 4150 + }, + { + "epoch": 0.6717668194021114, + "grad_norm": 0.7778745889663696, + "learning_rate": 0.0001, + "loss": 78.3374, + "step": 4160 + }, + { + "epoch": 0.6733816434872126, + "grad_norm": 0.7841216325759888, + "learning_rate": 0.0001, + "loss": 78.1657, + "step": 4170 + }, + { + "epoch": 0.6749964675723138, + "grad_norm": 1.0027995109558105, + "learning_rate": 0.0001, + "loss": 76.6232, + "step": 4180 + }, + { + "epoch": 0.6766112916574151, + "grad_norm": 1.0034840106964111, + "learning_rate": 0.0001, + "loss": 79.5785, + "step": 4190 + }, + { + "epoch": 0.6782261157425163, + "grad_norm": 0.8399350643157959, + "learning_rate": 0.0001, + "loss": 76.8101, + "step": 4200 + }, + { + "epoch": 0.6798409398276175, + "grad_norm": 0.8864424824714661, + "learning_rate": 0.0001, + "loss": 77.8671, + "step": 4210 + }, + { + "epoch": 0.6814557639127188, + "grad_norm": 0.8322188258171082, + "learning_rate": 0.0001, + "loss": 77.6597, + "step": 4220 + }, + { + "epoch": 0.68307058799782, + "grad_norm": 0.8962367177009583, + "learning_rate": 0.0001, + "loss": 77.7324, + "step": 4230 + }, + { + "epoch": 0.6846854120829212, + "grad_norm": 0.9727205038070679, + "learning_rate": 0.0001, + "loss": 77.8531, + "step": 4240 + }, + { + "epoch": 0.6863002361680225, + "grad_norm": 0.956584095954895, + "learning_rate": 0.0001, + "loss": 78.709, + "step": 4250 + }, + { + "epoch": 0.6879150602531237, + "grad_norm": 0.8531600832939148, + "learning_rate": 0.0001, + "loss": 78.7755, + "step": 4260 + }, + { + "epoch": 0.6895298843382249, + "grad_norm": 0.8059844374656677, + "learning_rate": 0.0001, + "loss": 77.1636, + "step": 4270 + }, + { + "epoch": 0.6911447084233261, + "grad_norm": 0.8373456597328186, + "learning_rate": 0.0001, + "loss": 78.6561, + "step": 4280 + }, + { + "epoch": 0.6927595325084274, + "grad_norm": 0.7819933891296387, + "learning_rate": 0.0001, + "loss": 77.6061, + "step": 4290 + }, + { + "epoch": 0.6943743565935286, + "grad_norm": 0.8829817771911621, + "learning_rate": 0.0001, + "loss": 77.3387, + "step": 4300 + }, + { + "epoch": 0.6959891806786298, + "grad_norm": 0.791772723197937, + "learning_rate": 0.0001, + "loss": 77.4368, + "step": 4310 + }, + { + "epoch": 0.6976040047637311, + "grad_norm": 0.9949231147766113, + "learning_rate": 0.0001, + "loss": 73.4359, + "step": 4320 + }, + { + "epoch": 0.6992188288488322, + "grad_norm": 0.8676188588142395, + "learning_rate": 0.0001, + "loss": 78.4836, + "step": 4330 + }, + { + "epoch": 0.7008336529339335, + "grad_norm": 1.1704007387161255, + "learning_rate": 0.0001, + "loss": 76.5091, + "step": 4340 + }, + { + "epoch": 0.7024484770190348, + "grad_norm": 0.8877903819084167, + "learning_rate": 0.0001, + "loss": 78.021, + "step": 4350 + }, + { + "epoch": 0.704063301104136, + "grad_norm": 1.381914496421814, + "learning_rate": 0.0001, + "loss": 77.8544, + "step": 4360 + }, + { + "epoch": 0.7056781251892372, + "grad_norm": 0.835023045539856, + "learning_rate": 0.0001, + "loss": 76.1552, + "step": 4370 + }, + { + "epoch": 0.7072929492743384, + "grad_norm": 0.9814149141311646, + "learning_rate": 0.0001, + "loss": 76.7646, + "step": 4380 + }, + { + "epoch": 0.7089077733594397, + "grad_norm": 0.8012816309928894, + "learning_rate": 0.0001, + "loss": 75.794, + "step": 4390 + }, + { + "epoch": 0.7105225974445409, + "grad_norm": 0.8340234160423279, + "learning_rate": 0.0001, + "loss": 77.4417, + "step": 4400 + }, + { + "epoch": 0.7121374215296421, + "grad_norm": 0.7257950305938721, + "learning_rate": 0.0001, + "loss": 77.1807, + "step": 4410 + }, + { + "epoch": 0.7137522456147434, + "grad_norm": 0.7342708706855774, + "learning_rate": 0.0001, + "loss": 75.5091, + "step": 4420 + }, + { + "epoch": 0.7153670696998445, + "grad_norm": 0.8298904895782471, + "learning_rate": 0.0001, + "loss": 76.6651, + "step": 4430 + }, + { + "epoch": 0.7169818937849458, + "grad_norm": 0.8564396500587463, + "learning_rate": 0.0001, + "loss": 76.0055, + "step": 4440 + }, + { + "epoch": 0.7185967178700471, + "grad_norm": 0.9145296812057495, + "learning_rate": 0.0001, + "loss": 76.0369, + "step": 4450 + }, + { + "epoch": 0.7202115419551482, + "grad_norm": 0.8074191212654114, + "learning_rate": 0.0001, + "loss": 77.4436, + "step": 4460 + }, + { + "epoch": 0.7218263660402495, + "grad_norm": 0.8212682008743286, + "learning_rate": 0.0001, + "loss": 76.3129, + "step": 4470 + }, + { + "epoch": 0.7234411901253507, + "grad_norm": 0.751323938369751, + "learning_rate": 0.0001, + "loss": 77.603, + "step": 4480 + }, + { + "epoch": 0.725056014210452, + "grad_norm": 0.9664170145988464, + "learning_rate": 0.0001, + "loss": 77.481, + "step": 4490 + }, + { + "epoch": 0.7266708382955531, + "grad_norm": 0.7369455695152283, + "learning_rate": 0.0001, + "loss": 76.3238, + "step": 4500 + }, + { + "epoch": 0.7282856623806544, + "grad_norm": 0.8448513746261597, + "learning_rate": 0.0001, + "loss": 76.5679, + "step": 4510 + }, + { + "epoch": 0.7299004864657557, + "grad_norm": 0.8605676293373108, + "learning_rate": 0.0001, + "loss": 75.7667, + "step": 4520 + }, + { + "epoch": 0.7315153105508568, + "grad_norm": 0.8034664392471313, + "learning_rate": 0.0001, + "loss": 76.2163, + "step": 4530 + }, + { + "epoch": 0.7331301346359581, + "grad_norm": 0.7846826910972595, + "learning_rate": 0.0001, + "loss": 73.5979, + "step": 4540 + }, + { + "epoch": 0.7347449587210593, + "grad_norm": 0.7422606348991394, + "learning_rate": 0.0001, + "loss": 76.2334, + "step": 4550 + }, + { + "epoch": 0.7363597828061605, + "grad_norm": 1.4478483200073242, + "learning_rate": 0.0001, + "loss": 77.6168, + "step": 4560 + }, + { + "epoch": 0.7379746068912618, + "grad_norm": 2.0957767963409424, + "learning_rate": 0.0001, + "loss": 76.3199, + "step": 4570 + }, + { + "epoch": 0.739589430976363, + "grad_norm": 0.6757177114486694, + "learning_rate": 0.0001, + "loss": 77.2923, + "step": 4580 + }, + { + "epoch": 0.7412042550614643, + "grad_norm": 0.9281920790672302, + "learning_rate": 0.0001, + "loss": 76.9963, + "step": 4590 + }, + { + "epoch": 0.7428190791465654, + "grad_norm": 0.8702669143676758, + "learning_rate": 0.0001, + "loss": 76.2553, + "step": 4600 + }, + { + "epoch": 0.7444339032316667, + "grad_norm": 0.7976512312889099, + "learning_rate": 0.0001, + "loss": 74.7622, + "step": 4610 + }, + { + "epoch": 0.746048727316768, + "grad_norm": 0.9116189479827881, + "learning_rate": 0.0001, + "loss": 76.3198, + "step": 4620 + }, + { + "epoch": 0.7476635514018691, + "grad_norm": 0.6239303946495056, + "learning_rate": 0.0001, + "loss": 75.7678, + "step": 4630 + }, + { + "epoch": 0.7492783754869704, + "grad_norm": 0.8380976915359497, + "learning_rate": 0.0001, + "loss": 76.8305, + "step": 4640 + }, + { + "epoch": 0.7508931995720716, + "grad_norm": 0.7394141554832458, + "learning_rate": 0.0001, + "loss": 76.5033, + "step": 4650 + }, + { + "epoch": 0.7525080236571728, + "grad_norm": 1.0694036483764648, + "learning_rate": 0.0001, + "loss": 75.0084, + "step": 4660 + }, + { + "epoch": 0.7541228477422741, + "grad_norm": 0.7530115246772766, + "learning_rate": 0.0001, + "loss": 74.4977, + "step": 4670 + }, + { + "epoch": 0.7557376718273753, + "grad_norm": 1.9561909437179565, + "learning_rate": 0.0001, + "loss": 74.7077, + "step": 4680 + }, + { + "epoch": 0.7573524959124766, + "grad_norm": 0.625341534614563, + "learning_rate": 0.0001, + "loss": 76.1238, + "step": 4690 + }, + { + "epoch": 0.7589673199975777, + "grad_norm": 0.9790500998497009, + "learning_rate": 0.0001, + "loss": 76.3997, + "step": 4700 + }, + { + "epoch": 0.760582144082679, + "grad_norm": 0.7087915539741516, + "learning_rate": 0.0001, + "loss": 75.6893, + "step": 4710 + }, + { + "epoch": 0.7621969681677803, + "grad_norm": 1.0995875597000122, + "learning_rate": 0.0001, + "loss": 74.8588, + "step": 4720 + }, + { + "epoch": 0.7638117922528814, + "grad_norm": 1.0309056043624878, + "learning_rate": 0.0001, + "loss": 73.5934, + "step": 4730 + }, + { + "epoch": 0.7654266163379827, + "grad_norm": 1.0132921934127808, + "learning_rate": 0.0001, + "loss": 73.9868, + "step": 4740 + }, + { + "epoch": 0.7670414404230839, + "grad_norm": 0.7138039469718933, + "learning_rate": 0.0001, + "loss": 76.4459, + "step": 4750 + }, + { + "epoch": 0.7686562645081851, + "grad_norm": 0.7055657505989075, + "learning_rate": 0.0001, + "loss": 75.1068, + "step": 4760 + }, + { + "epoch": 0.7702710885932864, + "grad_norm": 0.7398490905761719, + "learning_rate": 0.0001, + "loss": 75.3697, + "step": 4770 + }, + { + "epoch": 0.7718859126783876, + "grad_norm": 0.6836295127868652, + "learning_rate": 0.0001, + "loss": 77.4988, + "step": 4780 + }, + { + "epoch": 0.7735007367634888, + "grad_norm": 0.7552574276924133, + "learning_rate": 0.0001, + "loss": 75.149, + "step": 4790 + }, + { + "epoch": 0.77511556084859, + "grad_norm": 0.7930851578712463, + "learning_rate": 0.0001, + "loss": 73.8661, + "step": 4800 + }, + { + "epoch": 0.7767303849336913, + "grad_norm": 0.7549886107444763, + "learning_rate": 0.0001, + "loss": 75.042, + "step": 4810 + }, + { + "epoch": 0.7783452090187926, + "grad_norm": 0.9932214617729187, + "learning_rate": 0.0001, + "loss": 75.6619, + "step": 4820 + }, + { + "epoch": 0.7799600331038937, + "grad_norm": 0.7685471773147583, + "learning_rate": 0.0001, + "loss": 75.8404, + "step": 4830 + }, + { + "epoch": 0.781574857188995, + "grad_norm": 0.828737199306488, + "learning_rate": 0.0001, + "loss": 75.5839, + "step": 4840 + }, + { + "epoch": 0.7831896812740962, + "grad_norm": 0.8575420379638672, + "learning_rate": 0.0001, + "loss": 75.7889, + "step": 4850 + }, + { + "epoch": 0.7848045053591974, + "grad_norm": 0.7053834199905396, + "learning_rate": 0.0001, + "loss": 75.9372, + "step": 4860 + }, + { + "epoch": 0.7864193294442987, + "grad_norm": 0.7861505150794983, + "learning_rate": 0.0001, + "loss": 76.1532, + "step": 4870 + }, + { + "epoch": 0.7880341535293999, + "grad_norm": 0.8357386589050293, + "learning_rate": 0.0001, + "loss": 75.8536, + "step": 4880 + }, + { + "epoch": 0.7896489776145011, + "grad_norm": 3.448834180831909, + "learning_rate": 0.0001, + "loss": 74.1794, + "step": 4890 + }, + { + "epoch": 0.7912638016996023, + "grad_norm": 0.813522458076477, + "learning_rate": 0.0001, + "loss": 74.5202, + "step": 4900 + }, + { + "epoch": 0.7928786257847036, + "grad_norm": 0.840230405330658, + "learning_rate": 0.0001, + "loss": 74.6634, + "step": 4910 + }, + { + "epoch": 0.7944934498698049, + "grad_norm": 0.7855745553970337, + "learning_rate": 0.0001, + "loss": 73.5369, + "step": 4920 + }, + { + "epoch": 0.796108273954906, + "grad_norm": 0.7708212733268738, + "learning_rate": 0.0001, + "loss": 74.6225, + "step": 4930 + }, + { + "epoch": 0.7977230980400073, + "grad_norm": 0.7140830159187317, + "learning_rate": 0.0001, + "loss": 75.0585, + "step": 4940 + }, + { + "epoch": 0.7993379221251085, + "grad_norm": 0.7960936427116394, + "learning_rate": 0.0001, + "loss": 74.0574, + "step": 4950 + }, + { + "epoch": 0.8009527462102097, + "grad_norm": 0.7269165515899658, + "learning_rate": 0.0001, + "loss": 74.0379, + "step": 4960 + }, + { + "epoch": 0.802567570295311, + "grad_norm": 0.8505013585090637, + "learning_rate": 0.0001, + "loss": 74.7448, + "step": 4970 + }, + { + "epoch": 0.8041823943804122, + "grad_norm": 0.8935654759407043, + "learning_rate": 0.0001, + "loss": 75.4717, + "step": 4980 + }, + { + "epoch": 0.8057972184655134, + "grad_norm": 0.9447595477104187, + "learning_rate": 0.0001, + "loss": 75.9674, + "step": 4990 + }, + { + "epoch": 0.8074120425506146, + "grad_norm": 1.6739715337753296, + "learning_rate": 0.0001, + "loss": 73.9035, + "step": 5000 + }, + { + "epoch": 0.8090268666357159, + "grad_norm": 0.861902117729187, + "learning_rate": 0.0001, + "loss": 74.906, + "step": 5010 + }, + { + "epoch": 0.8106416907208172, + "grad_norm": 0.8328655362129211, + "learning_rate": 0.0001, + "loss": 74.1493, + "step": 5020 + }, + { + "epoch": 0.8122565148059183, + "grad_norm": 0.8193135261535645, + "learning_rate": 0.0001, + "loss": 74.6614, + "step": 5030 + }, + { + "epoch": 0.8138713388910196, + "grad_norm": 0.6954565048217773, + "learning_rate": 0.0001, + "loss": 73.7366, + "step": 5040 + }, + { + "epoch": 0.8154861629761208, + "grad_norm": 0.7381289005279541, + "learning_rate": 0.0001, + "loss": 73.5086, + "step": 5050 + }, + { + "epoch": 0.817100987061222, + "grad_norm": 1.090443730354309, + "learning_rate": 0.0001, + "loss": 75.0695, + "step": 5060 + }, + { + "epoch": 0.8187158111463233, + "grad_norm": 1.2544814348220825, + "learning_rate": 0.0001, + "loss": 72.89, + "step": 5070 + }, + { + "epoch": 0.8203306352314245, + "grad_norm": 0.632146418094635, + "learning_rate": 0.0001, + "loss": 74.8604, + "step": 5080 + }, + { + "epoch": 0.8219454593165257, + "grad_norm": 1.1038259267807007, + "learning_rate": 0.0001, + "loss": 74.7354, + "step": 5090 + }, + { + "epoch": 0.8235602834016269, + "grad_norm": 0.6886508464813232, + "learning_rate": 0.0001, + "loss": 75.4243, + "step": 5100 + }, + { + "epoch": 0.8251751074867282, + "grad_norm": 0.8367568850517273, + "learning_rate": 0.0001, + "loss": 75.2732, + "step": 5110 + }, + { + "epoch": 0.8267899315718293, + "grad_norm": 0.9302715063095093, + "learning_rate": 0.0001, + "loss": 75.7924, + "step": 5120 + }, + { + "epoch": 0.8284047556569306, + "grad_norm": 0.6871219873428345, + "learning_rate": 0.0001, + "loss": 74.2905, + "step": 5130 + }, + { + "epoch": 0.8300195797420319, + "grad_norm": 0.7319445013999939, + "learning_rate": 0.0001, + "loss": 73.7774, + "step": 5140 + }, + { + "epoch": 0.831634403827133, + "grad_norm": 0.7184522151947021, + "learning_rate": 0.0001, + "loss": 73.0554, + "step": 5150 + }, + { + "epoch": 0.8332492279122343, + "grad_norm": 0.8462343811988831, + "learning_rate": 0.0001, + "loss": 73.7615, + "step": 5160 + }, + { + "epoch": 0.8348640519973355, + "grad_norm": 0.9436241984367371, + "learning_rate": 0.0001, + "loss": 73.9464, + "step": 5170 + }, + { + "epoch": 0.8364788760824368, + "grad_norm": 0.9079630374908447, + "learning_rate": 0.0001, + "loss": 73.2727, + "step": 5180 + }, + { + "epoch": 0.838093700167538, + "grad_norm": 0.8280741572380066, + "learning_rate": 0.0001, + "loss": 73.9751, + "step": 5190 + }, + { + "epoch": 0.8397085242526392, + "grad_norm": 0.6475157141685486, + "learning_rate": 0.0001, + "loss": 74.7105, + "step": 5200 + }, + { + "epoch": 0.8413233483377405, + "grad_norm": 0.7090312838554382, + "learning_rate": 0.0001, + "loss": 73.6489, + "step": 5210 + }, + { + "epoch": 0.8429381724228416, + "grad_norm": 1.20967698097229, + "learning_rate": 0.0001, + "loss": 75.1335, + "step": 5220 + }, + { + "epoch": 0.8445529965079429, + "grad_norm": 2.053532600402832, + "learning_rate": 0.0001, + "loss": 75.3111, + "step": 5230 + }, + { + "epoch": 0.8461678205930442, + "grad_norm": 0.7563302516937256, + "learning_rate": 0.0001, + "loss": 72.9935, + "step": 5240 + }, + { + "epoch": 0.8477826446781453, + "grad_norm": 0.7801489233970642, + "learning_rate": 0.0001, + "loss": 74.7856, + "step": 5250 + }, + { + "epoch": 0.8493974687632466, + "grad_norm": 0.737870454788208, + "learning_rate": 0.0001, + "loss": 75.6281, + "step": 5260 + }, + { + "epoch": 0.8510122928483478, + "grad_norm": 0.704352617263794, + "learning_rate": 0.0001, + "loss": 75.4598, + "step": 5270 + }, + { + "epoch": 0.8526271169334491, + "grad_norm": 0.6452057361602783, + "learning_rate": 0.0001, + "loss": 74.0341, + "step": 5280 + }, + { + "epoch": 0.8542419410185503, + "grad_norm": 0.6863471865653992, + "learning_rate": 0.0001, + "loss": 74.3727, + "step": 5290 + }, + { + "epoch": 0.8558567651036515, + "grad_norm": 9.756170272827148, + "learning_rate": 0.0001, + "loss": 75.5231, + "step": 5300 + }, + { + "epoch": 0.8574715891887528, + "grad_norm": 0.8624544739723206, + "learning_rate": 0.0001, + "loss": 74.7821, + "step": 5310 + }, + { + "epoch": 0.8590864132738539, + "grad_norm": 0.6637155413627625, + "learning_rate": 0.0001, + "loss": 74.1153, + "step": 5320 + }, + { + "epoch": 0.8607012373589552, + "grad_norm": 1.0764445066452026, + "learning_rate": 0.0001, + "loss": 73.7208, + "step": 5330 + }, + { + "epoch": 0.8623160614440565, + "grad_norm": 0.8567569255828857, + "learning_rate": 0.0001, + "loss": 73.1399, + "step": 5340 + }, + { + "epoch": 0.8639308855291576, + "grad_norm": 0.7681576013565063, + "learning_rate": 0.0001, + "loss": 73.7577, + "step": 5350 + }, + { + "epoch": 0.8655457096142589, + "grad_norm": 0.9336384534835815, + "learning_rate": 0.0001, + "loss": 74.1022, + "step": 5360 + }, + { + "epoch": 0.8671605336993601, + "grad_norm": 0.8603724837303162, + "learning_rate": 0.0001, + "loss": 75.9579, + "step": 5370 + }, + { + "epoch": 0.8687753577844614, + "grad_norm": 0.77194744348526, + "learning_rate": 0.0001, + "loss": 72.4799, + "step": 5380 + }, + { + "epoch": 0.8703901818695626, + "grad_norm": 0.7480376958847046, + "learning_rate": 0.0001, + "loss": 76.1257, + "step": 5390 + }, + { + "epoch": 0.8720050059546638, + "grad_norm": 0.7841023206710815, + "learning_rate": 0.0001, + "loss": 73.1512, + "step": 5400 + }, + { + "epoch": 0.8736198300397651, + "grad_norm": 0.8741798996925354, + "learning_rate": 0.0001, + "loss": 74.2151, + "step": 5410 + }, + { + "epoch": 0.8752346541248662, + "grad_norm": 0.9762082099914551, + "learning_rate": 0.0001, + "loss": 76.7138, + "step": 5420 + }, + { + "epoch": 0.8768494782099675, + "grad_norm": 0.7670348286628723, + "learning_rate": 0.0001, + "loss": 72.9567, + "step": 5430 + }, + { + "epoch": 0.8784643022950688, + "grad_norm": 0.7449469566345215, + "learning_rate": 0.0001, + "loss": 74.8841, + "step": 5440 + }, + { + "epoch": 0.8800791263801699, + "grad_norm": 0.6984641551971436, + "learning_rate": 0.0001, + "loss": 73.8906, + "step": 5450 + }, + { + "epoch": 0.8816939504652712, + "grad_norm": 0.7854361534118652, + "learning_rate": 0.0001, + "loss": 74.626, + "step": 5460 + }, + { + "epoch": 0.8833087745503724, + "grad_norm": 0.9584240317344666, + "learning_rate": 0.0001, + "loss": 74.7048, + "step": 5470 + }, + { + "epoch": 0.8849235986354737, + "grad_norm": 0.6040284037590027, + "learning_rate": 0.0001, + "loss": 74.7194, + "step": 5480 + }, + { + "epoch": 0.8865384227205749, + "grad_norm": 0.7756215929985046, + "learning_rate": 0.0001, + "loss": 73.8178, + "step": 5490 + }, + { + "epoch": 0.8881532468056761, + "grad_norm": 0.9070586562156677, + "learning_rate": 0.0001, + "loss": 72.8309, + "step": 5500 + }, + { + "epoch": 0.8897680708907774, + "grad_norm": 0.8464986681938171, + "learning_rate": 0.0001, + "loss": 73.8847, + "step": 5510 + }, + { + "epoch": 0.8913828949758785, + "grad_norm": 0.7968605160713196, + "learning_rate": 0.0001, + "loss": 73.1691, + "step": 5520 + }, + { + "epoch": 0.8929977190609798, + "grad_norm": 0.9934583306312561, + "learning_rate": 0.0001, + "loss": 76.2662, + "step": 5530 + }, + { + "epoch": 0.8946125431460811, + "grad_norm": 0.7272152900695801, + "learning_rate": 0.0001, + "loss": 75.5295, + "step": 5540 + }, + { + "epoch": 0.8962273672311822, + "grad_norm": 0.857345461845398, + "learning_rate": 0.0001, + "loss": 75.3703, + "step": 5550 + }, + { + "epoch": 0.8978421913162835, + "grad_norm": 0.713918924331665, + "learning_rate": 0.0001, + "loss": 73.4563, + "step": 5560 + }, + { + "epoch": 0.8994570154013847, + "grad_norm": 0.759233832359314, + "learning_rate": 0.0001, + "loss": 73.2744, + "step": 5570 + }, + { + "epoch": 0.901071839486486, + "grad_norm": 0.7085754871368408, + "learning_rate": 0.0001, + "loss": 74.7792, + "step": 5580 + }, + { + "epoch": 0.9026866635715872, + "grad_norm": 0.7370456457138062, + "learning_rate": 0.0001, + "loss": 74.4216, + "step": 5590 + }, + { + "epoch": 0.9043014876566884, + "grad_norm": 0.779746949672699, + "learning_rate": 0.0001, + "loss": 75.9822, + "step": 5600 + }, + { + "epoch": 0.9059163117417897, + "grad_norm": 0.7915763854980469, + "learning_rate": 0.0001, + "loss": 74.3706, + "step": 5610 + }, + { + "epoch": 0.9075311358268908, + "grad_norm": 0.7886125445365906, + "learning_rate": 0.0001, + "loss": 72.9453, + "step": 5620 + }, + { + "epoch": 0.9091459599119921, + "grad_norm": 0.7804751992225647, + "learning_rate": 0.0001, + "loss": 73.514, + "step": 5630 + }, + { + "epoch": 0.9107607839970934, + "grad_norm": 0.7168806195259094, + "learning_rate": 0.0001, + "loss": 71.4159, + "step": 5640 + }, + { + "epoch": 0.9123756080821945, + "grad_norm": 0.7553626894950867, + "learning_rate": 0.0001, + "loss": 74.4353, + "step": 5650 + }, + { + "epoch": 0.9139904321672958, + "grad_norm": 0.6688916683197021, + "learning_rate": 0.0001, + "loss": 74.3786, + "step": 5660 + }, + { + "epoch": 0.915605256252397, + "grad_norm": 0.7425532341003418, + "learning_rate": 0.0001, + "loss": 73.3355, + "step": 5670 + }, + { + "epoch": 0.9172200803374982, + "grad_norm": 0.6473882794380188, + "learning_rate": 0.0001, + "loss": 74.2315, + "step": 5680 + }, + { + "epoch": 0.9188349044225994, + "grad_norm": 0.7222439050674438, + "learning_rate": 0.0001, + "loss": 74.2885, + "step": 5690 + }, + { + "epoch": 0.9204497285077007, + "grad_norm": 1.5716948509216309, + "learning_rate": 0.0001, + "loss": 74.6739, + "step": 5700 + }, + { + "epoch": 0.922064552592802, + "grad_norm": 0.8077759742736816, + "learning_rate": 0.0001, + "loss": 72.6617, + "step": 5710 + }, + { + "epoch": 0.9236793766779031, + "grad_norm": 0.6948631405830383, + "learning_rate": 0.0001, + "loss": 73.6247, + "step": 5720 + }, + { + "epoch": 0.9252942007630044, + "grad_norm": 0.8823757767677307, + "learning_rate": 0.0001, + "loss": 73.2536, + "step": 5730 + }, + { + "epoch": 0.9269090248481056, + "grad_norm": 0.7066764235496521, + "learning_rate": 0.0001, + "loss": 74.1748, + "step": 5740 + }, + { + "epoch": 0.9285238489332068, + "grad_norm": 0.6979793906211853, + "learning_rate": 0.0001, + "loss": 72.3189, + "step": 5750 + }, + { + "epoch": 0.9301386730183081, + "grad_norm": 0.6961462497711182, + "learning_rate": 0.0001, + "loss": 74.6196, + "step": 5760 + }, + { + "epoch": 0.9317534971034093, + "grad_norm": 0.7797656655311584, + "learning_rate": 0.0001, + "loss": 74.5472, + "step": 5770 + }, + { + "epoch": 0.9333683211885105, + "grad_norm": 1.0671838521957397, + "learning_rate": 0.0001, + "loss": 73.241, + "step": 5780 + }, + { + "epoch": 0.9349831452736117, + "grad_norm": 0.7566089034080505, + "learning_rate": 0.0001, + "loss": 73.6784, + "step": 5790 + }, + { + "epoch": 0.936597969358713, + "grad_norm": 0.8563273549079895, + "learning_rate": 0.0001, + "loss": 73.6871, + "step": 5800 + }, + { + "epoch": 0.9382127934438143, + "grad_norm": 0.7885987162590027, + "learning_rate": 0.0001, + "loss": 73.4447, + "step": 5810 + }, + { + "epoch": 0.9398276175289154, + "grad_norm": 0.8751112818717957, + "learning_rate": 0.0001, + "loss": 74.0531, + "step": 5820 + }, + { + "epoch": 0.9414424416140167, + "grad_norm": 0.6663336157798767, + "learning_rate": 0.0001, + "loss": 73.0557, + "step": 5830 + }, + { + "epoch": 0.9430572656991179, + "grad_norm": 0.7993099689483643, + "learning_rate": 0.0001, + "loss": 74.4467, + "step": 5840 + }, + { + "epoch": 0.9446720897842191, + "grad_norm": 0.6987188458442688, + "learning_rate": 0.0001, + "loss": 74.0327, + "step": 5850 + }, + { + "epoch": 0.9462869138693204, + "grad_norm": 0.8458583354949951, + "learning_rate": 0.0001, + "loss": 73.1757, + "step": 5860 + }, + { + "epoch": 0.9479017379544216, + "grad_norm": 0.6372436881065369, + "learning_rate": 0.0001, + "loss": 73.1067, + "step": 5870 + }, + { + "epoch": 0.9495165620395228, + "grad_norm": 0.7447753548622131, + "learning_rate": 0.0001, + "loss": 75.5461, + "step": 5880 + }, + { + "epoch": 0.951131386124624, + "grad_norm": 0.6633971929550171, + "learning_rate": 0.0001, + "loss": 72.9966, + "step": 5890 + }, + { + "epoch": 0.9527462102097253, + "grad_norm": 0.8576784729957581, + "learning_rate": 0.0001, + "loss": 71.5712, + "step": 5900 + }, + { + "epoch": 0.9543610342948265, + "grad_norm": 1.4696853160858154, + "learning_rate": 0.0001, + "loss": 74.7893, + "step": 5910 + }, + { + "epoch": 0.9559758583799277, + "grad_norm": 0.611210286617279, + "learning_rate": 0.0001, + "loss": 74.3426, + "step": 5920 + }, + { + "epoch": 0.957590682465029, + "grad_norm": 0.5947410464286804, + "learning_rate": 0.0001, + "loss": 71.2563, + "step": 5930 + }, + { + "epoch": 0.9592055065501301, + "grad_norm": 0.7169992923736572, + "learning_rate": 0.0001, + "loss": 71.6049, + "step": 5940 + }, + { + "epoch": 0.9608203306352314, + "grad_norm": 0.6773818731307983, + "learning_rate": 0.0001, + "loss": 75.0328, + "step": 5950 + }, + { + "epoch": 0.9624351547203327, + "grad_norm": 0.6714807748794556, + "learning_rate": 0.0001, + "loss": 71.5105, + "step": 5960 + }, + { + "epoch": 0.9640499788054339, + "grad_norm": 1.0064337253570557, + "learning_rate": 0.0001, + "loss": 75.8887, + "step": 5970 + }, + { + "epoch": 0.9656648028905351, + "grad_norm": 0.9802227020263672, + "learning_rate": 0.0001, + "loss": 73.5655, + "step": 5980 + }, + { + "epoch": 0.9672796269756363, + "grad_norm": 0.7097306251525879, + "learning_rate": 0.0001, + "loss": 73.298, + "step": 5990 + }, + { + "epoch": 0.9688944510607376, + "grad_norm": 0.6680361032485962, + "learning_rate": 0.0001, + "loss": 73.8873, + "step": 6000 + }, + { + "epoch": 0.9705092751458388, + "grad_norm": 1.1302506923675537, + "learning_rate": 0.0001, + "loss": 73.3328, + "step": 6010 + }, + { + "epoch": 0.97212409923094, + "grad_norm": 0.7163954377174377, + "learning_rate": 0.0001, + "loss": 70.5955, + "step": 6020 + }, + { + "epoch": 0.9737389233160413, + "grad_norm": 0.8182240128517151, + "learning_rate": 0.0001, + "loss": 72.6397, + "step": 6030 + }, + { + "epoch": 0.9753537474011424, + "grad_norm": 0.6752052903175354, + "learning_rate": 0.0001, + "loss": 73.1188, + "step": 6040 + }, + { + "epoch": 0.9769685714862437, + "grad_norm": 0.8829680681228638, + "learning_rate": 0.0001, + "loss": 75.7346, + "step": 6050 + }, + { + "epoch": 0.978583395571345, + "grad_norm": 0.9254767894744873, + "learning_rate": 0.0001, + "loss": 73.6221, + "step": 6060 + }, + { + "epoch": 0.9801982196564462, + "grad_norm": 0.7527495622634888, + "learning_rate": 0.0001, + "loss": 73.948, + "step": 6070 + }, + { + "epoch": 0.9818130437415474, + "grad_norm": 1.0135066509246826, + "learning_rate": 0.0001, + "loss": 72.8929, + "step": 6080 + }, + { + "epoch": 0.9834278678266486, + "grad_norm": 0.7093394994735718, + "learning_rate": 0.0001, + "loss": 73.5826, + "step": 6090 + }, + { + "epoch": 0.9850426919117499, + "grad_norm": 0.6651231646537781, + "learning_rate": 0.0001, + "loss": 73.167, + "step": 6100 + }, + { + "epoch": 0.9866575159968511, + "grad_norm": 0.720552921295166, + "learning_rate": 0.0001, + "loss": 74.8985, + "step": 6110 + }, + { + "epoch": 0.9882723400819523, + "grad_norm": 1.7821632623672485, + "learning_rate": 0.0001, + "loss": 74.1414, + "step": 6120 + }, + { + "epoch": 0.9898871641670536, + "grad_norm": 0.6405150890350342, + "learning_rate": 0.0001, + "loss": 73.5989, + "step": 6130 + }, + { + "epoch": 0.9915019882521547, + "grad_norm": 0.8207482695579529, + "learning_rate": 0.0001, + "loss": 73.0977, + "step": 6140 + }, + { + "epoch": 0.993116812337256, + "grad_norm": 0.7320757508277893, + "learning_rate": 0.0001, + "loss": 72.0241, + "step": 6150 + }, + { + "epoch": 0.9947316364223573, + "grad_norm": 0.9522950649261475, + "learning_rate": 0.0001, + "loss": 73.9831, + "step": 6160 + }, + { + "epoch": 0.9963464605074585, + "grad_norm": 0.7437159419059753, + "learning_rate": 0.0001, + "loss": 72.1054, + "step": 6170 + }, + { + "epoch": 0.9979612845925597, + "grad_norm": 0.7035623788833618, + "learning_rate": 0.0001, + "loss": 73.959, + "step": 6180 + }, + { + "epoch": 0.9995761086776609, + "grad_norm": 0.7559277415275574, + "learning_rate": 0.0001, + "loss": 71.4651, + "step": 6190 + }, + { + "epoch": 1.001130376859571, + "grad_norm": 0.852342426776886, + "learning_rate": 0.0001, + "loss": 71.8844, + "step": 6200 + }, + { + "epoch": 1.002745200944672, + "grad_norm": 0.8018911480903625, + "learning_rate": 0.0001, + "loss": 74.5093, + "step": 6210 + }, + { + "epoch": 1.0043600250297733, + "grad_norm": 0.6613730192184448, + "learning_rate": 0.0001, + "loss": 71.9953, + "step": 6220 + }, + { + "epoch": 1.0059748491148746, + "grad_norm": 0.9815977215766907, + "learning_rate": 0.0001, + "loss": 70.8632, + "step": 6230 + }, + { + "epoch": 1.0075896731999758, + "grad_norm": 0.781581699848175, + "learning_rate": 0.0001, + "loss": 72.3603, + "step": 6240 + }, + { + "epoch": 1.009204497285077, + "grad_norm": 0.8079054355621338, + "learning_rate": 0.0001, + "loss": 73.6153, + "step": 6250 + }, + { + "epoch": 1.0108193213701782, + "grad_norm": 0.8356053233146667, + "learning_rate": 0.0001, + "loss": 73.1562, + "step": 6260 + }, + { + "epoch": 1.0124341454552794, + "grad_norm": 0.7107778787612915, + "learning_rate": 0.0001, + "loss": 74.5276, + "step": 6270 + }, + { + "epoch": 1.0140489695403807, + "grad_norm": 0.8601720929145813, + "learning_rate": 0.0001, + "loss": 70.103, + "step": 6280 + }, + { + "epoch": 1.015663793625482, + "grad_norm": 0.675837516784668, + "learning_rate": 0.0001, + "loss": 72.6102, + "step": 6290 + }, + { + "epoch": 1.0172786177105833, + "grad_norm": 0.7160016298294067, + "learning_rate": 0.0001, + "loss": 72.2404, + "step": 6300 + }, + { + "epoch": 1.0188934417956843, + "grad_norm": 0.7286407351493835, + "learning_rate": 0.0001, + "loss": 72.7149, + "step": 6310 + }, + { + "epoch": 1.0205082658807856, + "grad_norm": 0.8183980584144592, + "learning_rate": 0.0001, + "loss": 73.5827, + "step": 6320 + }, + { + "epoch": 1.0221230899658869, + "grad_norm": 0.7300626039505005, + "learning_rate": 0.0001, + "loss": 72.3459, + "step": 6330 + }, + { + "epoch": 1.0237379140509881, + "grad_norm": 0.7222613096237183, + "learning_rate": 0.0001, + "loss": 71.881, + "step": 6340 + }, + { + "epoch": 1.0253527381360894, + "grad_norm": 0.7324674725532532, + "learning_rate": 0.0001, + "loss": 74.4781, + "step": 6350 + }, + { + "epoch": 1.0269675622211905, + "grad_norm": 0.9173555374145508, + "learning_rate": 0.0001, + "loss": 74.3222, + "step": 6360 + }, + { + "epoch": 1.0285823863062917, + "grad_norm": 0.8404964208602905, + "learning_rate": 0.0001, + "loss": 71.8959, + "step": 6370 + }, + { + "epoch": 1.030197210391393, + "grad_norm": 0.8735223412513733, + "learning_rate": 0.0001, + "loss": 72.7251, + "step": 6380 + }, + { + "epoch": 1.0318120344764943, + "grad_norm": 0.8467233777046204, + "learning_rate": 0.0001, + "loss": 72.5218, + "step": 6390 + }, + { + "epoch": 1.0334268585615956, + "grad_norm": 0.673565149307251, + "learning_rate": 0.0001, + "loss": 70.6392, + "step": 6400 + }, + { + "epoch": 1.0350416826466966, + "grad_norm": 0.6558571457862854, + "learning_rate": 0.0001, + "loss": 73.5305, + "step": 6410 + }, + { + "epoch": 1.0366565067317979, + "grad_norm": 0.7251600623130798, + "learning_rate": 0.0001, + "loss": 72.76, + "step": 6420 + }, + { + "epoch": 1.0382713308168992, + "grad_norm": 0.7458305358886719, + "learning_rate": 0.0001, + "loss": 73.6812, + "step": 6430 + }, + { + "epoch": 1.0398861549020004, + "grad_norm": 1.493802547454834, + "learning_rate": 0.0001, + "loss": 74.2455, + "step": 6440 + }, + { + "epoch": 1.0415009789871017, + "grad_norm": 0.894632875919342, + "learning_rate": 0.0001, + "loss": 74.2072, + "step": 6450 + }, + { + "epoch": 1.0431158030722028, + "grad_norm": 0.7142218947410583, + "learning_rate": 0.0001, + "loss": 71.4293, + "step": 6460 + }, + { + "epoch": 1.044730627157304, + "grad_norm": 0.702639639377594, + "learning_rate": 0.0001, + "loss": 71.5383, + "step": 6470 + }, + { + "epoch": 1.0463454512424053, + "grad_norm": 0.9378212094306946, + "learning_rate": 0.0001, + "loss": 72.8916, + "step": 6480 + }, + { + "epoch": 1.0479602753275066, + "grad_norm": 0.8298363089561462, + "learning_rate": 0.0001, + "loss": 72.3344, + "step": 6490 + }, + { + "epoch": 1.0495750994126078, + "grad_norm": 0.6617016792297363, + "learning_rate": 0.0001, + "loss": 71.0487, + "step": 6500 + }, + { + "epoch": 1.051189923497709, + "grad_norm": 0.638520359992981, + "learning_rate": 0.0001, + "loss": 72.2583, + "step": 6510 + }, + { + "epoch": 1.0528047475828102, + "grad_norm": 0.7101534008979797, + "learning_rate": 0.0001, + "loss": 70.8697, + "step": 6520 + }, + { + "epoch": 1.0544195716679114, + "grad_norm": 0.9425339698791504, + "learning_rate": 0.0001, + "loss": 71.0641, + "step": 6530 + }, + { + "epoch": 1.0560343957530127, + "grad_norm": 0.6572297811508179, + "learning_rate": 0.0001, + "loss": 72.6265, + "step": 6540 + }, + { + "epoch": 1.057649219838114, + "grad_norm": 0.745622456073761, + "learning_rate": 0.0001, + "loss": 73.0587, + "step": 6550 + }, + { + "epoch": 1.059264043923215, + "grad_norm": 0.7667096257209778, + "learning_rate": 0.0001, + "loss": 72.1779, + "step": 6560 + }, + { + "epoch": 1.0608788680083163, + "grad_norm": 0.7322410941123962, + "learning_rate": 0.0001, + "loss": 71.5503, + "step": 6570 + }, + { + "epoch": 1.0624936920934176, + "grad_norm": 0.5739616751670837, + "learning_rate": 0.0001, + "loss": 72.3846, + "step": 6580 + }, + { + "epoch": 1.0641085161785189, + "grad_norm": 0.7149553894996643, + "learning_rate": 0.0001, + "loss": 71.5459, + "step": 6590 + }, + { + "epoch": 1.0657233402636201, + "grad_norm": 0.6634597778320312, + "learning_rate": 0.0001, + "loss": 72.2528, + "step": 6600 + }, + { + "epoch": 1.0673381643487212, + "grad_norm": 0.7208803296089172, + "learning_rate": 0.0001, + "loss": 71.1115, + "step": 6610 + }, + { + "epoch": 1.0689529884338225, + "grad_norm": 0.9845868945121765, + "learning_rate": 0.0001, + "loss": 72.6418, + "step": 6620 + }, + { + "epoch": 1.0705678125189237, + "grad_norm": 0.9322167634963989, + "learning_rate": 0.0001, + "loss": 71.937, + "step": 6630 + }, + { + "epoch": 1.072182636604025, + "grad_norm": 0.6323093771934509, + "learning_rate": 0.0001, + "loss": 71.1537, + "step": 6640 + }, + { + "epoch": 1.0737974606891263, + "grad_norm": 0.6476802229881287, + "learning_rate": 0.0001, + "loss": 73.7811, + "step": 6650 + }, + { + "epoch": 1.0754122847742273, + "grad_norm": 0.7882822751998901, + "learning_rate": 0.0001, + "loss": 71.9548, + "step": 6660 + }, + { + "epoch": 1.0770271088593286, + "grad_norm": 0.7651168704032898, + "learning_rate": 0.0001, + "loss": 73.4066, + "step": 6670 + }, + { + "epoch": 1.07864193294443, + "grad_norm": 0.7536904215812683, + "learning_rate": 0.0001, + "loss": 72.1063, + "step": 6680 + }, + { + "epoch": 1.0802567570295312, + "grad_norm": 0.7908427119255066, + "learning_rate": 0.0001, + "loss": 72.6374, + "step": 6690 + }, + { + "epoch": 1.0818715811146324, + "grad_norm": 0.7796691060066223, + "learning_rate": 0.0001, + "loss": 72.2938, + "step": 6700 + }, + { + "epoch": 1.0834864051997335, + "grad_norm": 0.8512274622917175, + "learning_rate": 0.0001, + "loss": 71.556, + "step": 6710 + }, + { + "epoch": 1.0851012292848348, + "grad_norm": 0.7285851836204529, + "learning_rate": 0.0001, + "loss": 72.406, + "step": 6720 + }, + { + "epoch": 1.086716053369936, + "grad_norm": 0.6613008975982666, + "learning_rate": 0.0001, + "loss": 70.6509, + "step": 6730 + }, + { + "epoch": 1.0883308774550373, + "grad_norm": 0.7158271074295044, + "learning_rate": 0.0001, + "loss": 71.8434, + "step": 6740 + }, + { + "epoch": 1.0899457015401384, + "grad_norm": 0.8635967373847961, + "learning_rate": 0.0001, + "loss": 71.8279, + "step": 6750 + }, + { + "epoch": 1.0915605256252396, + "grad_norm": 0.7020159959793091, + "learning_rate": 0.0001, + "loss": 72.2417, + "step": 6760 + }, + { + "epoch": 1.093175349710341, + "grad_norm": 0.6389853954315186, + "learning_rate": 0.0001, + "loss": 72.1879, + "step": 6770 + }, + { + "epoch": 1.0947901737954422, + "grad_norm": 0.6460697650909424, + "learning_rate": 0.0001, + "loss": 72.3356, + "step": 6780 + }, + { + "epoch": 1.0964049978805435, + "grad_norm": 0.7152006030082703, + "learning_rate": 0.0001, + "loss": 71.3725, + "step": 6790 + }, + { + "epoch": 1.0980198219656445, + "grad_norm": 1.0050997734069824, + "learning_rate": 0.0001, + "loss": 72.5452, + "step": 6800 + }, + { + "epoch": 1.0996346460507458, + "grad_norm": 0.6967180371284485, + "learning_rate": 0.0001, + "loss": 72.0832, + "step": 6810 + }, + { + "epoch": 1.101249470135847, + "grad_norm": 0.7472837567329407, + "learning_rate": 0.0001, + "loss": 72.8122, + "step": 6820 + }, + { + "epoch": 1.1028642942209483, + "grad_norm": 0.8198382258415222, + "learning_rate": 0.0001, + "loss": 72.5116, + "step": 6830 + }, + { + "epoch": 1.1044791183060496, + "grad_norm": 0.8112207651138306, + "learning_rate": 0.0001, + "loss": 72.4218, + "step": 6840 + }, + { + "epoch": 1.1060939423911507, + "grad_norm": 0.8412114977836609, + "learning_rate": 0.0001, + "loss": 71.0198, + "step": 6850 + }, + { + "epoch": 1.107708766476252, + "grad_norm": 0.6730719804763794, + "learning_rate": 0.0001, + "loss": 71.2045, + "step": 6860 + }, + { + "epoch": 1.1093235905613532, + "grad_norm": 0.6895949244499207, + "learning_rate": 0.0001, + "loss": 72.3056, + "step": 6870 + }, + { + "epoch": 1.1109384146464545, + "grad_norm": 2.476609945297241, + "learning_rate": 0.0001, + "loss": 70.3865, + "step": 6880 + }, + { + "epoch": 1.1125532387315558, + "grad_norm": 0.7285001873970032, + "learning_rate": 0.0001, + "loss": 71.5761, + "step": 6890 + }, + { + "epoch": 1.1141680628166568, + "grad_norm": 0.7546679973602295, + "learning_rate": 0.0001, + "loss": 72.5979, + "step": 6900 + }, + { + "epoch": 1.115782886901758, + "grad_norm": 0.8974841237068176, + "learning_rate": 0.0001, + "loss": 74.5888, + "step": 6910 + }, + { + "epoch": 1.1173977109868594, + "grad_norm": 0.8785758018493652, + "learning_rate": 0.0001, + "loss": 71.662, + "step": 6920 + }, + { + "epoch": 1.1190125350719606, + "grad_norm": 0.6805328130722046, + "learning_rate": 0.0001, + "loss": 72.201, + "step": 6930 + }, + { + "epoch": 1.120627359157062, + "grad_norm": 0.8132315278053284, + "learning_rate": 0.0001, + "loss": 72.7473, + "step": 6940 + }, + { + "epoch": 1.122242183242163, + "grad_norm": 1.1625338792800903, + "learning_rate": 0.0001, + "loss": 70.5987, + "step": 6950 + }, + { + "epoch": 1.1238570073272642, + "grad_norm": 1.0984737873077393, + "learning_rate": 0.0001, + "loss": 72.7335, + "step": 6960 + }, + { + "epoch": 1.1254718314123655, + "grad_norm": 0.6798779368400574, + "learning_rate": 0.0001, + "loss": 72.1942, + "step": 6970 + }, + { + "epoch": 1.1270866554974668, + "grad_norm": 0.7546010613441467, + "learning_rate": 0.0001, + "loss": 70.8014, + "step": 6980 + }, + { + "epoch": 1.128701479582568, + "grad_norm": 1.0238053798675537, + "learning_rate": 0.0001, + "loss": 72.5617, + "step": 6990 + }, + { + "epoch": 1.130316303667669, + "grad_norm": 0.6535419821739197, + "learning_rate": 0.0001, + "loss": 70.2193, + "step": 7000 + }, + { + "epoch": 1.1319311277527704, + "grad_norm": 0.7354230284690857, + "learning_rate": 0.0001, + "loss": 72.0685, + "step": 7010 + }, + { + "epoch": 1.1335459518378717, + "grad_norm": 0.7848799824714661, + "learning_rate": 0.0001, + "loss": 72.2869, + "step": 7020 + }, + { + "epoch": 1.135160775922973, + "grad_norm": 0.6519262790679932, + "learning_rate": 0.0001, + "loss": 72.2023, + "step": 7030 + }, + { + "epoch": 1.1367756000080742, + "grad_norm": 0.8428511023521423, + "learning_rate": 0.0001, + "loss": 71.3396, + "step": 7040 + }, + { + "epoch": 1.1383904240931753, + "grad_norm": 0.8873165249824524, + "learning_rate": 0.0001, + "loss": 69.8284, + "step": 7050 + }, + { + "epoch": 1.1400052481782765, + "grad_norm": 0.9141374230384827, + "learning_rate": 0.0001, + "loss": 70.3666, + "step": 7060 + }, + { + "epoch": 1.1416200722633778, + "grad_norm": 0.6556552648544312, + "learning_rate": 0.0001, + "loss": 71.1316, + "step": 7070 + }, + { + "epoch": 1.143234896348479, + "grad_norm": 0.6736201047897339, + "learning_rate": 0.0001, + "loss": 70.2601, + "step": 7080 + }, + { + "epoch": 1.1448497204335804, + "grad_norm": 0.6965599656105042, + "learning_rate": 0.0001, + "loss": 71.9409, + "step": 7090 + }, + { + "epoch": 1.1464645445186814, + "grad_norm": 1.0600581169128418, + "learning_rate": 0.0001, + "loss": 72.7344, + "step": 7100 + }, + { + "epoch": 1.1480793686037827, + "grad_norm": 0.7501764297485352, + "learning_rate": 0.0001, + "loss": 72.3129, + "step": 7110 + }, + { + "epoch": 1.149694192688884, + "grad_norm": 0.7105675935745239, + "learning_rate": 0.0001, + "loss": 71.1356, + "step": 7120 + }, + { + "epoch": 1.1513090167739852, + "grad_norm": 0.7446026802062988, + "learning_rate": 0.0001, + "loss": 69.8208, + "step": 7130 + }, + { + "epoch": 1.1529238408590865, + "grad_norm": 0.6664865016937256, + "learning_rate": 0.0001, + "loss": 72.2466, + "step": 7140 + }, + { + "epoch": 1.1545386649441876, + "grad_norm": 1.7590374946594238, + "learning_rate": 0.0001, + "loss": 71.4987, + "step": 7150 + }, + { + "epoch": 1.1561534890292888, + "grad_norm": 0.7941271066665649, + "learning_rate": 0.0001, + "loss": 70.787, + "step": 7160 + }, + { + "epoch": 1.15776831311439, + "grad_norm": 0.7473573088645935, + "learning_rate": 0.0001, + "loss": 71.2516, + "step": 7170 + }, + { + "epoch": 1.1593831371994914, + "grad_norm": 0.7589637041091919, + "learning_rate": 0.0001, + "loss": 71.4781, + "step": 7180 + }, + { + "epoch": 1.1609979612845926, + "grad_norm": 0.8148510456085205, + "learning_rate": 0.0001, + "loss": 71.1491, + "step": 7190 + }, + { + "epoch": 1.1626127853696937, + "grad_norm": 0.6803365349769592, + "learning_rate": 0.0001, + "loss": 71.7937, + "step": 7200 + }, + { + "epoch": 1.164227609454795, + "grad_norm": 0.7314755320549011, + "learning_rate": 0.0001, + "loss": 70.6566, + "step": 7210 + }, + { + "epoch": 1.1658424335398963, + "grad_norm": 0.6494631171226501, + "learning_rate": 0.0001, + "loss": 70.5057, + "step": 7220 + }, + { + "epoch": 1.1674572576249975, + "grad_norm": 0.6076239943504333, + "learning_rate": 0.0001, + "loss": 72.2985, + "step": 7230 + }, + { + "epoch": 1.1690720817100988, + "grad_norm": 0.6650206446647644, + "learning_rate": 0.0001, + "loss": 71.0719, + "step": 7240 + }, + { + "epoch": 1.1706869057951999, + "grad_norm": 0.6926266551017761, + "learning_rate": 0.0001, + "loss": 70.3714, + "step": 7250 + }, + { + "epoch": 1.1723017298803011, + "grad_norm": 0.6534716486930847, + "learning_rate": 0.0001, + "loss": 71.3194, + "step": 7260 + }, + { + "epoch": 1.1739165539654024, + "grad_norm": 0.7194528579711914, + "learning_rate": 0.0001, + "loss": 72.3277, + "step": 7270 + }, + { + "epoch": 1.1755313780505037, + "grad_norm": 0.7496437430381775, + "learning_rate": 0.0001, + "loss": 71.1136, + "step": 7280 + }, + { + "epoch": 1.177146202135605, + "grad_norm": 0.690522313117981, + "learning_rate": 0.0001, + "loss": 70.065, + "step": 7290 + }, + { + "epoch": 1.178761026220706, + "grad_norm": 0.6966914534568787, + "learning_rate": 0.0001, + "loss": 73.1137, + "step": 7300 + }, + { + "epoch": 1.1803758503058073, + "grad_norm": 0.7304641604423523, + "learning_rate": 0.0001, + "loss": 71.2466, + "step": 7310 + }, + { + "epoch": 1.1819906743909085, + "grad_norm": 0.7564008235931396, + "learning_rate": 0.0001, + "loss": 71.4147, + "step": 7320 + }, + { + "epoch": 1.1836054984760098, + "grad_norm": 0.61886066198349, + "learning_rate": 0.0001, + "loss": 70.3297, + "step": 7330 + }, + { + "epoch": 1.185220322561111, + "grad_norm": 0.6281574964523315, + "learning_rate": 0.0001, + "loss": 70.9607, + "step": 7340 + }, + { + "epoch": 1.1868351466462121, + "grad_norm": 0.6583316326141357, + "learning_rate": 0.0001, + "loss": 70.3443, + "step": 7350 + }, + { + "epoch": 1.1884499707313134, + "grad_norm": 1.8159942626953125, + "learning_rate": 0.0001, + "loss": 71.4792, + "step": 7360 + }, + { + "epoch": 1.1900647948164147, + "grad_norm": 0.7253140807151794, + "learning_rate": 0.0001, + "loss": 72.2182, + "step": 7370 + }, + { + "epoch": 1.191679618901516, + "grad_norm": 0.9141016602516174, + "learning_rate": 0.0001, + "loss": 68.7443, + "step": 7380 + }, + { + "epoch": 1.193294442986617, + "grad_norm": 0.7392430901527405, + "learning_rate": 0.0001, + "loss": 69.1888, + "step": 7390 + }, + { + "epoch": 1.1949092670717183, + "grad_norm": 0.7713924646377563, + "learning_rate": 0.0001, + "loss": 69.1301, + "step": 7400 + }, + { + "epoch": 1.1965240911568196, + "grad_norm": 0.6000798344612122, + "learning_rate": 0.0001, + "loss": 71.2576, + "step": 7410 + }, + { + "epoch": 1.1981389152419208, + "grad_norm": 0.687111496925354, + "learning_rate": 0.0001, + "loss": 70.2919, + "step": 7420 + }, + { + "epoch": 1.1997537393270221, + "grad_norm": 1.1718226671218872, + "learning_rate": 0.0001, + "loss": 71.9498, + "step": 7430 + }, + { + "epoch": 1.2013685634121232, + "grad_norm": 0.7176038026809692, + "learning_rate": 0.0001, + "loss": 71.91, + "step": 7440 + }, + { + "epoch": 1.2029833874972244, + "grad_norm": 0.7705256938934326, + "learning_rate": 0.0001, + "loss": 70.1241, + "step": 7450 + }, + { + "epoch": 1.2045982115823257, + "grad_norm": 0.8004609942436218, + "learning_rate": 0.0001, + "loss": 69.5121, + "step": 7460 + }, + { + "epoch": 1.206213035667427, + "grad_norm": 0.6708329319953918, + "learning_rate": 0.0001, + "loss": 73.0173, + "step": 7470 + }, + { + "epoch": 1.2078278597525283, + "grad_norm": 0.8777983784675598, + "learning_rate": 0.0001, + "loss": 71.6843, + "step": 7480 + }, + { + "epoch": 1.2094426838376293, + "grad_norm": 0.7683864831924438, + "learning_rate": 0.0001, + "loss": 71.6914, + "step": 7490 + }, + { + "epoch": 1.2110575079227306, + "grad_norm": 0.8022210597991943, + "learning_rate": 0.0001, + "loss": 69.1724, + "step": 7500 + }, + { + "epoch": 1.2126723320078319, + "grad_norm": 0.7250640392303467, + "learning_rate": 0.0001, + "loss": 70.6568, + "step": 7510 + }, + { + "epoch": 1.2142871560929331, + "grad_norm": 0.5625214576721191, + "learning_rate": 0.0001, + "loss": 69.4849, + "step": 7520 + }, + { + "epoch": 1.2159019801780344, + "grad_norm": 0.7980468273162842, + "learning_rate": 0.0001, + "loss": 70.8164, + "step": 7530 + }, + { + "epoch": 1.2175168042631355, + "grad_norm": 0.6900951862335205, + "learning_rate": 0.0001, + "loss": 71.1719, + "step": 7540 + }, + { + "epoch": 1.2191316283482367, + "grad_norm": 0.6284971833229065, + "learning_rate": 0.0001, + "loss": 70.1751, + "step": 7550 + }, + { + "epoch": 1.220746452433338, + "grad_norm": 0.6641798615455627, + "learning_rate": 0.0001, + "loss": 71.9043, + "step": 7560 + }, + { + "epoch": 1.2223612765184393, + "grad_norm": 0.7439492344856262, + "learning_rate": 0.0001, + "loss": 70.7286, + "step": 7570 + }, + { + "epoch": 1.2239761006035406, + "grad_norm": 0.8018355369567871, + "learning_rate": 0.0001, + "loss": 71.638, + "step": 7580 + }, + { + "epoch": 1.2255909246886416, + "grad_norm": 0.7318522930145264, + "learning_rate": 0.0001, + "loss": 70.0867, + "step": 7590 + }, + { + "epoch": 1.2272057487737429, + "grad_norm": 0.6799285411834717, + "learning_rate": 0.0001, + "loss": 71.3427, + "step": 7600 + }, + { + "epoch": 1.2288205728588442, + "grad_norm": 0.6796864867210388, + "learning_rate": 0.0001, + "loss": 71.1698, + "step": 7610 + }, + { + "epoch": 1.2304353969439454, + "grad_norm": 0.7256141304969788, + "learning_rate": 0.0001, + "loss": 71.1051, + "step": 7620 + }, + { + "epoch": 1.2320502210290467, + "grad_norm": 0.8318199515342712, + "learning_rate": 0.0001, + "loss": 71.3179, + "step": 7630 + }, + { + "epoch": 1.2336650451141478, + "grad_norm": 0.6594699025154114, + "learning_rate": 0.0001, + "loss": 70.4495, + "step": 7640 + }, + { + "epoch": 1.235279869199249, + "grad_norm": 0.7900195717811584, + "learning_rate": 0.0001, + "loss": 72.0139, + "step": 7650 + }, + { + "epoch": 1.2368946932843503, + "grad_norm": 0.7845672965049744, + "learning_rate": 0.0001, + "loss": 70.808, + "step": 7660 + }, + { + "epoch": 1.2385095173694516, + "grad_norm": 0.6598045229911804, + "learning_rate": 0.0001, + "loss": 69.8953, + "step": 7670 + }, + { + "epoch": 1.2401243414545529, + "grad_norm": 1.3222978115081787, + "learning_rate": 0.0001, + "loss": 68.787, + "step": 7680 + }, + { + "epoch": 1.241739165539654, + "grad_norm": 0.6660508513450623, + "learning_rate": 0.0001, + "loss": 71.5831, + "step": 7690 + }, + { + "epoch": 1.2433539896247552, + "grad_norm": 0.6305698156356812, + "learning_rate": 0.0001, + "loss": 70.3232, + "step": 7700 + }, + { + "epoch": 1.2449688137098565, + "grad_norm": 1.2648323774337769, + "learning_rate": 0.0001, + "loss": 67.8901, + "step": 7710 + }, + { + "epoch": 1.2465836377949577, + "grad_norm": 0.7177619934082031, + "learning_rate": 0.0001, + "loss": 69.6582, + "step": 7720 + }, + { + "epoch": 1.248198461880059, + "grad_norm": 0.6716091632843018, + "learning_rate": 0.0001, + "loss": 70.2034, + "step": 7730 + }, + { + "epoch": 1.24981328596516, + "grad_norm": 0.6050158143043518, + "learning_rate": 0.0001, + "loss": 69.3942, + "step": 7740 + }, + { + "epoch": 1.2514281100502613, + "grad_norm": 0.7001014947891235, + "learning_rate": 0.0001, + "loss": 69.5971, + "step": 7750 + }, + { + "epoch": 1.2530429341353626, + "grad_norm": 0.7221280932426453, + "learning_rate": 0.0001, + "loss": 71.3651, + "step": 7760 + }, + { + "epoch": 1.2546577582204639, + "grad_norm": 0.9130340218544006, + "learning_rate": 0.0001, + "loss": 68.3038, + "step": 7770 + }, + { + "epoch": 1.2562725823055652, + "grad_norm": 0.7703199982643127, + "learning_rate": 0.0001, + "loss": 71.6127, + "step": 7780 + }, + { + "epoch": 1.2578874063906662, + "grad_norm": 0.6334695816040039, + "learning_rate": 0.0001, + "loss": 68.9074, + "step": 7790 + }, + { + "epoch": 1.2595022304757675, + "grad_norm": 0.7074242234230042, + "learning_rate": 0.0001, + "loss": 70.0569, + "step": 7800 + }, + { + "epoch": 1.2611170545608688, + "grad_norm": 0.6956025958061218, + "learning_rate": 0.0001, + "loss": 69.892, + "step": 7810 + }, + { + "epoch": 1.26273187864597, + "grad_norm": 0.7235708236694336, + "learning_rate": 0.0001, + "loss": 71.1777, + "step": 7820 + }, + { + "epoch": 1.2643467027310713, + "grad_norm": 0.6157007813453674, + "learning_rate": 0.0001, + "loss": 71.5091, + "step": 7830 + }, + { + "epoch": 1.2659615268161724, + "grad_norm": 0.5352533459663391, + "learning_rate": 0.0001, + "loss": 70.083, + "step": 7840 + }, + { + "epoch": 1.2675763509012736, + "grad_norm": 0.7171182632446289, + "learning_rate": 0.0001, + "loss": 68.4585, + "step": 7850 + }, + { + "epoch": 1.269191174986375, + "grad_norm": 0.7503113746643066, + "learning_rate": 0.0001, + "loss": 70.4206, + "step": 7860 + }, + { + "epoch": 1.2708059990714762, + "grad_norm": 0.6552693843841553, + "learning_rate": 0.0001, + "loss": 70.8455, + "step": 7870 + }, + { + "epoch": 1.2724208231565775, + "grad_norm": 0.7173179984092712, + "learning_rate": 0.0001, + "loss": 68.82, + "step": 7880 + }, + { + "epoch": 1.2740356472416785, + "grad_norm": 0.7352942824363708, + "learning_rate": 0.0001, + "loss": 70.4974, + "step": 7890 + }, + { + "epoch": 1.2756504713267798, + "grad_norm": 0.6120197176933289, + "learning_rate": 0.0001, + "loss": 72.1638, + "step": 7900 + }, + { + "epoch": 1.277265295411881, + "grad_norm": 0.6271925568580627, + "learning_rate": 0.0001, + "loss": 69.3267, + "step": 7910 + }, + { + "epoch": 1.2788801194969823, + "grad_norm": 0.752636194229126, + "learning_rate": 0.0001, + "loss": 70.6761, + "step": 7920 + }, + { + "epoch": 1.2804949435820836, + "grad_norm": 0.7307374477386475, + "learning_rate": 0.0001, + "loss": 70.2909, + "step": 7930 + }, + { + "epoch": 1.2821097676671847, + "grad_norm": 0.5977495908737183, + "learning_rate": 0.0001, + "loss": 72.2969, + "step": 7940 + }, + { + "epoch": 1.283724591752286, + "grad_norm": 0.7927457094192505, + "learning_rate": 0.0001, + "loss": 70.3784, + "step": 7950 + }, + { + "epoch": 1.2853394158373872, + "grad_norm": 0.7520087957382202, + "learning_rate": 0.0001, + "loss": 70.7122, + "step": 7960 + }, + { + "epoch": 1.2869542399224885, + "grad_norm": 0.6036237478256226, + "learning_rate": 0.0001, + "loss": 69.8718, + "step": 7970 + }, + { + "epoch": 1.2885690640075897, + "grad_norm": 0.6805875897407532, + "learning_rate": 0.0001, + "loss": 70.2391, + "step": 7980 + }, + { + "epoch": 1.2901838880926908, + "grad_norm": 0.774559497833252, + "learning_rate": 0.0001, + "loss": 71.1605, + "step": 7990 + }, + { + "epoch": 1.291798712177792, + "grad_norm": 0.6991063356399536, + "learning_rate": 0.0001, + "loss": 69.9779, + "step": 8000 + } + ], + "logging_steps": 10, + "max_steps": 20000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 2000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.795550324511998e+19, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6a151f546555c78ebc502dd7b703b665f0672d6f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35235df8408202e91954a585851d27187bafaf4e1d75c4e63c51fbf721136ff5 +size 6840 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/zero_to_fp32.py b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-8000/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41977b2f3a9519ec9c48f4a65fa23d191bb157d0 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "A2DLlamaLMHeadModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "a2d-llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.0", + "use_cache": true, + "vocab_size": 128257 +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/generation_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f224c6aafc43cf039ad155d87799487dbf01b9 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "transformers_version": "4.57.0" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/model.safetensors b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cfd874628aafba88ac0f1a19bc9d2ae4cfefff3e --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51b6f0a49c63a173d7bf73a950a91d902caa70000a71e19a984c971efa35bf5f +size 2996990536 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/special_tokens_map.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..5419d0566af22f9412f76107e0ff63cc3598162d --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/tokenizer.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6535352bee8cb1e843f276f4f716752cbed8e402 --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba9849bc519c9fd47be755d42fdcf4862e5078395d0bb682f4faa5a46735fe9 +size 17210105 diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/tokenizer_config.json b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5f016d8ca265a5a76188b417bfe7af05ec1e536b --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/tokenizer_config.json @@ -0,0 +1,2073 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|mask|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "mask_token": "<|mask|>", + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/training_args.bin b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..6a151f546555c78ebc502dd7b703b665f0672d6f --- /dev/null +++ b/dllm-t_checkpoints_mdlm_v2/mathprosf/checkpoint-final/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35235df8408202e91954a585851d27187bafaf4e1d75c4e63c51fbf721136ff5 +size 6840